<jupyter_start><jupyter_text><jupyter_code>from keras.preprocessing.text import Tokenizer, text_to_word_sequence
from keras.preprocessing.sequence import skipgrams

text = "I love green eggs and ham ."

tokenizer = Tokenizer()
tokenizer.fit_on_texts([text])

word2id = tokenizer.word_index
id2word = {v: k for k, v in word2id.items()}

wids = [word2id[w] for w in text_to_word_sequence(text)]
pairs, labels = skipgrams(wids, len(word2id))
print(len(pairs), len(labels))
for i in range(10):
    print("({:s} ({:d}), {:s} ({:d})) -> {:d}".format(
        id2word[pairs[i][0]], pairs[i][0],
        id2word[pairs[i][1]], pairs[i][1],
        labels[i]))<jupyter_output>56 56
(love (2), green (3)) -> 1
(love (2), green (3)) -> 0
(ham (6), love (2)) -> 1
(eggs (4), ham (6)) -> 1
(and (5), and (5)) -> 0
(i (1), love (2)) -> 1
(eggs (4), eggs (4)) -> 0
(and (5), love (2)) -> 1
(ham (6), i (1)) -> 0
(ham (6), green (3)) -> 0
StarcoderdataJupyterStructuredCleanDedup