@@ -0,0 +1,1157 @@
+%0Afrom itertools import islice%0A%0Afrom .embeddings import text_to_pairs%0A%0A%0Adef iter_pairs(fin, vocab, batch_size=10, nsamples=2, window=5):%0A    '''%0A    Convert a document stream to batches of pairs used for training embeddings.%0A%0A    iter_pairs is a generator that yields batches of pairs that can%0A    be passed to GaussianEmbedding.train%0A%0A    fin = an iterator of documents / sentences (e.g. a file like object)%0A        Each element is a string of raw text%0A    vocab = something implementing the Vocabulary interface%0A    batch_size = size of batches%0A%0A    window = Number of words to the left and right of center word to include%0A        as positive pairs%0A    nsamples = number of negative samples to drawn for each center word%0A    '''%0A    documents = iter(fin)%0A    batch = list(islice(documents, batch_size))%0A    while len(batch) %3E 0:%0A        text = %5B%0A            vocab.tokenize(doc, remove_oov=False, return_ids=True)%0A            for doc in batch%0A        %5D%0A        pairs = text_to_pairs(text, vocab.random_ids,%0A            nsamples_per_word=nsamples,%0A            half_window_size=window)%0A        yield pairs%0A        batch = list(islice(documents, batch_size))%0A%0A

Add words.py