import sys

import billion

regularize=billion.util.regularize

class Gaps(object):
  def __init__(self, vocab, small_limit):
    self.vocab_index = billion.util.load_vocab(vocab)
    self.small_limit = small_limit

    if False:
      for w in ['the', 'computer', 'investor', 'xNONEXISTENTx', ]:
        print w, ' -> ', self.vocab_index[w]

    if False:
      self.missing_ones = ['the', 'and', 'of', 'to', 'for', 'a', 'an', 'on', 'in', 'at', 'by', 'from', ]
      if len(self.missing_ones)>30:
        print "missing_ones list too long to pack into integer"
        
      print(["%d=%s" % (self.vocab_index[w], w) for w in self.missing_ones ], "\n")

  def generate_training(self, line):
    words = regularize(line)
    vocab_indices = [ self.vocab_index[w] for w in words ]
    
    #print "Words in Line : %d" % (len(words),)
    
    for i in range(len(words)-2):
      if vocab_indices[i] is None or vocab_indices[i+1] is None or vocab_indices[i+2] is None:
        continue
        
      # Middle word missing :: So ans>0
      x = [ vocab_indices[i], vocab_indices[i+2] ]
        
      # Pick out small words for 'easy' identification
      missing = vocab_indices[i+1]
      ans = (missing+2) if missing<self.small_limit else 1
      
      # So, ans==1 if this is a 'complex' word
      # small_limit+2>ans>1 if this is a 'simple' word
      # i.e. ans>0  => there is some word missing
      #      ans==0 => no word missing
      
      yield (x, ans)
    
    for i in range(len(words)-1):
      if vocab_indices[i] is None or vocab_indices[i+1] is None:
        continue
        
      # Word not missing :: So ans==0
      x = [ vocab_indices[i], vocab_indices[i+1] ]
      
      yield (x, 0)

"""
for l, line in enumerate(inputfile):  
  if 0 == l % 10000:
    billion.util.print_thousands("Line # ", l)
    # Skip this iter - since it's going into our 'holdout set'
    continue
"""

@@ -864,32 +864,33 @@
 ords)-2):%0A      
+#
 if vocab_indices
@@ -960,32 +960,33 @@
  is None:%0A      
+#
   continue%0A     
@@ -1082,24 +1082,229 @@
 +2%5D %5D%0A      
+%0A      # This may loop around to end/beginning of sentence...%0A      #x = %5B vocal_indices%5Bi-1%5D, vocab_indices%5Bi%5D, vocab_indices%5Bi+2%5D, vocab_indices%5Bi+3%5D %5D%0A        %0A      if None in x: %0A        continue%0A    
   %0A      # P
@@ -1714,16 +1714,17 @@
 :%0A      
+#
 if vocab
@@ -1776,24 +1776,25 @@
 None:%0A      
+#
   continue%0A 
@@ -1889,32 +1889,239 @@
 s%5Bi+1%5D %5D%0A      %0A
+      # This may loop around to end/beginning of sentence...%0A      #x = %5B vocab_indices%5Bi-1%5D, vocab_indices%5Bi%5D, vocab_indices%5Bi+1%5D, vocab_indices%5Bi+2%5D %5D%0A      %0A      if None in x: %0A        continue%0A        %0A
       yield (x, 

Add code for long window : -2,-1,+1,+2