import io
from collections import defaultdict

def remove_comprising(words):
    seen = set()
    result_words = []
    for word in words:
        for small_word in words:
            if small_word in word and small_word != word:
                if small_word in seen:
                    word = word.replace(small_word, '')
                else:
                    seen.add(small_word)
        result_words.append(word)
    return [word.strip() for word in result_words if word not in seen]

dictionary = defaultdict(list)
with io.open('JST.enja') as dicfin:
    for line in dicfin:
        en, ja = line.strip().split('\t')
        dictionary[ja].append(en)


fout = io.open('test.pervasive.ja', 'w')
with io.open('test.ja') as testfin:
    for line in testfin:
        substitutes = {}

        for i in dictionary:
            if i in line and i.split()[0] in line.split():
                substitutes[i] = "||".join(dictionary[i])

        toreplace = remove_comprising(substitutes.keys())
        for i in sorted(toreplace, key=len):
            try:
                if line.index(i)-1 not in ['>','|'] and line.index(i)+1 not in ["=", "|", "_"] and len(i) > 3:
                    x = substitutes[i].replace(' ', '_')
                    numy = len(x.split('||'))
                    y = "||".join([str(1/float(numy))]*numy)
                    z = i
                    line = line.replace(i, '''<n translation="'''+x+'''" prob="'''+y+'''">'''+z+'''</n>''') #; print line
            except:
                continue
        fout.write(line.strip()+'\n')

