from collections import defaultdict, Counter
from itertools import chain, takewhile

from config import START_TOKEN, END_TOKEN, UNKNOWN_TOKEN

class Vocabulary:
    def __new__(cls, words_generator, th):
        obj = super().__new__(cls)

        word_freq = Counter(chain.from_iterable((words_generator)))

        obj.__wtoi = defaultdict(lambda: 0)
        obj.__wtoi[UNKNOWN_TOKEN] = 0
        obj.__wtoi[START_TOKEN] = 1
        obj.__wtoi[END_TOKEN] = 2
        obj.__itow = []
        obj.__itow.append(UNKNOWN_TOKEN)
        obj.__itow.append(START_TOKEN)
        obj.__itow.append(END_TOKEN)

        i = 3
        for (k, v) in takewhile(lambda x: x[1] > th, word_freq.most_common()):
            obj.__wtoi[k] = i
            obj.__itow.append(k)
            i += 1

        return obj

    def __len__(self):
        return len(self.itow)

    @property
    def wtoi(self):
        return self.__wtoi

    @property
    def itow(self):
        return self.__itow

@@ -194,16 +194,18 @@
 ator, th
+=5
 ):%0A     
@@ -312,26 +312,24 @@
         obj.
-__
 wtoi = defau
@@ -354,26 +354,24 @@
         obj.
-__
 wtoi%5BUNKNOWN
@@ -390,26 +390,24 @@
         obj.
-__
 wtoi%5BSTART_T
@@ -424,26 +424,24 @@
         obj.
-__
 wtoi%5BEND_TOK
@@ -456,26 +456,24 @@
         obj.
-__
 itow = %5B%5D%0A  
@@ -474,34 +474,32 @@
  %5B%5D%0A        obj.
-__
 itow.append(UNKN
@@ -513,34 +513,32 @@
 EN)%0A        obj.
-__
 itow.append(STAR
@@ -550,34 +550,32 @@
 EN)%0A        obj.
-__
 itow.append(END_
@@ -691,18 +691,16 @@
     obj.
-__
 wtoi%5Bk%5D 
@@ -719,18 +719,16 @@
     obj.
-__
 itow.app
@@ -831,128 +831,4 @@
 ow)%0A
-%0A    @property%0A    def wtoi(self):%0A        return self.__wtoi%0A%0A    @property%0A    def itow(self):%0A        return self.__itow%0A

update Vocabulary class