class Vocab:
    def __init__(self, data_reader):
        self._data_reader = data_reader

        self._char2index = {}
        self._index2char = []

        self._part2index = {}
        self._index2part = []

        self._loaded = False

    def _feed_char(self, char):
        if char not in self._char2index:
            index = len(self._char2index)
            self._char2index[char] = index
            self._index2char.append(char)

    def _feed_speech_part(self, speech_part):
        if speech_part not in self._part2index:
            index = len(self._part2index)
            self._part2index[speech_part] = index
            self._index2part.append(speech_part)

    def _load_initial_chars(self):
        self._feed_char(' ')
        self._feed_char('{')
        self._feed_char('}')

    def _load_initial_parts(self):
        self._feed_speech_part('UNKNOWN')

    def _load_chars(self):
        uniq_chars = self._data_reader.get_uniq_chars()
        self._load_initial_chars()
        for c in sorted(uniq_chars):
            self._feed_char(c)

    def _load_speech_parts(self):
        uniq_speech_parts = self._data_reader.get_uniq_speech_parts()
        self._load_initial_parts()
        for part in sorted(uniq_speech_parts):
            self._feed_speech_part(part)

    def load(self):
        self._load_chars()
        self._load_speech_parts()
        self._loaded = True

    def char_to_index(self, char):
        if not self._loaded:
            raise BaseException('chars not loaded')
        return self._char2index[char]

    def part_to_index(self, speech_part):
        if not self._loaded:
            raise BaseException('chars not loaded')
        return self._part2index[speech_part]

    def char_vocab_size(self):
        return len(self._index2char)

    def part_vocab_size(self):
        return len(self._index2part)

@@ -1,16 +1,51 @@
-class Vocab:
+import math%0A%0Aclass Vocab:%0A    EMPTY_CHAR = ' '%0A
 %0A   
@@ -237,24 +237,96 @@
 2part = %5B%5D%0A%0A
+        self._char_freq_threshold = 0%0A        self._skipped_chars = %5B%5D%0A%0A
         self
@@ -367,32 +367,92 @@
 ar(self, char):%0A
+        if char in self._skipped_chars:%0A            return%0A%0A
         if char 
@@ -899,19 +899,31 @@
 ed_char(
-' '
+self.EMPTY_CHAR
 )%0A      
@@ -1480,19 +1480,539 @@
 def 
-load(self):
+_calculate_char_freq_threshold(self):%0A         _char, max_freq = self._data_reader.get_chars_freq()%5B-1%5D%0A         self._char_freq_threshold = math.ceil(max_freq * 0.01) # 1%25 of max freshold%0A%0A    def _find_low_freq_chars(self):%0A        self._calculate_char_freq_threshold()%0A%0A        for char, freq in self._data_reader.get_chars_freq():%0A            if freq %3C= self._char_freq_threshold:%0A                self._skipped_chars.append(char)%0A            else:%0A                break%0A%0A    def load(self):%0A        self._find_low_freq_chars()%0A
 %0A   
@@ -2061,32 +2061,33 @@
 _speech_parts()%0A
+%0A
         self._lo
@@ -2246,21 +2246,62 @@
 ar2index
-%5Bchar
+.get(char) or self._char2index%5Bself.EMPTY_CHAR
 %5D%0A%0A    d
@@ -2597,12 +2597,408 @@
 index2part)%0A
+%0Aif __name__ == '__main__':%0A    # debug%0A    from data_reader import DataReader%0A    SENTENCES_SOURCE = 'data/sentences.xml'%0A%0A    data_reader = DataReader(SENTENCES_SOURCE)%0A    data_reader.load()%0A%0A    vocab = Vocab(data_reader)%0A    vocab.load()%0A%0A    print('skipped chars threshold', vocab._char_freq_threshold)%0A    print('skipped chars', vocab._skipped_chars)%0A    print('vocab', vocab._char2index)%0A

Replace low freq chars with empty chars.