




"""simple docstring"""

_lowerCamelCase     :	List[str]         =				[
    '''Audio''',
    '''Array2D''',
    '''Array3D''',
    '''Array4D''',
    '''Array5D''',
    '''ClassLabel''',
    '''Features''',
    '''Sequence''',
    '''Value''',
    '''Image''',
    '''Translation''',
    '''TranslationVariableLanguages''',
]
from .audio import Audio
from .features import ArrayaD, ArrayaD, ArrayaD, ArrayaD, ClassLabel, Features, Sequence, Value
from .image import Image
from .translation import Translation, TranslationVariableLanguages













"""simple docstring"""





import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing






class      a						(					__snake_case		):






    def __init__(       self     :							List[Any]       ,						__SCREAMING_SNAKE_CASE     :							str = "▁"       ,						__SCREAMING_SNAKE_CASE     :							bool = True       ,						__SCREAMING_SNAKE_CASE     :							Union[str, AddedToken] = "<unk>"       ,						__SCREAMING_SNAKE_CASE     :							Union[str, AddedToken] = "</s>"       ,						__SCREAMING_SNAKE_CASE     :							Union[str, AddedToken] = "<pad>"       ,						)     ->		Optional[int]:
      lowerCamelCase_         =						{
          'pad': {'id': 0, 'token': pad_token},
          'eos': {'id': 1, 'token': eos_token},
          'unk': {'id': 2, 'token': unk_token},
      }

      lowerCamelCase_         =						[None] * len(self.special_tokens					)
      for token_dict in self.special_tokens.values():
        lowerCamelCase_         =						token_dict['token']

      lowerCamelCase_         =						Tokenizer(Unigram()					)

      lowerCamelCase_         =						normalizers.Sequence(
          [
              normalizers.Nmt(),
              normalizers.NFKC(),
              normalizers.Replace(Regex(' {2,}'					)       ,						' '					),
              normalizers.Lowercase(),
          ]					)
      lowerCamelCase_         =						pre_tokenizers.Sequence(
          [
              pre_tokenizers.Metaspace(replacement=__SCREAMING_SNAKE_CASE       ,						add_prefix_space=__SCREAMING_SNAKE_CASE					),
              pre_tokenizers.Digits(individual_digits=__SCREAMING_SNAKE_CASE					),
              pre_tokenizers.Punctuation(),
          ]					)
      lowerCamelCase_         =						decoders.Metaspace(replacement=__SCREAMING_SNAKE_CASE       ,						add_prefix_space=__SCREAMING_SNAKE_CASE					)

      lowerCamelCase_         =						TemplateProcessing(
          single=F'''$A {self.special_tokens["eos"]["token"]}'''       ,						special_tokens=[(self.special_tokens['eos']['token'], self.special_tokens['eos']['id'])]       ,						)

      lowerCamelCase_         =						{
          'model': 'SentencePieceUnigram',
          'replacement': replacement,
          'add_prefix_space': add_prefix_space,
      }

      super().__init__(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					)






    def    UpperCamelCase       (       self     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Union[str, List[str]]       ,						__SCREAMING_SNAKE_CASE     :							int = 8000       ,						__SCREAMING_SNAKE_CASE     :							bool = True       ,						)     ->		Tuple:
      lowerCamelCase_         =						trainers.UnigramTrainer(
          vocab_size=__SCREAMING_SNAKE_CASE       ,						special_tokens=self.special_tokens_list       ,						show_progress=__SCREAMING_SNAKE_CASE       ,						)

      if isinstance(__SCREAMING_SNAKE_CASE       ,						__SCREAMING_SNAKE_CASE					):
        lowerCamelCase_         =						[files]
      self._tokenizer.train(__SCREAMING_SNAKE_CASE       ,						trainer=__SCREAMING_SNAKE_CASE					)

      self.add_unk_id()






    def    UpperCamelCase       (       self     :							Tuple       ,						__SCREAMING_SNAKE_CASE     :							Union[Iterator[str], Iterator[Iterator[str]]]       ,						__SCREAMING_SNAKE_CASE     :							int = 8000       ,						__SCREAMING_SNAKE_CASE     :							bool = True       ,						)     ->		str:
      lowerCamelCase_         =						trainers.UnigramTrainer(
          vocab_size=__SCREAMING_SNAKE_CASE       ,						special_tokens=self.special_tokens_list       ,						show_progress=__SCREAMING_SNAKE_CASE       ,						)

      self._tokenizer.train_from_iterator(__SCREAMING_SNAKE_CASE       ,						trainer=__SCREAMING_SNAKE_CASE					)

      self.add_unk_id()






    def    UpperCamelCase       (       self     :							List[Any]					)     ->		Dict:
      lowerCamelCase_         =						json.loads(self._tokenizer.to_str()					)

      lowerCamelCase_         =						self.special_tokens['unk']['id']

      lowerCamelCase_         =						Tokenizer.from_str(json.dumps(__SCREAMING_SNAKE_CASE					)					)




