code: 
  
  
  
  import json
  import os
  import re
  import warnings
  from pathlib import Path
  from shutil import copyfile
  from typing import Any, Dict, List, Optional, Tuple, Union
  
  import sentencepiece
  
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
  
  
  __lowerCamelCase  				=					logging.get_logger(__name__)
  
  __lowerCamelCase  				=					{
      """source_spm""": """source.spm""",
      """target_spm""": """target.spm""",
      """vocab""": """vocab.json""",
      """target_vocab_file""": """target_vocab.json""",
      """tokenizer_config_file""": """tokenizer_config.json""",
  }
  
  __lowerCamelCase  				=					{
      """source_spm""": {
          """Helsinki-NLP/opus-mt-en-de""": """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/source.spm"""
      },
      """target_spm""": {
          """Helsinki-NLP/opus-mt-en-de""": """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/target.spm"""
      },
      """vocab""": {
          """Helsinki-NLP/opus-mt-en-de""": """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/vocab.json"""
      },
      """tokenizer_config_file""": {
          """Helsinki-NLP/opus-mt-en-de""": (
              """https://huggingface.co/Helsinki-NLP/opus-mt-en-de/resolve/main/tokenizer_config.json"""
          )
      },
  }
  
  __lowerCamelCase  				=					{"""Helsinki-NLP/opus-mt-en-de""": 5_1_2}
  __lowerCamelCase  				=					{}
  
  
  
  class 		_snake_case      (			lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			VOCAB_FILES_NAMES
   lowerCamelCase_          =			PRETRAINED_VOCAB_FILES_MAP
   lowerCamelCase_          =			PRETRAINED_INIT_CONFIGURATION
   lowerCamelCase_          =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   lowerCamelCase_          =			['''input_ids''', '''attention_mask''']
   lowerCamelCase_          =			re.compile('''>>.+<<'''				)  # type: re.Pattern
   def __init__(						self	,    a	,    a	,    a	,    a=None	,    a=None	,    a=None	,    a="<unk>"	,    a="</s>"	,    a="<pad>"	,    a=5_1_2	,    a = None	,    a=False	,    **a	,    )	->							None:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    {} if sp_model_kwargs is None else sp_model_kwargs
  
     super().__init__(
         # bos_token=bos_token,  unused. Start decoding with config.decoder_start_token_id
         source_lang=a	,    target_lang=a	,    unk_token=a	,    eos_token=a	,    pad_token=a	,    model_max_length=a	,    sp_model_kwargs=self.sp_model_kwargs	,    target_vocab_file=a	,    separate_vocabs=a	,    **a	,    )
     assert Path(a					).exists(), f'''cannot find spm source {source_spm}'''
  
     _A           =    separate_vocabs
     _A           =    load_json(a					)
     if self.unk_token not in self.encoder:
       raise KeyError('''<unk> token must be in vocab'''					)
     assert self.pad_token in self.encoder
  
     if separate_vocabs:
       _A           =    load_json(a					)
       _A           =    {v: k for k, v in self.target_encoder.items()}
       _A           =    []
     else:
       _A           =    {v: k for k, v in self.encoder.items()}
       _A           =    [k for k in self.encoder if k.startswith('''>>'''					) and k.endswith('''<<'''					)]
  
     _A           =    source_lang
     _A           =    target_lang
     _A           =    [source_spm, target_spm]
  
     # load SentencePiece model for pre-processing
     _A           =    load_spm(a	,    self.sp_model_kwargs					)
     _A           =    load_spm(a	,    self.sp_model_kwargs					)
     _A           =    self.spm_source
     _A           =    self.encoder
  
     # Multilingual target side: default to using first supported language code.
  
     self._setup_normalizer()
   def lowercase_		(						self					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     try:
       from sacremoses import MosesPunctNormalizer
  
       _A           =    MosesPunctNormalizer(self.source_lang					).normalize
     except (ImportError, FileNotFoundError):
       warnings.warn('''Recommended: pip install sacremoses.'''					)
       _A           =    lambda a					: x
   def lowercase_		(						self	,    a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return self.punc_normalizer(a					) if x else ""
   def lowercase_		(						self	,    a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return self.current_encoder.get(a	,    self.current_encoder[self.unk_token]					)
   def lowercase_		(						self	,    a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.language_code_re.match(a					)
     _A           =    [match.group(0					)] if match else []
     return code, self.language_code_re.sub(''''''	,    a					)
   def lowercase_		(						self	,    a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A    ,   _A           =    self.remove_language_code(a					)
     _A           =    self.current_spm.encode(a	,    out_type=a					)
     return code + pieces
   def lowercase_		(						self	,    a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return self.decoder.get(a	,    self.unk_token					)
   def lowercase_		(						self	,    a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return super().batch_decode(a	,    **a					)
   def lowercase_		(						self	,    a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return super().decode(a	,    **a					)
   def lowercase_		(						self	,    a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.spm_source if self._decode_use_source_tokenizer else self.spm_target
     _A           =    []
     _A           =    ''''''
     for token in tokens:
       # make sure that special tokens are not decoded using sentencepiece model
       if token in self.all_special_tokens:
         out_string += sp_model.decode_pieces(a					) + token + " "
         _A           =    []
       else:
         current_sub_tokens.append(a					)
     out_string += sp_model.decode_pieces(a					)
     return out_string.strip()
   def lowercase_		(						self	,    a	,    a=None					)	->							List[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     if token_ids_a is None:
       return token_ids_a + [self.eos_token_id]
     # We don't expect to process pairs, but leave the pair logic for API consistency
     return token_ids_a + token_ids_a + [self.eos_token_id]
   def lowercase_		(						self					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.spm_source
     _A           =    self.encoder
   def lowercase_		(						self					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.spm_target
     if self.separate_vocabs:
       _A           =    self.target_encoder
   @property
   def lowercase_		(						self					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return len(self.encoder					)
   def lowercase_		(						self	,    a	,    a = None					)	->							Tuple[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     if not os.path.isdir(a					):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''					)
       return
     _A           =    []
  
     if self.separate_vocabs:
       _A           =    os.path.join(
           a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab''']	,    )
       _A           =    os.path.join(
           a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''target_vocab_file''']	,    )
       save_json(self.encoder	,    a					)
       save_json(self.target_encoder	,    a					)
       saved_files.append(a					)
       saved_files.append(a					)
     else:
       _A           =    os.path.join(
           a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab''']					)
       save_json(self.encoder	,    a					)
       saved_files.append(a					)
  
     for spm_save_filename, spm_orig_path, spm_model in zip(
         [VOCAB_FILES_NAMES['''source_spm'''], VOCAB_FILES_NAMES['''target_spm''']]	,    self.spm_files	,    [self.spm_source, self.spm_target]	,    ):
       _A           =    os.path.join(
           a	,    (filename_prefix + '''-''' if filename_prefix else '''''') + spm_save_filename					)
       if os.path.abspath(a					) != os.path.abspath(a					) and os.path.isfile(a					):
         copyfile(a	,    a					)
         saved_files.append(a					)
       elif not os.path.isfile(a					):
         with open(a	,    '''wb'''					) as fi:
           _A           =    spm_model.serialized_model_proto()
           fi.write(a					)
         saved_files.append(a					)
  
     return tuple(a					)
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return self.get_src_vocab()
   def lowercase_		(						self					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return dict(self.encoder	,    **self.added_tokens_encoder					)
   def lowercase_		(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return dict(self.target_encoder	,    **self.added_tokens_decoder					)
   def __getstate__(						self					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    self.__dict__.copy()
     state.update(
         {k: None for k in ['''spm_source''', '''spm_target''', '''current_spm''', '''punc_normalizer''', '''target_vocab_file''']}					)
     return state
   def __setstate__(						self	,    a					)	->							None:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    d
  
     # for backward compatibility
     if not hasattr(self	,    '''sp_model_kwargs'''					):
       _A           =    {}
  
     _A    ,   _A           =    (load_spm(a	,    self.sp_model_kwargs					) for f in self.spm_files)
     _A           =    self.spm_source
     self._setup_normalizer()
   def lowercase_		(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     return 1
   def lowercase_		(						self	,    a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     _A           =    set(self.all_special_ids					)  # call it once instead of inside list comp
     all_special_ids.remove(self.unk_token_id					)  # <unk> is only sometimes special
     return [1 if x in all_special_ids else 0 for x in seq]
  
  
  
   def lowercase_		(						self	,    a	,    a = None	,    a = False					)	->							List[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     if already_has_special_tokens:
       return self._special_token_mask(a					)
     elif token_ids_a is None:
       return self._special_token_mask(a					) + [1]
     else:
       return self._special_token_mask(token_ids_a + token_ids_a					) + [1]
  
  
  
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case							)   -> sentencepiece.SentencePieceProcessor:
    _A           =    sentencepiece.SentencePieceProcessor(**__snake_case							)
    spm.Load(__snake_case							)
    return spm
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case						,				__snake_case							)   -> None:
    with open(__snake_case						,				'''w'''							) as f:
      json.dump(__snake_case						,				__snake_case						,				indent=2							)
  
  
  
  
  def     UpperCAmelCase__ (							__snake_case							)   -> Union[Dict, List]:
    with open(__snake_case						,				'''r'''							) as f:
      return json.load(__snake_case							)
style_context: 
  
  
  
  from ..utils import DummyObject, requires_backends
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[str]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Tuple:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							List[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							str:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Dict:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[int]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Optional[Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Union[str, Any]:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							int:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)
  
  
  
  
  
  
  
  class 		_snake_case      (			metaclass=lowerCamelCase				):
  
  
  
  
  
   """simple docstring"""
  
  
  
   lowerCamelCase_          =			['''tokenizers''']
   def __init__(						self	,    *a	,    **a					)	->							Any:
  
  
  
  
     """simple docstring"""
  
  
  
  
  
  
  
     requires_backends(self	,    ['''tokenizers''']					)