




'''simple docstring'''







import json
import os
import re
import unicodedata
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_    	=							logging.get_logger(__name__)

lowerCAmelCase_    	=							{
    'src_vocab_file': 'vocab-src.json',
    'tgt_vocab_file': 'vocab-tgt.json',
    'merges_file': 'merges.txt',
}

lowerCAmelCase_    	=							{
    'src_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-src.json'
    },
    'tgt_vocab_file': {
        'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/vocab-tgt.json'
    },
    'merges_file': {'stas/tiny-wmt19-en-de': 'https://huggingface.co/stas/tiny-wmt19-en-de/resolve/main/merges.txt'},
}

lowerCAmelCase_    	=							{'stas/tiny-wmt19-en-de': 1_0_2_4}
lowerCAmelCase_    	=							{
    'stas/tiny-wmt19-en-de': {
        'langs': ['en', 'de'],
        'model_max_length': 1_0_2_4,
        'special_tokens_map_file': None,
        'full_tokenizer_file': None,
    }
}







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   A__         =    set()
   A__         =    word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)	)
      A__         =    char
   return pairs







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   A__         =    text.replace('，'			,','	)
   A__         =    re.sub(r'。\s*'			,'. '			,UpperCAmelCase	)
   A__         =    text.replace('、'			,','	)
   A__         =    text.replace('”'			,'\"'	)
   A__         =    text.replace('“'			,'\"'	)
   A__         =    text.replace('∶'			,':'	)
   A__         =    text.replace('：'			,':'	)
   A__         =    text.replace('？'			,'?'	)
   A__         =    text.replace('《'			,'\"'	)
   A__         =    text.replace('》'			,'\"'	)
   A__         =    text.replace('）'			,')'	)
   A__         =    text.replace('！'			,'!'	)
   A__         =    text.replace('（'			,'('	)
   A__         =    text.replace('；'			,';'	)
   A__         =    text.replace('１'			,'1'	)
   A__         =    text.replace('」'			,'\"'	)
   A__         =    text.replace('「'			,'\"'	)
   A__         =    text.replace('０'			,'0'	)
   A__         =    text.replace('３'			,'3'	)
   A__         =    text.replace('２'			,'2'	)
   A__         =    text.replace('５'			,'5'	)
   A__         =    text.replace('６'			,'6'	)
   A__         =    text.replace('９'			,'9'	)
   A__         =    text.replace('７'			,'7'	)
   A__         =    text.replace('８'			,'8'	)
   A__         =    text.replace('４'			,'4'	)
   A__         =    re.sub(r'．\s*'			,'. '			,UpperCAmelCase	)
   A__         =    text.replace('～'			,'~'	)
   A__         =    text.replace('’'			,'\''	)
   A__         =    text.replace('…'			,'...'	)
   A__         =    text.replace('━'			,'-'	)
   A__         =    text.replace('〈'			,'<'	)
   A__         =    text.replace('〉'			,'>'	)
   A__         =    text.replace('【'			,'['	)
   A__         =    text.replace('】'			,']'	)
   A__         =    text.replace('％'			,'%'	)
   return text







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   A__         =    []
   for char in text:
      A__         =    unicodedata.category(UpperCAmelCase	)
      if cat.startswith('C'	):
         continue
      output.append(UpperCAmelCase	)
   return "".join(UpperCAmelCase	)







class   _snake_case( A__			):
 __snake_case:       Union[str, Any]									=				VOCAB_FILES_NAMES
 __snake_case:       Optional[Any]									=				PRETRAINED_VOCAB_FILES_MAP
 __snake_case:       Tuple									=				PRETRAINED_INIT_CONFIGURATION
 __snake_case:       int									=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case:       Any									=				['''input_ids''', '''attention_mask''']






 def __init__(self					:     str						,		a					:     List[str]=None						,		a					:     Tuple=None						,		a					:     Tuple=None						,		a					:     Dict=None						,		a					:     Any=False						,		a					:     Dict="<unk>"						,		a					:     int="<s>"						,		a					:     Optional[int]="</s>"						,		a					:     Optional[int]="<pad>"						,		**a					:     Dict						,		)					->      int:






    """simple docstring"""


    super().__init__(
        langs=_lowerCAmelCase						,		src_vocab_file=_lowerCAmelCase						,		tgt_vocab_file=_lowerCAmelCase						,		merges_file=_lowerCAmelCase						,		do_lower_case=_lowerCAmelCase						,		unk_token=_lowerCAmelCase						,		bos_token=_lowerCAmelCase						,		sep_token=_lowerCAmelCase						,		pad_token=_lowerCAmelCase						,		**_lowerCAmelCase						,		)

    try:
       import sacremoses
    except ImportError:
       raise ImportError(
           'You need to install sacremoses to use XLMTokenizer. '
           'See https://pypi.org/project/sacremoses/ for installation.'						)

    A__         =    sacremoses

    A__         =    src_vocab_file
    A__         =    tgt_vocab_file
    A__         =    merges_file
    A__         =    do_lower_case

    # cache of sm.MosesPunctNormalizer instance
    A__         =    {}
    # cache of sm.MosesTokenizer instance
    A__         =    {}
    A__         =    {}

    if langs and len(_lowerCAmelCase						) == 2:
       A__     , A__         =    langs
    else:
       raise ValueError(
           f"""arg `langs` needs to be a list of 2 langs, e.g. ['en', 'ru'], but got {langs}. """
           'Usually that means that tokenizer can\'t find a mapping for the given model path '
           'in PRETRAINED_VOCAB_FILES_MAP, and other maps of this tokenizer.'						)

    with open(_lowerCAmelCase						,		encoding='utf-8'						) as src_vocab_handle:
       A__         =    json.load(_lowerCAmelCase						)
    with open(_lowerCAmelCase						,		encoding='utf-8'						) as tgt_vocab_handle:
       A__         =    json.load(_lowerCAmelCase						)
       A__         =    {v: k for k, v in tgt_vocab.items()}
    with open(_lowerCAmelCase						,		encoding='utf-8'						) as merges_handle:
       A__         =    merges_handle.read().split('\n'						)[:-1]
    A__         =    [tuple(merge.split()[:2]						) for merge in merges]
    A__         =    dict(zip(_lowerCAmelCase						,		range(len(_lowerCAmelCase						)						)						)						)
    A__         =    {}






 def 	_UpperCamelCase   (self					:     Dict						)					->      Dict:






    """simple docstring"""


    return self.get_src_vocab()






 @property
 def 	_UpperCamelCase   (self					:     Any						)					->      List[str]:






    """simple docstring"""


    return self.src_vocab_size






 def 	_UpperCamelCase   (self					:     Union[str, Any]						,		a					:     Union[str, Any]						,		a					:     Optional[Any]						)					->      List[str]:






    """simple docstring"""


    if lang not in self.cache_moses_punct_normalizer:
       A__         =    self.sm.MosesPunctNormalizer(lang=_lowerCAmelCase						)
       A__         =    punct_normalizer
    return self.cache_moses_punct_normalizer[lang].normalize(_lowerCAmelCase						)






 def 	_UpperCamelCase   (self					:     Tuple						,		a					:     str						,		a					:     str						)					->      int:






    """simple docstring"""


    if lang not in self.cache_moses_tokenizer:
       A__         =    self.sm.MosesTokenizer(lang=_lowerCAmelCase						)
       A__         =    moses_tokenizer
    return self.cache_moses_tokenizer[lang].tokenize(
        _lowerCAmelCase						,		aggressive_dash_splits=_lowerCAmelCase						,		return_str=_lowerCAmelCase						,		escape=_lowerCAmelCase						)






 def 	_UpperCamelCase   (self					:     Any						,		a					:     str						,		a					:     Dict						)					->      Any:






    """simple docstring"""


    if lang not in self.cache_moses_detokenizer:
       A__         =    self.sm.MosesDetokenizer(lang=_lowerCAmelCase						)
       A__         =    moses_detokenizer
    return self.cache_moses_detokenizer[lang].detokenize(_lowerCAmelCase						)






 def 	_UpperCamelCase   (self					:     Union[str, Any]						,		a					:     Optional[int]						,		a					:     Tuple						)					->      Dict:






    """simple docstring"""


    A__         =    replace_unicode_punct(_lowerCAmelCase						)
    A__         =    self.moses_punct_norm(_lowerCAmelCase						,		_lowerCAmelCase						)
    A__         =    remove_non_printing_char(_lowerCAmelCase						)
    return text






 @property
 def 	_UpperCamelCase   (self					:     int						)					->      Any:






    """simple docstring"""


    return len(self.encoder						)






 @property
 def 	_UpperCamelCase   (self					:     int						)					->      Optional[Any]:






    """simple docstring"""


    return len(self.decoder						)






 def 	_UpperCamelCase   (self					:     Optional[Any]						)					->      Dict:






    """simple docstring"""


    return dict(self.encoder						,		**self.added_tokens_encoder						)






 def 	_UpperCamelCase   (self					:     int						)					->      int:






    """simple docstring"""


    return dict(self.decoder						,		**self.added_tokens_decoder						)






 def 	_UpperCamelCase   (self					:     Union[str, Any]						,		a					:     str						)					->      Tuple:






    """simple docstring"""


    A__         =    tuple(token[:-1]						) + (token[-1] + '</w>',)
    if token in self.cache:
       return self.cache[token]
    A__         =    get_pairs(_lowerCAmelCase						)

    if not pairs:
       return token + "</w>"

    while True:
       A__         =    min(_lowerCAmelCase						,		key=lambda a						: self.bpe_ranks.get(_lowerCAmelCase						,		float('inf'						)						)						)
       if bigram not in self.bpe_ranks:
          break
       A__     , A__         =    bigram
       A__         =    []
       A__         =    0
       while i < len(_lowerCAmelCase						):
          try:
             A__         =    word.index(_lowerCAmelCase						,		_lowerCAmelCase						)
          except ValueError:
             new_word.extend(word[i:]						)
             break
          else:
             new_word.extend(word[i:j]						)
             A__         =    j

          if word[i] == first and i < len(_lowerCAmelCase						) - 1 and word[i + 1] == second:
             new_word.append(first + second						)
             i += 2
          else:
             new_word.append(word[i]						)
             i += 1
       A__         =    tuple(_lowerCAmelCase						)
       A__         =    new_word
       if len(_lowerCAmelCase						) == 1:
          break
       else:
          A__         =    get_pairs(_lowerCAmelCase						)
    A__         =    ' '.join(_lowerCAmelCase						)
    if word == "\n  </w>":
       A__         =    '\n</w>'
    A__         =    word
    return word






 def 	_UpperCamelCase   (self					:     Optional[int]						,		a					:     str						,		a					:     str="en"						,		a					:     List[str]=False						)					->      List[Any]:






    """simple docstring"""


    A__         =    self.src_lang

    if self.do_lower_case:
       A__         =    text.lower()

    if bypass_tokenizer:
       A__         =    text.split()
    else:
       A__         =    self.moses_pipeline(_lowerCAmelCase						,		lang=_lowerCAmelCase						)
       A__         =    self.moses_tokenize(_lowerCAmelCase						,		lang=_lowerCAmelCase						)

    A__         =    []
    for token in text:
       if token:
          split_tokens.extend(list(self.bpe(_lowerCAmelCase						).split(' '						)						)						)

    return split_tokens






 def 	_UpperCamelCase   (self					:     str						,		a					:     Tuple						)					->      Tuple:






    """simple docstring"""


    return self.encoder.get(_lowerCAmelCase						,		self.encoder.get(self.unk_token						)						)






 def 	_UpperCamelCase   (self					:     List[Any]						,		a					:     List[Any]						)					->      List[str]:






    """simple docstring"""


    return self.decoder.get(_lowerCAmelCase						,		self.unk_token						)






 def 	_UpperCamelCase   (self					:     Optional[Any]						,		a					:     Tuple						)					->      Union[str, Any]:






    """simple docstring"""


    A__         =    [t.replace(' '						,		''						).replace('</w>'						,		' '						) for t in tokens]
    A__         =    ''.join(_lowerCAmelCase						).split()
    # detokenize
    A__         =    self.moses_detokenize(_lowerCAmelCase						,		self.tgt_lang						)
    return text






 def 	_UpperCamelCase   (self					:     Optional[Any]						,		a					:     List[int]						,		a					:     Optional[List[int]] = None						)					->      Tuple:






    """simple docstring"""


    A__         =    [self.sep_token_id]

    # no bos used in fairseq
    if token_ids_a is None:
       return token_ids_a + sep
    return token_ids_a + sep + token_ids_a + sep






 def 	_UpperCamelCase   (self					:     List[str]						,		a					:     List[int]						,		a					:     Optional[List[int]] = None						,		a					:     bool = False						)					->      Tuple:






    """simple docstring"""


    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=_lowerCAmelCase						,		token_ids_a=_lowerCAmelCase						,		already_has_special_tokens=_lowerCAmelCase						)
    # no bos used in fairseq
    if token_ids_a is not None:
       return ([0] * len(_lowerCAmelCase						)) + [1] + ([0] * len(_lowerCAmelCase						)) + [1]
    return ([0] * len(_lowerCAmelCase						)) + [1]






 def 	_UpperCamelCase   (self					:     Optional[int]						,		a					:     List[int]						,		a					:     Optional[List[int]] = None						)					->      Optional[int]:






    """simple docstring"""


    A__         =    [self.sep_token_id]

    # no bos used in fairseq
    if token_ids_a is None:
       return len(token_ids_a + sep						) * [0]
    return len(token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






 def 	_UpperCamelCase   (self					:     Optional[Any]						,		a					:     str						,		a					:     Optional[str] = None						)					->      Optional[int]:






    """simple docstring"""


    if not os.path.isdir(_lowerCAmelCase						):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
       return

    A__         =    os.path.join(
        _lowerCAmelCase						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['src_vocab_file']						)
    A__         =    os.path.join(
        _lowerCAmelCase						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['tgt_vocab_file']						)
    A__         =    os.path.join(
        _lowerCAmelCase						,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']						)

    with open(_lowerCAmelCase						,		'w'						,		encoding='utf-8'						) as f:
       f.write(json.dumps(self.encoder						,		indent=2						,		sort_keys=_lowerCAmelCase						,		ensure_ascii=_lowerCAmelCase						) + '\n'						)

    with open(_lowerCAmelCase						,		'w'						,		encoding='utf-8'						) as f:
       A__         =    {v: k for k, v in self.decoder.items()}
       f.write(json.dumps(_lowerCAmelCase						,		indent=2						,		sort_keys=_lowerCAmelCase						,		ensure_ascii=_lowerCAmelCase						) + '\n'						)

    A__         =    0
    with open(_lowerCAmelCase						,		'w'						,		encoding='utf-8'						) as writer:
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,		key=lambda a						: kv[1]						):
          if index != token_index:
             logger.warning(
                 f"""Saving vocabulary to {merges_file}: BPE merge indices are not consecutive."""
                 ' Please check that the tokenizer is not corrupted!'						)
             A__         =    token_index
          writer.write(' '.join(_lowerCAmelCase						) + '\n'						)
          index += 1

    return src_vocab_file, tgt_vocab_file, merges_file






 def __getstate__(self					:     List[Any]						)					->      List[Any]:






    """simple docstring"""


    A__         =    self.__dict__.copy()
    A__         =    None
    return state






 def __setstate__(self					:     Optional[int]						,		a					:     Tuple						)					->      Tuple:






    """simple docstring"""


    A__         =    d

    try:
       import sacremoses
    except ImportError:
       raise ImportError(
           'You need to install sacremoses to use XLMTokenizer. '
           'See https://pypi.org/project/sacremoses/ for installation.'						)

    A__         =    sacremoses










'''simple docstring'''







from __future__ import annotations

from enum import Enum, unique
from typing import TypeVar

# Create a generic variable that can be 'Enum', or any subclass.
lowerCAmelCase_    	=							TypeVar('''T''', bound='''Enum''')

@unique
class   _snake_case( UpperCAmelCase			):
 __snake_case:       List[Any]									=				80
 __snake_case:       Optional[int]									=				70
 __snake_case:       Optional[int]									=				60
 __snake_case:       int									=				50
 __snake_case:       Dict									=				40
 __snake_case:       Dict									=				30
 __snake_case:       str									=				20
 __snake_case:       Optional[int]									=				10




@unique
class   _snake_case( UpperCAmelCase			):
 __snake_case:       List[Any]									=				24
 __snake_case:       Union[str, Any]									=				21
 __snake_case:       Dict									=				18
 __snake_case:       List[str]									=				15
 __snake_case:       Union[str, Any]									=				12
 __snake_case:       Tuple									=				9
 __snake_case:       int									=				6
 __snake_case:       Tuple									=				3
 __snake_case:       List[Any]									=				2
 __snake_case:       int									=				1
 __snake_case:       List[str]									=				-1
 __snake_case:       Union[str, Any]									=				-2
 __snake_case:       List[str]									=				-3
 __snake_case:       Any									=				-6
 __snake_case:       Tuple									=				-9
 __snake_case:       List[str]									=				-12
 __snake_case:       Dict									=				-15
 __snake_case:       Optional[int]									=				-18
 __snake_case:       str									=				-21
 __snake_case:       Any									=				-24






 @classmethod
 def 	_UpperCamelCase   (cls					:     type[T]						)					->      dict:






    """simple docstring"""


    return {unit.name: unit.value for unit in cls if unit.value > 0}






 @classmethod
 def 	_UpperCamelCase   (cls					:     type[T]						)					->      dict:






    """simple docstring"""


    return {unit.name: unit.value for unit in cls if unit.value < 0}







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   A__         =    SIUnit.get_positive() if value > 0 else SIUnit.get_negative()
   for name_prefix, value_prefix in prefixes.items():
      A__         =    value / (10**value_prefix)
      if numerical_part > 1:
         return F"""{numerical_part!s} {name_prefix}"""
   return str(UpperCAmelCase	)







def 	_A			(   UpperCAmelCase	):






   '''simple docstring'''







   for prefix in BinaryUnit:
      A__         =    value / (2**prefix.value)
      if numerical_part > 1:
         return F"""{numerical_part!s} {prefix.name}"""
   return str(UpperCAmelCase	)


if __name__ == "__main__":
    import doctest

    doctest.testmod()


