"""simple docstring"""






import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowerCAmelCase    :       List[Any]					     =get_tests_dir("""fixtures/test_sentencepiece.model""")





@require_sentencepiece
@require_tokenizers
class  _A						(       __a  ,    unittest.TestCase	):
      snake_case__			:   Tuple							  =	XLNetTokenizer
      snake_case__			:   Union[str, Any]							  =	XLNetTokenizerFast
      snake_case__			:   Any							  =	True
      snake_case__			:   int							  =	True

      def 				A__       (						self						):







            """simple docstring"""



            super().setUp()

            # We have a SentencePiece fixture for testing
            lowercase														=	XLNetTokenizer(_A		,       keep_accents=_A						)
            tokenizer.sanitize_special_tokens()
            tokenizer.save_pretrained(self.tmpdirname						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	'''<s>'''
            lowercase														=	1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(_A						)		,       _A						)
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(_A						)		,       _A						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	list(self.get_tokenizer().get_vocab().keys()						)

            self.assertEqual(vocab_keys[0]		,       """<unk>"""						)
            self.assertEqual(vocab_keys[1]		,       """<s>"""						)
            self.assertEqual(vocab_keys[-1]		,       """<eod>"""						)
            self.assertEqual(len(_A						)		,       1006						)

      def 				A__       (						self						):







            """simple docstring"""



            self.assertEqual(self.get_tokenizer().vocab_size		,       1000						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	XLNetTokenizer(_A		,       keep_accents=_A						)

            lowercase														=	tokenizer.tokenize("""This is a test"""						)
            self.assertListEqual(_A		,       ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]						)

            self.assertListEqual(tokenizer.convert_tokens_to_ids(_A						)		,       [285, 46, 10, 170, 382]						)

            lowercase														=	tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
            self.assertListEqual(
                _A		,       [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]		,       )
            lowercase														=	tokenizer.convert_tokens_to_ids(_A						)
            self.assertListEqual(_A		,       [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						)

            lowercase														=	tokenizer.convert_ids_to_tokens(_A						)
            self.assertListEqual(
                _A		,       [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]		,       )

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	XLNetTokenizer(_A		,       do_lower_case=_A						)
            lowercase														=	tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
            self.assertListEqual(
                _A		,       [
                    SPIECE_UNDERLINE + """""",
                    """i""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """se""",
                    """.""",
                ]		,       )
            self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""						)		,       ["""▁he""", """ll""", """o"""]						)

      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	XLNetTokenizer(_A		,       do_lower_case=_A						)
            lowercase														=	tokenizer.tokenize("""I was born in 92000, and this is falsé."""						)
            self.assertListEqual(
                _A		,       [
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """se""",
                    """.""",
                ]		,       )

      @slow
      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	XLNetTokenizer.from_pretrained("""xlnet-base-cased"""						)

            lowercase														=	tokenizer.encode("""sequence builders"""		,       add_special_tokens=_A						)
            lowercase														=	tokenizer.encode("""multi-sequence build"""		,       add_special_tokens=_A						)

            lowercase														=	tokenizer.build_inputs_with_special_tokens(_A						)
            lowercase														=	tokenizer.build_inputs_with_special_tokens(_A		,       _A						)

            assert encoded_sentence == text + [4, 3]
            assert encoded_pair == text + [4] + text_a + [4, 3]







      @slow
      def 				A__       (						self						):







            """simple docstring"""



            lowercase														=	{'''input_ids''': [[17, 2_1442, 270, 17, 10, 1_4645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 2_2018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 1_4431, 13, 5500, 11, 1176, 580, 13, 1_6819, 4797, 23, 17, 10, 1_7135, 658, 19, 457, 7932, 13, 184, 19, 3154, 1_7135, 6468, 19, 1404, 1_2269, 19, 4229, 5356, 1_6264, 46, 19, 17, 2_0545, 1_0395, 9, 9, 9, 11, 28, 6421, 9531, 2_0729, 17, 10, 353, 1_7022, 11, 21, 6421, 9531, 1_6949, 17, 10, 1_1509, 753, 11, 33, 95, 2421, 7385, 956, 1_4431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 2_4738, 19, 1_3203, 658, 218, 787, 21, 430, 1_8482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 2_2178, 27, 1064, 22, 956, 13, 1_1101, 1429, 5854, 2_4313, 1_8953, 40, 422, 2_4366, 68, 1758, 37, 1_0483, 1_4257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 1_3894, 3380, 23, 95, 18, 1_7634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=_A		,       model_name="""xlnet-base-cased"""		,       revision="""c841166438c31ec7ca9a106dee7bb312b73ae511"""		,       )







'''simple docstring'''
import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase__    					=      logging.get_logger(__name__)


UpperCamelCase__    					=      {'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt'''}

# See all BART models at https://huggingface.co/models?filter=bart
UpperCamelCase__    					=      {
    '''vocab_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/vocab.json''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/vocab.json''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''facebook/bart-base''': '''https://huggingface.co/facebook/bart-base/resolve/main/merges.txt''',
        '''facebook/bart-large''': '''https://huggingface.co/facebook/bart-large/resolve/main/merges.txt''',
        '''facebook/bart-large-mnli''': '''https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt''',
        '''facebook/bart-large-cnn''': '''https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt''',
        '''facebook/bart-large-xsum''': '''https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt''',
        '''yjernite/bart_eli5''': '''https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt''',
    },
}

UpperCamelCase__    					=      {
    '''facebook/bart-base''': 1_0_2_4,
    '''facebook/bart-large''': 1_0_2_4,
    '''facebook/bart-large-mnli''': 1_0_2_4,
    '''facebook/bart-large-cnn''': 1_0_2_4,
    '''facebook/bart-large-xsum''': 1_0_2_4,
    '''yjernite/bart_eli5''': 1_0_2_4,
}





@lru_cache()
def 					a__       (					)    ->	List[Any]:
    UpperCAmelCase__			:		int													=				(
        list(range(ord('''!'''						)  ,					ord('''~'''						) + 1						)						) + list(range(ord('''¡'''						)  ,					ord('''¬'''						) + 1						)						) + list(range(ord('''®'''						)  ,					ord('''ÿ'''						) + 1						)						)
    )
    UpperCAmelCase__			:		Optional[int]													=				bs[:]
    UpperCAmelCase__			:		List[str]													=				0
    for b in range(2**8						):
        if b not in bs:
            bs.append(lowerCAmelCase__						)
            cs.append(2**8 + n						)
            n += 1
    UpperCAmelCase__			:		Any													=				[chr(lowerCAmelCase__						) for n in cs]
    return dict(zip(lowerCAmelCase__  ,					lowerCAmelCase__						)						)





def 					a__       (					lowerCAmelCase__						)    ->	Union[str, Any]:
    UpperCAmelCase__			:		str													=				set()
    UpperCAmelCase__			:		str													=				word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)						)
        UpperCAmelCase__			:		Optional[int]													=				char
    return pairs







class 			lowerCamelCase_   (	__a      ):
      lowerCAmelCase__		     =     VOCAB_FILES_NAMES
      lowerCAmelCase__		     =     PRETRAINED_VOCAB_FILES_MAP
      lowerCAmelCase__		     =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      lowerCAmelCase__		     =     ['input_ids', 'attention_mask']






      def __init__(					self				:    Optional[int]						,						_A				:    Optional[int]						,						_A				:    List[Any]						,						_A				:    int="replace"						,						_A				:    List[Any]="<s>"						,						_A				:    List[Any]="</s>"						,						_A				:    List[Any]="</s>"						,						_A				:    Optional[int]="<s>"						,						_A				:    List[str]="<unk>"						,						_A				:    List[str]="<pad>"						,						_A				:    Union[str, Any]="<mask>"						,						_A				:    Any=False						,						**_A				:    Dict						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else bos_token
          UpperCAmelCase__			:		Any													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else eos_token
          UpperCAmelCase__			:		Optional[int]													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else sep_token
          UpperCAmelCase__			:		Tuple													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else cls_token
          UpperCAmelCase__			:		int													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else unk_token
          UpperCAmelCase__			:		Optional[Any]													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else pad_token

          # Mask token behave like a normal word, i.e. include the space before it
          UpperCAmelCase__			:		Optional[int]													=				AddedToken(_A						,						lstrip=_A						,						rstrip=_A ) if isinstance(_A						,						_A ) else mask_token

          super().__init__(
              errors=_A						,						bos_token=_A						,						eos_token=_A						,						unk_token=_A						,						sep_token=_A						,						cls_token=_A						,						pad_token=_A						,						mask_token=_A						,						add_prefix_space=_A						,						**_A						,						)

          with open(_A						,						encoding='''utf-8''' ) as vocab_handle:
              UpperCAmelCase__			:		Optional[Any]													=				json.load(_A )
          UpperCAmelCase__			:		Any													=				{v: k for k, v in self.encoder.items()}
          UpperCAmelCase__			:		List[str]													=				errors  # how to handle errors in decoding
          UpperCAmelCase__			:		str													=				bytes_to_unicode()
          UpperCAmelCase__			:		Dict													=				{v: k for k, v in self.byte_encoder.items()}
          with open(_A						,						encoding='''utf-8''' ) as merges_handle:
              UpperCAmelCase__			:		str													=				merges_handle.read().split('''\n''' )[1:-1]
          UpperCAmelCase__			:		Union[str, Any]													=				[tuple(merge.split() ) for merge in bpe_merges]
          UpperCAmelCase__			:		Optional[Any]													=				dict(zip(_A						,						range(len(_A ) ) ) )
          UpperCAmelCase__			:		Optional[int]													=				{}
          UpperCAmelCase__			:		int													=				add_prefix_space

          # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
          UpperCAmelCase__			:		List[Any]													=				re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''' )






      @property
      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          return len(self.encoder )






      def lowercase_							(					self				:    Tuple ):


          '''simple docstring'''


          return dict(self.encoder						,						**self.added_tokens_encoder )






      def lowercase_							(					self				:    List[Any]						,						_A				:    Tuple ):


          '''simple docstring'''


          if token in self.cache:
              return self.cache[token]
          UpperCAmelCase__			:		Optional[Any]													=				tuple(_A )
          UpperCAmelCase__			:		Dict													=				get_pairs(_A )

          if not pairs:
              return token

          while True:
              UpperCAmelCase__			:		Optional[Any]													=				min(_A						,						key=lambda _A : self.bpe_ranks.get(_A						,						float('''inf''' ) ) )
              if bigram not in self.bpe_ranks:
                  break
              UpperCAmelCase__						,						UpperCAmelCase__			:		str													=				bigram
              UpperCAmelCase__			:		int													=				[]
              UpperCAmelCase__			:		Tuple													=				0
              while i < len(_A ):
                  try:
                      UpperCAmelCase__			:		Optional[int]													=				word.index(_A						,						_A )
                  except ValueError:
                      new_word.extend(word[i:] )
                      break
                  else:
                      new_word.extend(word[i:j] )
                      UpperCAmelCase__			:		Tuple													=				j

                  if word[i] == first and i < len(_A ) - 1 and word[i + 1] == second:
                      new_word.append(first + second )
                      i += 2
                  else:
                      new_word.append(word[i] )
                      i += 1
              UpperCAmelCase__			:		Optional[Any]													=				tuple(_A )
              UpperCAmelCase__			:		List[Any]													=				new_word
              if len(_A ) == 1:
                  break
              else:
                  UpperCAmelCase__			:		Union[str, Any]													=				get_pairs(_A )
          UpperCAmelCase__			:		Optional[Any]													=				''' '''.join(_A )
          UpperCAmelCase__			:		List[Any]													=				word
          return word






      def lowercase_							(					self				:    str						,						_A				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				[]
          for token in re.findall(self.pat						,						_A ):
              UpperCAmelCase__			:		str													=				''''''.join(
                  self.byte_encoder[b] for b in token.encode('''utf-8''' ) )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(_A ).split(''' ''' ) )
          return bpe_tokens






      def lowercase_							(					self				:    List[str]						,						_A				:    Any ):


          '''simple docstring'''


          return self.encoder.get(_A						,						self.encoder.get(self.unk_token ) )






      def lowercase_							(					self				:    int						,						_A				:    List[str] ):


          '''simple docstring'''


          return self.decoder.get(_A )






      def lowercase_							(					self				:    Tuple						,						_A				:    Any ):


          '''simple docstring'''


          UpperCAmelCase__			:		Any													=				''''''.join(_A )
          UpperCAmelCase__			:		List[str]													=				bytearray([self.byte_decoder[c] for c in text] ).decode('''utf-8'''						,						errors=self.errors )
          return text






      def lowercase_							(					self				:    int						,						_A				:    str						,						_A				:    Optional[str] = None ):


          '''simple docstring'''


          if not os.path.isdir(_A ):
              logger.error(f"""Vocabulary path ({save_directory}) should be a directory""" )
              return
          UpperCAmelCase__			:		Tuple													=				os.path.join(
              _A						,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
          UpperCAmelCase__			:		Any													=				os.path.join(
              _A						,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

          with open(_A						,						'''w'''						,						encoding='''utf-8''' ) as f:
              f.write(json.dumps(self.encoder						,						indent=2						,						sort_keys=_A						,						ensure_ascii=_A ) + '''\n''' )

          UpperCAmelCase__			:		Union[str, Any]													=				0
          with open(_A						,						'''w'''						,						encoding='''utf-8''' ) as writer:
              writer.write('''#version: 0.2\n''' )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,						key=lambda _A : kv[1] ):
                  if index != token_index:
                      logger.warning(
                          f"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                          ''' Please check that the tokenizer is not corrupted!''' )
                      UpperCAmelCase__			:		List[str]													=				token_index
                  writer.write(''' '''.join(_A ) + '''\n''' )
                  index += 1

          return vocab_file, merge_file






      def lowercase_							(					self				:    str						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None ):


          '''simple docstring'''


          if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
          UpperCAmelCase__			:		List[str]													=				[self.cls_token_id]
          UpperCAmelCase__			:		Union[str, Any]													=				[self.sep_token_id]
          return cls + token_ids_a + sep + sep + token_ids_a + sep






      def lowercase_							(					self				:    Optional[int]						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None						,						_A				:    bool = False ):


          '''simple docstring'''


          if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=_A						,						token_ids_a=_A						,						already_has_special_tokens=_A )

          if token_ids_a is None:
              return [1] + ([0] * len(_A )) + [1]
          return [1] + ([0] * len(_A )) + [1, 1] + ([0] * len(_A )) + [1]






      def lowercase_							(					self				:    Dict						,						_A				:    List[int]						,						_A				:    Optional[List[int]] = None ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				[self.sep_token_id]
          UpperCAmelCase__			:		Dict													=				[self.cls_token_id]

          if token_ids_a is None:
              return len(cls + token_ids_a + sep ) * [0]
          return len(cls + token_ids_a + sep + sep + token_ids_a + sep ) * [0]






      def lowercase_							(					self				:    Optional[Any]						,						_A				:    Any						,						_A				:    Dict=False						,						**_A				:    Union[str, Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		Any													=				kwargs.pop('''add_prefix_space'''						,						self.add_prefix_space )
          if (is_split_into_words or add_prefix_space) and (len(_A ) > 0 and not text[0].isspace()):
              UpperCAmelCase__			:		Tuple													=				''' ''' + text
          return (text, kwargs)
