






import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__		:       Any   							=							logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__		:       Optional[int]   							=							{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

SCREAMING_SNAKE_CASE__		:       Optional[int]   							=							{
    """vocab_file""": {
        """microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json""",
    },
    """merges_file""": {"""microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"""},
}

SCREAMING_SNAKE_CASE__		:       Tuple   							=							{
    """microsoft/biogpt""": 10_24,
}





def    _lowerCamelCase	(	__lowerCamelCase					)   -> str:

     '''simple docstring'''
     UpperCAmelCase__			:     str										= set()
     UpperCAmelCase__			:     str										= word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)					)
          UpperCAmelCase__			:     Optional[Any]										= char
     return pairs







class 							UpperCAmelCase_     (		__lowerCamelCase		):
     __lowerCamelCase										=     VOCAB_FILES_NAMES
     __lowerCamelCase										=     PRETRAINED_VOCAB_FILES_MAP
     __lowerCamelCase										=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __lowerCamelCase										=     ['input_ids', 'attention_mask']




     def __init__(							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase  ,	_lowerCAmelCase="<unk>"  ,	_lowerCAmelCase="<s>"  ,	_lowerCAmelCase="</s>"  ,	_lowerCAmelCase="</s>"  ,	_lowerCAmelCase="<pad>"  ,	**_lowerCAmelCase  ,	):
          super().__init__(
              bos_token=_lowerCAmelCase  ,	eos_token=_lowerCAmelCase  ,	sep_token=_lowerCAmelCase  ,	unk_token=_lowerCAmelCase  ,	pad_token=_lowerCAmelCase  ,	**_lowerCAmelCase  ,	)

          try:
               import sacremoses
          except ImportError:
               raise ImportError(
                   """You need to install sacremoses to use BioGptTokenizer. """
                   """See https://pypi.org/project/sacremoses/ for installation."""		)

          UpperCAmelCase__			:     str										= """en"""
          UpperCAmelCase__			:     str										= sacremoses
          # cache of sm.MosesTokenizer instance
          UpperCAmelCase__			:     List[str]										= {}
          UpperCAmelCase__			:     Optional[Any]										= {}
          with open(_lowerCAmelCase  ,	encoding="""utf-8"""		) as vocab_handle:
               UpperCAmelCase__			:     List[Any]										= json.load(_lowerCAmelCase		)
          UpperCAmelCase__			:     List[Any]										= {v: k for k, v in self.encoder.items()}
          with open(_lowerCAmelCase  ,	encoding="""utf-8"""		) as merges_handle:
               UpperCAmelCase__			:     Any										= merges_handle.read().split("""\n"""		)[:-1]
          UpperCAmelCase__			:     Union[str, Any]										= [tuple(merge.split()[:2]		) for merge in merges]
          UpperCAmelCase__			:     List[str]										= dict(zip(_lowerCAmelCase  ,	range(len(_lowerCAmelCase		)		)		)		)
          UpperCAmelCase__			:     Any										= {}




     @property
     def      __UpperCAmelCase     (							self		):
          return len(self.encoder		)




     def      __UpperCAmelCase     (							self		):
          return dict(self.encoder  ,	**self.added_tokens_encoder		)




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
          if lang not in self.cache_moses_tokenizer:
               UpperCAmelCase__			:     Dict										= self.sm.MosesTokenizer(lang=_lowerCAmelCase		)
               UpperCAmelCase__			:     Dict										= moses_tokenizer
          return self.cache_moses_tokenizer[lang].tokenize(
              _lowerCAmelCase  ,	aggressive_dash_splits=_lowerCAmelCase  ,	return_str=_lowerCAmelCase  ,	escape=_lowerCAmelCase		)




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
          if lang not in self.cache_moses_detokenizer:
               UpperCAmelCase__			:     Optional[Any]										= self.sm.MosesDetokenizer(lang=_lowerCAmelCase		)
               UpperCAmelCase__			:     Tuple										= moses_detokenizer
          return self.cache_moses_detokenizer[lang].detokenize(_lowerCAmelCase		)




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
          UpperCAmelCase__			:     Tuple										= tuple(token[:-1]		) + (token[-1] + """</w>""",)
          if token in self.cache:
               return self.cache[token]
          UpperCAmelCase__			:     Union[str, Any]										= get_pairs(_lowerCAmelCase		)

          if not pairs:
               return token + "</w>"

          while True:
               UpperCAmelCase__			:     Union[str, Any]										= min(_lowerCAmelCase  ,	key=lambda _lowerCAmelCase		: self.bpe_ranks.get(_lowerCAmelCase  ,	float("""inf"""		)		)		)
               if bigram not in self.bpe_ranks:
                    break
               UpperCAmelCase__			,		UpperCAmelCase__			:     List[Any]										= bigram
               UpperCAmelCase__			:     Any										= []
               UpperCAmelCase__			:     List[str]										= 0
               while i < len(_lowerCAmelCase		):
                    try:
                         UpperCAmelCase__			:     List[str]										= word.index(_lowerCAmelCase  ,	_lowerCAmelCase		)
                    except ValueError:
                         new_word.extend(word[i:]		)
                         break
                    else:
                         new_word.extend(word[i:j]		)
                         UpperCAmelCase__			:     Any										= j

                    if word[i] == first and i < len(_lowerCAmelCase		) - 1 and word[i + 1] == second:
                         new_word.append(first + second		)
                         i += 2
                    else:
                         new_word.append(word[i]		)
                         i += 1
               UpperCAmelCase__			:     List[str]										= tuple(_lowerCAmelCase		)
               UpperCAmelCase__			:     List[Any]										= new_word
               if len(_lowerCAmelCase		) == 1:
                    break
               else:
                    UpperCAmelCase__			:     List[Any]										= get_pairs(_lowerCAmelCase		)
          UpperCAmelCase__			:     Any										= """ """.join(_lowerCAmelCase		)
          if word == "\n  </w>":
               UpperCAmelCase__			:     Optional[Any]										= """\n</w>"""
          UpperCAmelCase__			:     Union[str, Any]										= word
          return word




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase=False		):
          if bypass_tokenizer:
               UpperCAmelCase__			:     List[str]										= text.split()
          else:
               UpperCAmelCase__			:     str										= self.moses_tokenize(_lowerCAmelCase  ,	self.lang		)

          UpperCAmelCase__			:     Union[str, Any]										= []
          for token in text:
               if token:
                    split_tokens.extend(list(self.bpe(_lowerCAmelCase		).split(""" """		)		)		)

          return split_tokens




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
          return self.encoder.get(_lowerCAmelCase  ,	self.encoder.get(self.unk_token		)		)




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
          return self.decoder.get(_lowerCAmelCase  ,	self.unk_token		)




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
          UpperCAmelCase__			:     List[str]										= [t.replace(""" """  ,	""""""		).replace("""</w>"""  ,	""" """		) for t in tokens]
          UpperCAmelCase__			:     Any										= """""".join(_lowerCAmelCase		).split()
          # detokenize
          UpperCAmelCase__			:     List[str]										= self.moses_detokenize(_lowerCAmelCase  ,	self.lang		)
          return text




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None		):
          if token_ids_a is None:
               return [self.sep_token_id] + token_ids_a
          UpperCAmelCase__			:     Union[str, Any]										= [self.sep_token_id]
          return sep + token_ids_a + sep + token_ids_a




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None  ,	_lowerCAmelCase = False		):
          if already_has_special_tokens:
               return super().get_special_tokens_mask(
                   token_ids_a=_lowerCAmelCase  ,	token_ids_a=_lowerCAmelCase  ,	already_has_special_tokens=_lowerCAmelCase		)
          # no bos used in fairseq
          if token_ids_a is not None:
               return [1] + ([0] * len(_lowerCAmelCase		)) + [1] + ([0] * len(_lowerCAmelCase		))
          return [1] + ([0] * len(_lowerCAmelCase		))




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None		):
          UpperCAmelCase__			:     Optional[int]										= [self.sep_token_id]

          # no bos used in fairseq
          if token_ids_a is None:
               return len(token_ids_a + sep		) * [0]
          return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None		):
          if not os.path.isdir(_lowerCAmelCase		):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
               return
          UpperCAmelCase__			:     Tuple										= os.path.join(
              _lowerCAmelCase  ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
          UpperCAmelCase__			:     List[str]										= os.path.join(
              _lowerCAmelCase  ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)

          with open(_lowerCAmelCase  ,	"""w"""  ,	encoding="""utf-8"""		) as f:
               f.write(json.dumps(self.encoder  ,	indent=2  ,	sort_keys=_lowerCAmelCase  ,	ensure_ascii=_lowerCAmelCase		) + """\n"""		)

          UpperCAmelCase__			:     Dict										= 0
          with open(_lowerCAmelCase  ,	"""w"""  ,	encoding="""utf-8"""		) as writer:
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,	key=lambda _lowerCAmelCase		: kv[1]		):
                    if index != token_index:
                         logger.warning(
                             f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                             """ Please check that the tokenizer is not corrupted!"""		)
                         UpperCAmelCase__			:     Dict										= token_index
                    writer.write(""" """.join(_lowerCAmelCase		) + """\n"""		)
                    index += 1

          return vocab_file, merge_file




     def __getstate__(							self		):
          UpperCAmelCase__			:     List[Any]										= self.__dict__.copy()
          UpperCAmelCase__			:     Dict										= None
          return state




     def __setstate__(							self  ,	_lowerCAmelCase		):
          UpperCAmelCase__			:     Optional[Any]										= d

          try:
               import sacremoses
          except ImportError:
               raise ImportError(
                   """You need to install sacremoses to use XLMTokenizer. """
                   """See https://pypi.org/project/sacremoses/ for installation."""		)

          UpperCAmelCase__			:     Tuple										= sacremoses














import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 							UpperCAmelCase_     (		__lowerCamelCase       ,			unittest.TestCase		):
     __lowerCamelCase										=     PreTrainedTokenizerFast
     __lowerCamelCase										=     False
     __lowerCamelCase										=     True
     __lowerCamelCase										=     'tokenizer_file'




     def      __UpperCAmelCase     (							self		):
          UpperCAmelCase__			:     Union[str, Any]										= False  # because we don't have pretrained_vocab_files_map
          super().setUp()
          UpperCAmelCase__			:     Dict										= True

          UpperCAmelCase__			:     int										= ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
          UpperCAmelCase__			:     List[Any]										= """SaulLu/dummy-tokenizer-bytelevel-bpe"""

          # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
          UpperCAmelCase__			:     int										= [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

          UpperCAmelCase__			:     Union[str, Any]										= PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
          tokenizer.save_pretrained(self.tmpdirname		)




     def      __UpperCAmelCase     (							self		):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass




     def      __UpperCAmelCase     (							self		):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass




     def      __UpperCAmelCase     (							self		):
          # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
          # model
          pass




     def      __UpperCAmelCase     (							self		):
          # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
          pass




     def      __UpperCAmelCase     (							self		):
          UpperCAmelCase__			:     Optional[int]										= self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                    try:
                         UpperCAmelCase__			:     str										= tempfile.mkdtemp()
                         UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)

                         tokenizer.save_pretrained(self.tmpdirname		)
                         super().test_training_new_tokenizer()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname		)
                         UpperCAmelCase__			:     str										= tmpdirname_orig




     def      __UpperCAmelCase     (							self		):
          UpperCAmelCase__			:     Tuple										= self.tmpdirname
          # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                    try:
                         UpperCAmelCase__			:     Optional[Any]										= tempfile.mkdtemp()
                         UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)

                         tokenizer.save_pretrained(self.tmpdirname		)
                         super().test_training_new_tokenizer_with_special_tokens_change()
                    finally:
                         # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                         # is restored
                         shutil.rmtree(self.tmpdirname		)
                         UpperCAmelCase__			:     str										= tmpdirname_orig




     def      __UpperCAmelCase     (							self		):
          UpperCAmelCase__			:     Tuple										= self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)

          UpperCAmelCase__			:     Dict										= ("""a""" for _ in range(1000		))
          UpperCAmelCase__			:     int										= tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase  ,	length=1000  ,	vocab_size=50		)

          UpperCAmelCase__			:     Optional[Any]										= new_tokenizer.encode("""a🤗"""		)
          self.assertEqual(_lowerCAmelCase  ,	[64, 172, 253, 97, 245]		)







@require_tokenizers
class 							UpperCAmelCase_     (		unittest.TestCase		):




     def      __UpperCAmelCase     (							self		):
          UpperCAmelCase__			:     List[str]										= AutoTokenizer.from_pretrained("""bert-base-cased"""		)
          UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
          UpperCAmelCase__			:     Optional[Any]										= len(_lowerCAmelCase		)

          with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               UpperCAmelCase__			:     Tuple										= ["""tokenizer.4.0.0.json"""]
               tokenizer.save_pretrained(_lowerCAmelCase		)
               json.dump(_lowerCAmelCase  ,	open(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	"""w"""		)		)

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               UpperCAmelCase__			:     Optional[int]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
               self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		) + 1		)
               UpperCAmelCase__			:     Any										= json.loads(new_tokenizer._tokenizer.to_str()		)
               self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	os.path.join(_lowerCAmelCase  ,	"""tokenizer.42.0.0.json"""		)		)
               UpperCAmelCase__			:     Union[str, Any]										= ["""tokenizer.42.0.0.json"""]
               tokenizer.save_pretrained(_lowerCAmelCase		)
               UpperCAmelCase__			:     Optional[Any]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
               self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		)		)
               UpperCAmelCase__			:     List[Any]										= json.loads(new_tokenizer._tokenizer.to_str()		)
               self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)




     def      __UpperCAmelCase     (							self		):
          # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
          UpperCAmelCase__			:     List[str]										= """hf-internal-testing/test-two-tokenizers"""

          # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
          UpperCAmelCase__			:     int										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
          self.assertEqual(len(_lowerCAmelCase		)  ,	28997		)
          UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
          self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)

          # Testing an older version by monkey-patching the version in the module it's used.
          import transformers as old_transformers

          UpperCAmelCase__			:     Any										= """3.0.0"""
          UpperCAmelCase__			:     Optional[int]										= old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase		)
          self.assertEqual(len(_lowerCAmelCase		)  ,	28996		)
          UpperCAmelCase__			:     Optional[int]										= json.loads(old_tokenizer._tokenizer.to_str()		)
          self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)







@require_tokenizers
class 							UpperCAmelCase_     (		unittest.TestCase		):




     def      __UpperCAmelCase     (							self		):
          # See https://github.com/huggingface/transformers/pull/12550
          # and https://github.com/huggingface/tokenizers/issues/537
          UpperCAmelCase__			:     int										= PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""		)
          UpperCAmelCase__			:     List[str]										= """The Matrix is a 1999 science fiction action film."""

          with concurrent.futures.ThreadPoolExecutor() as executor:
               UpperCAmelCase__			:     List[str]										= [executor.submit(self.fetch  ,	_lowerCAmelCase  ,	_lowerCAmelCase		) for i in range(10		)]
               UpperCAmelCase__			:     Dict										= [future.result() for future in futures]
               self.assertEqual(_lowerCAmelCase  ,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)




     def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
          return tokenizer.encode(_lowerCAmelCase  ,	truncation="""longest_first"""  ,	padding="""longest"""		)






