code: 
  
  
  
  
  
  
  import json
  import os
  from typing import List, Optional, Tuple
  
  from ...tokenization_utils import PreTrainedTokenizer
  from ...utils import logging
  
  
  SCREAMING_SNAKE_CASE__		:       Any   							=							logging.get_logger(__name__)
  
  SCREAMING_SNAKE_CASE__		:       Optional[int]   							=							{
      """vocab_file""": """vocab.json""",
      """merges_file""": """merges.txt""",
  }
  
  SCREAMING_SNAKE_CASE__		:       Optional[int]   							=							{
      """vocab_file""": {
          """microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json""",
      },
      """merges_file""": {"""microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"""},
  }
  
  SCREAMING_SNAKE_CASE__		:       Tuple   							=							{
      """microsoft/biogpt""": 10_24,
  }
  
  
  
  
  
  def    _lowerCamelCase	(	__lowerCamelCase					)   -> str:
  
       '''simple docstring'''
       UpperCAmelCase__			:     str										= set()
       UpperCAmelCase__			:     str										= word[0]
       for char in word[1:]:
            pairs.add((prev_char, char)					)
            UpperCAmelCase__			:     Optional[Any]										= char
       return pairs
  
  
  
  
  
  
  
  class 							UpperCAmelCase_     (		__lowerCamelCase		):
       __lowerCamelCase										=     VOCAB_FILES_NAMES
       __lowerCamelCase										=     PRETRAINED_VOCAB_FILES_MAP
       __lowerCamelCase										=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowerCamelCase										=     ['input_ids', 'attention_mask']
  
  
  
  
       def __init__(							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase  ,	_lowerCAmelCase="<unk>"  ,	_lowerCAmelCase="<s>"  ,	_lowerCAmelCase="</s>"  ,	_lowerCAmelCase="</s>"  ,	_lowerCAmelCase="<pad>"  ,	**_lowerCAmelCase  ,	):
            super().__init__(
                bos_token=_lowerCAmelCase  ,	eos_token=_lowerCAmelCase  ,	sep_token=_lowerCAmelCase  ,	unk_token=_lowerCAmelCase  ,	pad_token=_lowerCAmelCase  ,	**_lowerCAmelCase  ,	)
  
            try:
                 import sacremoses
            except ImportError:
                 raise ImportError(
                     """You need to install sacremoses to use BioGptTokenizer. """
                     """See https://pypi.org/project/sacremoses/ for installation."""		)
  
            UpperCAmelCase__			:     str										= """en"""
            UpperCAmelCase__			:     str										= sacremoses
            # cache of sm.MosesTokenizer instance
            UpperCAmelCase__			:     List[str]										= {}
            UpperCAmelCase__			:     Optional[Any]										= {}
            with open(_lowerCAmelCase  ,	encoding="""utf-8"""		) as vocab_handle:
                 UpperCAmelCase__			:     List[Any]										= json.load(_lowerCAmelCase		)
            UpperCAmelCase__			:     List[Any]										= {v: k for k, v in self.encoder.items()}
            with open(_lowerCAmelCase  ,	encoding="""utf-8"""		) as merges_handle:
                 UpperCAmelCase__			:     Any										= merges_handle.read().split("""\n"""		)[:-1]
            UpperCAmelCase__			:     Union[str, Any]										= [tuple(merge.split()[:2]		) for merge in merges]
            UpperCAmelCase__			:     List[str]										= dict(zip(_lowerCAmelCase  ,	range(len(_lowerCAmelCase		)		)		)		)
            UpperCAmelCase__			:     Any										= {}
  
  
  
  
       @property
       def      __UpperCAmelCase     (							self		):
            return len(self.encoder		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            return dict(self.encoder  ,	**self.added_tokens_encoder		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            if lang not in self.cache_moses_tokenizer:
                 UpperCAmelCase__			:     Dict										= self.sm.MosesTokenizer(lang=_lowerCAmelCase		)
                 UpperCAmelCase__			:     Dict										= moses_tokenizer
            return self.cache_moses_tokenizer[lang].tokenize(
                _lowerCAmelCase  ,	aggressive_dash_splits=_lowerCAmelCase  ,	return_str=_lowerCAmelCase  ,	escape=_lowerCAmelCase		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            if lang not in self.cache_moses_detokenizer:
                 UpperCAmelCase__			:     Optional[Any]										= self.sm.MosesDetokenizer(lang=_lowerCAmelCase		)
                 UpperCAmelCase__			:     Tuple										= moses_detokenizer
            return self.cache_moses_detokenizer[lang].detokenize(_lowerCAmelCase		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
            UpperCAmelCase__			:     Tuple										= tuple(token[:-1]		) + (token[-1] + """</w>""",)
            if token in self.cache:
                 return self.cache[token]
            UpperCAmelCase__			:     Union[str, Any]										= get_pairs(_lowerCAmelCase		)
  
            if not pairs:
                 return token + "</w>"
  
            while True:
                 UpperCAmelCase__			:     Union[str, Any]										= min(_lowerCAmelCase  ,	key=lambda _lowerCAmelCase		: self.bpe_ranks.get(_lowerCAmelCase  ,	float("""inf"""		)		)		)
                 if bigram not in self.bpe_ranks:
                      break
                 UpperCAmelCase__			,		UpperCAmelCase__			:     List[Any]										= bigram
                 UpperCAmelCase__			:     Any										= []
                 UpperCAmelCase__			:     List[str]										= 0
                 while i < len(_lowerCAmelCase		):
                      try:
                           UpperCAmelCase__			:     List[str]										= word.index(_lowerCAmelCase  ,	_lowerCAmelCase		)
                      except ValueError:
                           new_word.extend(word[i:]		)
                           break
                      else:
                           new_word.extend(word[i:j]		)
                           UpperCAmelCase__			:     Any										= j
  
                      if word[i] == first and i < len(_lowerCAmelCase		) - 1 and word[i + 1] == second:
                           new_word.append(first + second		)
                           i += 2
                      else:
                           new_word.append(word[i]		)
                           i += 1
                 UpperCAmelCase__			:     List[str]										= tuple(_lowerCAmelCase		)
                 UpperCAmelCase__			:     List[Any]										= new_word
                 if len(_lowerCAmelCase		) == 1:
                      break
                 else:
                      UpperCAmelCase__			:     List[Any]										= get_pairs(_lowerCAmelCase		)
            UpperCAmelCase__			:     Any										= """ """.join(_lowerCAmelCase		)
            if word == "\n  </w>":
                 UpperCAmelCase__			:     Optional[Any]										= """\n</w>"""
            UpperCAmelCase__			:     Union[str, Any]										= word
            return word
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase=False		):
            if bypass_tokenizer:
                 UpperCAmelCase__			:     List[str]										= text.split()
            else:
                 UpperCAmelCase__			:     str										= self.moses_tokenize(_lowerCAmelCase  ,	self.lang		)
  
            UpperCAmelCase__			:     Union[str, Any]										= []
            for token in text:
                 if token:
                      split_tokens.extend(list(self.bpe(_lowerCAmelCase		).split(""" """		)		)		)
  
            return split_tokens
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
            return self.encoder.get(_lowerCAmelCase  ,	self.encoder.get(self.unk_token		)		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
            return self.decoder.get(_lowerCAmelCase  ,	self.unk_token		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase		):
            UpperCAmelCase__			:     List[str]										= [t.replace(""" """  ,	""""""		).replace("""</w>"""  ,	""" """		) for t in tokens]
            UpperCAmelCase__			:     Any										= """""".join(_lowerCAmelCase		).split()
            # detokenize
            UpperCAmelCase__			:     List[str]										= self.moses_detokenize(_lowerCAmelCase  ,	self.lang		)
            return text
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None		):
            if token_ids_a is None:
                 return [self.sep_token_id] + token_ids_a
            UpperCAmelCase__			:     Union[str, Any]										= [self.sep_token_id]
            return sep + token_ids_a + sep + token_ids_a
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None  ,	_lowerCAmelCase = False		):
            if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=_lowerCAmelCase  ,	token_ids_a=_lowerCAmelCase  ,	already_has_special_tokens=_lowerCAmelCase		)
            # no bos used in fairseq
            if token_ids_a is not None:
                 return [1] + ([0] * len(_lowerCAmelCase		)) + [1] + ([0] * len(_lowerCAmelCase		))
            return [1] + ([0] * len(_lowerCAmelCase		))
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None		):
            UpperCAmelCase__			:     Optional[int]										= [self.sep_token_id]
  
            # no bos used in fairseq
            if token_ids_a is None:
                 return len(token_ids_a + sep		) * [0]
            return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase = None		):
            if not os.path.isdir(_lowerCAmelCase		):
                 logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
                 return
            UpperCAmelCase__			:     Tuple										= os.path.join(
                _lowerCAmelCase  ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)
            UpperCAmelCase__			:     List[str]										= os.path.join(
                _lowerCAmelCase  ,	(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]		)
  
            with open(_lowerCAmelCase  ,	"""w"""  ,	encoding="""utf-8"""		) as f:
                 f.write(json.dumps(self.encoder  ,	indent=2  ,	sort_keys=_lowerCAmelCase  ,	ensure_ascii=_lowerCAmelCase		) + """\n"""		)
  
            UpperCAmelCase__			:     Dict										= 0
            with open(_lowerCAmelCase  ,	"""w"""  ,	encoding="""utf-8"""		) as writer:
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,	key=lambda _lowerCAmelCase		: kv[1]		):
                      if index != token_index:
                           logger.warning(
                               f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                               """ Please check that the tokenizer is not corrupted!"""		)
                           UpperCAmelCase__			:     Dict										= token_index
                      writer.write(""" """.join(_lowerCAmelCase		) + """\n"""		)
                      index += 1
  
            return vocab_file, merge_file
  
  
  
  
       def __getstate__(							self		):
            UpperCAmelCase__			:     List[Any]										= self.__dict__.copy()
            UpperCAmelCase__			:     Dict										= None
            return state
  
  
  
  
       def __setstate__(							self  ,	_lowerCAmelCase		):
            UpperCAmelCase__			:     Optional[Any]										= d
  
            try:
                 import sacremoses
            except ImportError:
                 raise ImportError(
                     """You need to install sacremoses to use XLMTokenizer. """
                     """See https://pypi.org/project/sacremoses/ for installation."""		)
  
            UpperCAmelCase__			:     Tuple										= sacremoses
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		__lowerCamelCase       ,			unittest.TestCase		):
       __lowerCamelCase										=     PreTrainedTokenizerFast
       __lowerCamelCase										=     False
       __lowerCamelCase										=     True
       __lowerCamelCase										=     'tokenizer_file'
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Union[str, Any]										= False  # because we don't have pretrained_vocab_files_map
            super().setUp()
            UpperCAmelCase__			:     Dict										= True
  
            UpperCAmelCase__			:     int										= ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
            UpperCAmelCase__			:     List[Any]										= """SaulLu/dummy-tokenizer-bytelevel-bpe"""
  
            # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
            UpperCAmelCase__			:     int										= [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
            UpperCAmelCase__			:     Union[str, Any]										= PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
            tokenizer.save_pretrained(self.tmpdirname		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Optional[int]										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     str										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     Optional[Any]										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer_with_special_tokens_change()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)
  
            UpperCAmelCase__			:     Dict										= ("""a""" for _ in range(1000		))
            UpperCAmelCase__			:     int										= tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase  ,	length=1000  ,	vocab_size=50		)
  
            UpperCAmelCase__			:     Optional[Any]										= new_tokenizer.encode("""a🤗"""		)
            self.assertEqual(_lowerCAmelCase  ,	[64, 172, 253, 97, 245]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     List[str]										= AutoTokenizer.from_pretrained("""bert-base-cased"""		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            UpperCAmelCase__			:     Optional[Any]										= len(_lowerCAmelCase		)
  
            with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 UpperCAmelCase__			:     Tuple										= ["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 json.dump(_lowerCAmelCase  ,	open(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	"""w"""		)		)
  
                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 UpperCAmelCase__			:     Optional[int]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		) + 1		)
                 UpperCAmelCase__			:     Any										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	os.path.join(_lowerCAmelCase  ,	"""tokenizer.42.0.0.json"""		)		)
                 UpperCAmelCase__			:     Union[str, Any]										= ["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 UpperCAmelCase__			:     Optional[Any]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		)		)
                 UpperCAmelCase__			:     List[Any]										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
            UpperCAmelCase__			:     List[str]										= """hf-internal-testing/test-two-tokenizers"""
  
            # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
            UpperCAmelCase__			:     int										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28997		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
            # Testing an older version by monkey-patching the version in the module it's used.
            import transformers as old_transformers
  
            UpperCAmelCase__			:     Any										= """3.0.0"""
            UpperCAmelCase__			:     Optional[int]										= old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28996		)
            UpperCAmelCase__			:     Optional[int]										= json.loads(old_tokenizer._tokenizer.to_str()		)
            self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # See https://github.com/huggingface/transformers/pull/12550
            # and https://github.com/huggingface/tokenizers/issues/537
            UpperCAmelCase__			:     int										= PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""		)
            UpperCAmelCase__			:     List[str]										= """The Matrix is a 1999 science fiction action film."""
  
            with concurrent.futures.ThreadPoolExecutor() as executor:
                 UpperCAmelCase__			:     List[str]										= [executor.submit(self.fetch  ,	_lowerCAmelCase  ,	_lowerCAmelCase		) for i in range(10		)]
                 UpperCAmelCase__			:     Dict										= [future.result() for future in futures]
                 self.assertEqual(_lowerCAmelCase  ,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            return tokenizer.encode(_lowerCAmelCase  ,	truncation="""longest_first"""  ,	padding="""longest"""		)
  
  
  
  
  
  
  