code: 
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  
  @require_tokenizers
  class 					__lowercase					(       a_ ,       unittest.TestCase     ):
  
  
  
  		"""simple docstring"""
  
  		UpperCamelCase    :				List[str] 					=		PreTrainedTokenizerFast
  		UpperCamelCase    :				List[str] 					=		False
  		UpperCamelCase    :				int 					=		True
  		UpperCamelCase    :				Any 					=		"tokenizer_file"
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Dict:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     False  # because we don't have pretrained_vocab_files_map
  					super().setUp()
  					lowerCamelCase								=     True
  
  					lowerCamelCase								=     ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
  					lowerCamelCase								=     """SaulLu/dummy-tokenizer-bytelevel-bpe"""
  
  					# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
  					lowerCamelCase								=     [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
  					lowerCamelCase								=     PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
  					tokenizer.save_pretrained(self.tmpdirname  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   List[str]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					pass
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Optional[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					pass
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Optional[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					pass
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Union[str, Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					pass
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   List[Any]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     self.tmpdirname
  					# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  								with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
  											try:
  														lowerCamelCase								=     tempfile.mkdtemp()
  														lowerCamelCase								=     self.rust_tokenizer_class.from_pretrained(A		,			**A  )
  
  														tokenizer.save_pretrained(self.tmpdirname  )
  														super().test_training_new_tokenizer()
  											finally:
  														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  														# is restored
  														shutil.rmtree(self.tmpdirname  )
  														lowerCamelCase								=     tmpdirname_orig
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   List[str]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     self.tmpdirname
  					# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  								with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
  											try:
  														lowerCamelCase								=     tempfile.mkdtemp()
  														lowerCamelCase								=     self.rust_tokenizer_class.from_pretrained(A		,			**A  )
  
  														tokenizer.save_pretrained(self.tmpdirname  )
  														super().test_training_new_tokenizer_with_special_tokens_change()
  											finally:
  														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  														# is restored
  														shutil.rmtree(self.tmpdirname  )
  														lowerCamelCase								=     tmpdirname_orig
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   Optional[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )
  
  					lowerCamelCase								=     ("""a""" for _ in range(10_00  ))
  					lowerCamelCase								=     tokenizer.train_new_from_iterator(text_iterator=A		,			length=10_00		,			vocab_size=50  )
  
  					lowerCamelCase								=     new_tokenizer.encode("""a🤗"""  )
  					self.assertEqual(A		,			[64, 1_72, 2_53, 97, 2_45]  )
  
  
  
  
  
  
  
  @require_tokenizers
  class 					__lowercase					(       unittest.TestCase     ):
  
  
  
  		"""simple docstring"""
  
  		def 				__A      (   self  )     ->   Dict:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     AutoTokenizer.from_pretrained("""bert-base-cased"""  )
  					lowerCamelCase								=     json.loads(tokenizer._tokenizer.to_str()  )
  					lowerCamelCase								=     len(A  )
  
  					with tempfile.TemporaryDirectory() as tmp_dir:
  								# Hack to save this in the tokenizer_config.json
  								lowerCamelCase								=     ["""tokenizer.4.0.0.json"""]
  								tokenizer.save_pretrained(A  )
  								json.dump(A		,			open(os.path.join(A		,			"""tokenizer.4.0.0.json"""  )		,			"""w"""  )  )
  
  								# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  								lowerCamelCase								=     AutoTokenizer.from_pretrained(A  )
  								self.assertEqual(len(A  )		,			len(A  ) + 1  )
  								lowerCamelCase								=     json.loads(new_tokenizer._tokenizer.to_str()  )
  								self.assertIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )
  
  								# Will need to be adjusted if we reach v42 and this test is still here.
  								# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
  								shutil.move(os.path.join(A		,			"""tokenizer.4.0.0.json"""  )		,			os.path.join(A		,			"""tokenizer.42.0.0.json"""  )  )
  								lowerCamelCase								=     ["""tokenizer.42.0.0.json"""]
  								tokenizer.save_pretrained(A  )
  								lowerCamelCase								=     AutoTokenizer.from_pretrained(A  )
  								self.assertEqual(len(A  )		,			len(A  )  )
  								lowerCamelCase								=     json.loads(new_tokenizer._tokenizer.to_str()  )
  								self.assertNotIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )
  
  
  
  
  
  
  
  		def 				__A      (   self  )     ->   str:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     """hf-internal-testing/test-two-tokenizers"""
  
  					# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  					lowerCamelCase								=     AutoTokenizer.from_pretrained(A  )
  					self.assertEqual(len(A  )		,			2_89_97  )
  					lowerCamelCase								=     json.loads(tokenizer._tokenizer.to_str()  )
  					self.assertIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )
  
  					# Testing an older version by monkey-patching the version in the module it's used.
  					import transformers as old_transformers
  
  					lowerCamelCase								=     """3.0.0"""
  					lowerCamelCase								=     old_transformers.models.auto.AutoTokenizer.from_pretrained(A  )
  					self.assertEqual(len(A  )		,			2_89_96  )
  					lowerCamelCase								=     json.loads(old_tokenizer._tokenizer.to_str()  )
  					self.assertNotIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )
  
  
  
  
  
  
  
  @require_tokenizers
  class 					__lowercase					(       unittest.TestCase     ):
  
  
  
  		"""simple docstring"""
  
  		def 				__A      (   self  )     ->   Optional[int]:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					lowerCamelCase								=     PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""  )
  					lowerCamelCase								=     """The Matrix is a 1999 science fiction action film."""
  
  					with concurrent.futures.ThreadPoolExecutor() as executor:
  								lowerCamelCase								=     [executor.submit(self.fetch		,			A		,			A  ) for i in range(10  )]
  								lowerCamelCase								=     [future.result() for future in futures]
  								self.assertEqual(A		,			[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )
  
  
  
  
  
  
  
  		def 				__A      (   self		,			A		,			A  )     ->   Tuple:
  
  
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  
  					return tokenizer.encode(A		,			truncation="""longest_first"""		,			padding="""longest"""  )
  
  
style_context: 
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_sentencepiece_available,
      is_tf_available,
      is_tokenizers_available,
      is_torch_available,
  )
  
  
  UpperCAmelCase      :						List[str]    =		{
      "configuration_big_bird": ["BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP", "BigBirdConfig", "BigBirdOnnxConfig"],
  }
  
  try:
  		if not is_sentencepiece_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		UpperCAmelCase      :						str    =		["BigBirdTokenizer"]
  
  try:
  		if not is_tokenizers_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		UpperCAmelCase      :						str    =		["BigBirdTokenizerFast"]
  
  try:
  		if not is_torch_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		UpperCAmelCase      :						Dict    =		[
  		    "BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST",
  		    "BigBirdForCausalLM",
  		    "BigBirdForMaskedLM",
  		    "BigBirdForMultipleChoice",
  		    "BigBirdForPreTraining",
  		    "BigBirdForQuestionAnswering",
  		    "BigBirdForSequenceClassification",
  		    "BigBirdForTokenClassification",
  		    "BigBirdLayer",
  		    "BigBirdModel",
  		    "BigBirdPreTrainedModel",
  		    "load_tf_weights_in_big_bird",
  		]
  
  try:
  		if not is_flax_available():
  				raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
  		pass
  else:
  		UpperCAmelCase      :						Dict    =		[
  		    "FlaxBigBirdForCausalLM",
  		    "FlaxBigBirdForMaskedLM",
  		    "FlaxBigBirdForMultipleChoice",
  		    "FlaxBigBirdForPreTraining",
  		    "FlaxBigBirdForQuestionAnswering",
  		    "FlaxBigBirdForSequenceClassification",
  		    "FlaxBigBirdForTokenClassification",
  		    "FlaxBigBirdModel",
  		    "FlaxBigBirdPreTrainedModel",
  		]
  
  if TYPE_CHECKING:
  		from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig
  
  		try:
  				if not is_sentencepiece_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .tokenization_big_bird import BigBirdTokenizer
  
  		try:
  				if not is_tokenizers_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .tokenization_big_bird_fast import BigBirdTokenizerFast
  
  		try:
  				if not is_torch_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .modeling_big_bird import (
  				    BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
  				    BigBirdForCausalLM,
  				    BigBirdForMaskedLM,
  				    BigBirdForMultipleChoice,
  				    BigBirdForPreTraining,
  				    BigBirdForQuestionAnswering,
  				    BigBirdForSequenceClassification,
  				    BigBirdForTokenClassification,
  				    BigBirdLayer,
  				    BigBirdModel,
  				    BigBirdPreTrainedModel,
  				    load_tf_weights_in_big_bird,
  				)
  
  		try:
  				if not is_flax_available():
  						raise OptionalDependencyNotAvailable()
  		except OptionalDependencyNotAvailable:
  				pass
  		else:
  				from .modeling_flax_big_bird import (
  				    FlaxBigBirdForCausalLM,
  				    FlaxBigBirdForMaskedLM,
  				    FlaxBigBirdForMultipleChoice,
  				    FlaxBigBirdForPreTraining,
  				    FlaxBigBirdForQuestionAnswering,
  				    FlaxBigBirdForSequenceClassification,
  				    FlaxBigBirdForTokenClassification,
  				    FlaxBigBirdModel,
  				    FlaxBigBirdPreTrainedModel,
  				)
  
  else:
  		import sys
  
  		UpperCAmelCase      :						Dict    =		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
  
  