code: 
  
  
  
  
  
  
  from typing import TYPE_CHECKING
  
  from ...utils import (
      OptionalDependencyNotAvailable,
      _LazyModule,
      is_flax_available,
      is_tf_available,
      is_tokenizers_available,
      is_torch_available,
  )
  
  
  SCREAMING_SNAKE_CASE__		:       Union[str, Any]   							=							{
      """configuration_bart""": ["""BART_PRETRAINED_CONFIG_ARCHIVE_MAP""", """BartConfig""", """BartOnnxConfig"""],
      """tokenization_bart""": ["""BartTokenizer"""],
  }
  
  try:
        if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
        pass
  else:
        SCREAMING_SNAKE_CASE__		:       Any   							=							["""BartTokenizerFast"""]
  
  try:
        if not is_torch_available():
              raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
        pass
  else:
        SCREAMING_SNAKE_CASE__		:       str   							=							[
            """BART_PRETRAINED_MODEL_ARCHIVE_LIST""",
            """BartForCausalLM""",
            """BartForConditionalGeneration""",
            """BartForQuestionAnswering""",
            """BartForSequenceClassification""",
            """BartModel""",
            """BartPretrainedModel""",
            """PretrainedBartModel""",
        ]
  
  try:
        if not is_tf_available():
              raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
        pass
  else:
        SCREAMING_SNAKE_CASE__		:       Dict   							=							[
            """TFBartForConditionalGeneration""",
            """TFBartForSequenceClassification""",
            """TFBartModel""",
            """TFBartPretrainedModel""",
        ]
  
  try:
        if not is_flax_available():
              raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
        pass
  else:
        SCREAMING_SNAKE_CASE__		:       Dict   							=							[
            """FlaxBartDecoderPreTrainedModel""",
            """FlaxBartForCausalLM""",
            """FlaxBartForConditionalGeneration""",
            """FlaxBartForQuestionAnswering""",
            """FlaxBartForSequenceClassification""",
            """FlaxBartModel""",
            """FlaxBartPreTrainedModel""",
        ]
  
  if TYPE_CHECKING:
        from .configuration_bart import BART_PRETRAINED_CONFIG_ARCHIVE_MAP, BartConfig, BartOnnxConfig
        from .tokenization_bart import BartTokenizer
  
        try:
              if not is_tokenizers_available():
                    raise OptionalDependencyNotAvailable()
        except OptionalDependencyNotAvailable:
              pass
        else:
              from .tokenization_bart_fast import BartTokenizerFast
  
        try:
              if not is_torch_available():
                    raise OptionalDependencyNotAvailable()
        except OptionalDependencyNotAvailable:
              pass
        else:
              from .modeling_bart import (
                  BART_PRETRAINED_MODEL_ARCHIVE_LIST,
                  BartForCausalLM,
                  BartForConditionalGeneration,
                  BartForQuestionAnswering,
                  BartForSequenceClassification,
                  BartModel,
                  BartPretrainedModel,
                  PretrainedBartModel,
              )
  
        try:
              if not is_tf_available():
                    raise OptionalDependencyNotAvailable()
        except OptionalDependencyNotAvailable:
              pass
        else:
              from .modeling_tf_bart import (
                  TFBartForConditionalGeneration,
                  TFBartForSequenceClassification,
                  TFBartModel,
                  TFBartPretrainedModel,
              )
  
        try:
              if not is_flax_available():
                    raise OptionalDependencyNotAvailable()
        except OptionalDependencyNotAvailable:
              pass
        else:
              from .modeling_flax_bart import (
                  FlaxBartDecoderPreTrainedModel,
                  FlaxBartForCausalLM,
                  FlaxBartForConditionalGeneration,
                  FlaxBartForQuestionAnswering,
                  FlaxBartForSequenceClassification,
                  FlaxBartModel,
                  FlaxBartPreTrainedModel,
              )
  
  else:
        import sys
  
        SCREAMING_SNAKE_CASE__		:       Any   							=							_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)
  
  
  
  
  
  
  
style_context: 
  
  
  
  
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		__lowerCamelCase       ,			unittest.TestCase		):
       __lowerCamelCase										=     PreTrainedTokenizerFast
       __lowerCamelCase										=     False
       __lowerCamelCase										=     True
       __lowerCamelCase										=     'tokenizer_file'
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Union[str, Any]										= False  # because we don't have pretrained_vocab_files_map
            super().setUp()
            UpperCAmelCase__			:     Dict										= True
  
            UpperCAmelCase__			:     int										= ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
            UpperCAmelCase__			:     List[Any]										= """SaulLu/dummy-tokenizer-bytelevel-bpe"""
  
            # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
            UpperCAmelCase__			:     int										= [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
            UpperCAmelCase__			:     Union[str, Any]										= PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
            tokenizer.save_pretrained(self.tmpdirname		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
            pass
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Optional[int]										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     str										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"		):
                      try:
                           UpperCAmelCase__			:     Optional[Any]										= tempfile.mkdtemp()
                           UpperCAmelCase__			:     Any										= self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase  ,	**_lowerCAmelCase		)
  
                           tokenizer.save_pretrained(self.tmpdirname		)
                           super().test_training_new_tokenizer_with_special_tokens_change()
                      finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname		)
                           UpperCAmelCase__			:     str										= tmpdirname_orig
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     Tuple										= self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)
  
            UpperCAmelCase__			:     Dict										= ("""a""" for _ in range(1000		))
            UpperCAmelCase__			:     int										= tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase  ,	length=1000  ,	vocab_size=50		)
  
            UpperCAmelCase__			:     Optional[Any]										= new_tokenizer.encode("""a🤗"""		)
            self.assertEqual(_lowerCAmelCase  ,	[64, 172, 253, 97, 245]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            UpperCAmelCase__			:     List[str]										= AutoTokenizer.from_pretrained("""bert-base-cased"""		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            UpperCAmelCase__			:     Optional[Any]										= len(_lowerCAmelCase		)
  
            with tempfile.TemporaryDirectory() as tmp_dir:
                 # Hack to save this in the tokenizer_config.json
                 UpperCAmelCase__			:     Tuple										= ["""tokenizer.4.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 json.dump(_lowerCAmelCase  ,	open(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	"""w"""		)		)
  
                 # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                 UpperCAmelCase__			:     Optional[int]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		) + 1		)
                 UpperCAmelCase__			:     Any										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
                 # Will need to be adjusted if we reach v42 and this test is still here.
                 # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                 shutil.move(os.path.join(_lowerCAmelCase  ,	"""tokenizer.4.0.0.json"""		)  ,	os.path.join(_lowerCAmelCase  ,	"""tokenizer.42.0.0.json"""		)		)
                 UpperCAmelCase__			:     Union[str, Any]										= ["""tokenizer.42.0.0.json"""]
                 tokenizer.save_pretrained(_lowerCAmelCase		)
                 UpperCAmelCase__			:     Optional[Any]										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
                 self.assertEqual(len(_lowerCAmelCase		)  ,	len(_lowerCAmelCase		)		)
                 UpperCAmelCase__			:     List[Any]										= json.loads(new_tokenizer._tokenizer.to_str()		)
                 self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
            UpperCAmelCase__			:     List[str]										= """hf-internal-testing/test-two-tokenizers"""
  
            # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
            UpperCAmelCase__			:     int										= AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28997		)
            UpperCAmelCase__			:     List[Any]										= json.loads(tokenizer._tokenizer.to_str()		)
            self.assertIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
            # Testing an older version by monkey-patching the version in the module it's used.
            import transformers as old_transformers
  
            UpperCAmelCase__			:     Any										= """3.0.0"""
            UpperCAmelCase__			:     Optional[int]										= old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase		)
            self.assertEqual(len(_lowerCAmelCase		)  ,	28996		)
            UpperCAmelCase__			:     Optional[int]										= json.loads(old_tokenizer._tokenizer.to_str()		)
            self.assertNotIn("""huggingface"""  ,	json_tokenizer["""model"""]["""vocab"""]		)
  
  
  
  
  
  
  
  @require_tokenizers
  class 							UpperCAmelCase_     (		unittest.TestCase		):
  
  
  
  
       def      __UpperCAmelCase     (							self		):
            # See https://github.com/huggingface/transformers/pull/12550
            # and https://github.com/huggingface/tokenizers/issues/537
            UpperCAmelCase__			:     int										= PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""		)
            UpperCAmelCase__			:     List[str]										= """The Matrix is a 1999 science fiction action film."""
  
            with concurrent.futures.ThreadPoolExecutor() as executor:
                 UpperCAmelCase__			:     List[str]										= [executor.submit(self.fetch  ,	_lowerCAmelCase  ,	_lowerCAmelCase		) for i in range(10		)]
                 UpperCAmelCase__			:     Dict										= [future.result() for future in futures]
                 self.assertEqual(_lowerCAmelCase  ,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10		)]		)
  
  
  
  
       def      __UpperCAmelCase     (							self  ,	_lowerCAmelCase  ,	_lowerCAmelCase		):
            return tokenizer.encode(_lowerCAmelCase  ,	truncation="""longest_first"""  ,	padding="""longest"""		)
  
  
  
  
  
  
  