


import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class 					__lowercase					(       a_ ,       unittest.TestCase     ):



		"""simple docstring"""

		UpperCamelCase    :				List[str] 					=		PreTrainedTokenizerFast
		UpperCamelCase    :				List[str] 					=		False
		UpperCamelCase    :				int 					=		True
		UpperCamelCase    :				Any 					=		"tokenizer_file"







		def 				__A      (   self  )     ->   Dict:







					'''simple docstring'''







					lowerCamelCase								=     False  # because we don't have pretrained_vocab_files_map
					super().setUp()
					lowerCamelCase								=     True

					lowerCamelCase								=     ["""robot-test/dummy-tokenizer-fast""", """robot-test/dummy-tokenizer-wordlevel"""]
					lowerCamelCase								=     """SaulLu/dummy-tokenizer-bytelevel-bpe"""

					# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
					lowerCamelCase								=     [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

					lowerCamelCase								=     PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
					tokenizer.save_pretrained(self.tmpdirname  )







		def 				__A      (   self  )     ->   List[str]:







					'''simple docstring'''







					pass







		def 				__A      (   self  )     ->   Optional[Any]:







					'''simple docstring'''







					pass







		def 				__A      (   self  )     ->   Optional[Any]:







					'''simple docstring'''







					pass







		def 				__A      (   self  )     ->   Union[str, Any]:







					'''simple docstring'''







					pass







		def 				__A      (   self  )     ->   List[Any]:







					'''simple docstring'''







					lowerCamelCase								=     self.tmpdirname
					# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
											try:
														lowerCamelCase								=     tempfile.mkdtemp()
														lowerCamelCase								=     self.rust_tokenizer_class.from_pretrained(A		,			**A  )

														tokenizer.save_pretrained(self.tmpdirname  )
														super().test_training_new_tokenizer()
											finally:
														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
														# is restored
														shutil.rmtree(self.tmpdirname  )
														lowerCamelCase								=     tmpdirname_orig







		def 				__A      (   self  )     ->   List[str]:







					'''simple docstring'''







					lowerCamelCase								=     self.tmpdirname
					# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
								with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'  ):
											try:
														lowerCamelCase								=     tempfile.mkdtemp()
														lowerCamelCase								=     self.rust_tokenizer_class.from_pretrained(A		,			**A  )

														tokenizer.save_pretrained(self.tmpdirname  )
														super().test_training_new_tokenizer_with_special_tokens_change()
											finally:
														# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
														# is restored
														shutil.rmtree(self.tmpdirname  )
														lowerCamelCase								=     tmpdirname_orig







		def 				__A      (   self  )     ->   Optional[int]:







					'''simple docstring'''







					lowerCamelCase								=     self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

					lowerCamelCase								=     ("""a""" for _ in range(10_00  ))
					lowerCamelCase								=     tokenizer.train_new_from_iterator(text_iterator=A		,			length=10_00		,			vocab_size=50  )

					lowerCamelCase								=     new_tokenizer.encode("""a🤗"""  )
					self.assertEqual(A		,			[64, 1_72, 2_53, 97, 2_45]  )







@require_tokenizers
class 					__lowercase					(       unittest.TestCase     ):



		"""simple docstring"""

		def 				__A      (   self  )     ->   Dict:







					'''simple docstring'''







					lowerCamelCase								=     AutoTokenizer.from_pretrained("""bert-base-cased"""  )
					lowerCamelCase								=     json.loads(tokenizer._tokenizer.to_str()  )
					lowerCamelCase								=     len(A  )

					with tempfile.TemporaryDirectory() as tmp_dir:
								# Hack to save this in the tokenizer_config.json
								lowerCamelCase								=     ["""tokenizer.4.0.0.json"""]
								tokenizer.save_pretrained(A  )
								json.dump(A		,			open(os.path.join(A		,			"""tokenizer.4.0.0.json"""  )		,			"""w"""  )  )

								# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
								lowerCamelCase								=     AutoTokenizer.from_pretrained(A  )
								self.assertEqual(len(A  )		,			len(A  ) + 1  )
								lowerCamelCase								=     json.loads(new_tokenizer._tokenizer.to_str()  )
								self.assertIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )

								# Will need to be adjusted if we reach v42 and this test is still here.
								# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
								shutil.move(os.path.join(A		,			"""tokenizer.4.0.0.json"""  )		,			os.path.join(A		,			"""tokenizer.42.0.0.json"""  )  )
								lowerCamelCase								=     ["""tokenizer.42.0.0.json"""]
								tokenizer.save_pretrained(A  )
								lowerCamelCase								=     AutoTokenizer.from_pretrained(A  )
								self.assertEqual(len(A  )		,			len(A  )  )
								lowerCamelCase								=     json.loads(new_tokenizer._tokenizer.to_str()  )
								self.assertNotIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )







		def 				__A      (   self  )     ->   str:







					'''simple docstring'''







					lowerCamelCase								=     """hf-internal-testing/test-two-tokenizers"""

					# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
					lowerCamelCase								=     AutoTokenizer.from_pretrained(A  )
					self.assertEqual(len(A  )		,			2_89_97  )
					lowerCamelCase								=     json.loads(tokenizer._tokenizer.to_str()  )
					self.assertIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )

					# Testing an older version by monkey-patching the version in the module it's used.
					import transformers as old_transformers

					lowerCamelCase								=     """3.0.0"""
					lowerCamelCase								=     old_transformers.models.auto.AutoTokenizer.from_pretrained(A  )
					self.assertEqual(len(A  )		,			2_89_96  )
					lowerCamelCase								=     json.loads(old_tokenizer._tokenizer.to_str()  )
					self.assertNotIn("""huggingface"""		,			json_tokenizer["""model"""]["""vocab"""]  )







@require_tokenizers
class 					__lowercase					(       unittest.TestCase     ):



		"""simple docstring"""

		def 				__A      (   self  )     ->   Optional[int]:







					'''simple docstring'''







					lowerCamelCase								=     PreTrainedTokenizerFast.from_pretrained("""robot-test/dummy-tokenizer-wordlevel"""  )
					lowerCamelCase								=     """The Matrix is a 1999 science fiction action film."""

					with concurrent.futures.ThreadPoolExecutor() as executor:
								lowerCamelCase								=     [executor.submit(self.fetch		,			A		,			A  ) for i in range(10  )]
								lowerCamelCase								=     [future.result() for future in futures]
								self.assertEqual(A		,			[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )







		def 				__A      (   self		,			A		,			A  )     ->   Tuple:







					'''simple docstring'''







					return tokenizer.encode(A		,			truncation="""longest_first"""		,			padding="""longest"""  )





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCAmelCase      :						List[str]    =		{
    "configuration_big_bird": ["BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP", "BigBirdConfig", "BigBirdOnnxConfig"],
}

try:
		if not is_sentencepiece_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase      :						str    =		["BigBirdTokenizer"]

try:
		if not is_tokenizers_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase      :						str    =		["BigBirdTokenizerFast"]

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase      :						Dict    =		[
		    "BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST",
		    "BigBirdForCausalLM",
		    "BigBirdForMaskedLM",
		    "BigBirdForMultipleChoice",
		    "BigBirdForPreTraining",
		    "BigBirdForQuestionAnswering",
		    "BigBirdForSequenceClassification",
		    "BigBirdForTokenClassification",
		    "BigBirdLayer",
		    "BigBirdModel",
		    "BigBirdPreTrainedModel",
		    "load_tf_weights_in_big_bird",
		]

try:
		if not is_flax_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		UpperCAmelCase      :						Dict    =		[
		    "FlaxBigBirdForCausalLM",
		    "FlaxBigBirdForMaskedLM",
		    "FlaxBigBirdForMultipleChoice",
		    "FlaxBigBirdForPreTraining",
		    "FlaxBigBirdForQuestionAnswering",
		    "FlaxBigBirdForSequenceClassification",
		    "FlaxBigBirdForTokenClassification",
		    "FlaxBigBirdModel",
		    "FlaxBigBirdPreTrainedModel",
		]

if TYPE_CHECKING:
		from .configuration_big_bird import BIG_BIRD_PRETRAINED_CONFIG_ARCHIVE_MAP, BigBirdConfig, BigBirdOnnxConfig

		try:
				if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_big_bird import BigBirdTokenizer

		try:
				if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .tokenization_big_bird_fast import BigBirdTokenizerFast

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_big_bird import (
				    BIG_BIRD_PRETRAINED_MODEL_ARCHIVE_LIST,
				    BigBirdForCausalLM,
				    BigBirdForMaskedLM,
				    BigBirdForMultipleChoice,
				    BigBirdForPreTraining,
				    BigBirdForQuestionAnswering,
				    BigBirdForSequenceClassification,
				    BigBirdForTokenClassification,
				    BigBirdLayer,
				    BigBirdModel,
				    BigBirdPreTrainedModel,
				    load_tf_weights_in_big_bird,
				)

		try:
				if not is_flax_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_flax_big_bird import (
				    FlaxBigBirdForCausalLM,
				    FlaxBigBirdForMaskedLM,
				    FlaxBigBirdForMultipleChoice,
				    FlaxBigBirdForPreTraining,
				    FlaxBigBirdForQuestionAnswering,
				    FlaxBigBirdForSequenceClassification,
				    FlaxBigBirdForTokenClassification,
				    FlaxBigBirdModel,
				    FlaxBigBirdPreTrainedModel,
				)

else:
		import sys

		UpperCAmelCase      :						Dict    =		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

