




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class 							__SCREAMING_SNAKE_CASE				(						lowerCamelCase  ,	unittest.TestCase    ):
					_lowerCamelCase						    =					PreTrainedTokenizerFast
					_lowerCamelCase						    =					False
					_lowerCamelCase						    =					True
					_lowerCamelCase						    =					"""tokenizer_file"""

					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[int]				  =				False  # because we don't have pretrained_vocab_files_map
						super().setUp()
						lowerCamelCase_				:						List[str]				  =				True

						lowerCamelCase_				:						List[str]				  =				['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
						lowerCamelCase_				:						Optional[Any]				  =				'SaulLu/dummy-tokenizer-bytelevel-bpe'

						# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
						lowerCamelCase_				:						Dict				  =				[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

						lowerCamelCase_				:						Optional[Any]				  =				PreTrainedTokenizerFast.from_pretrained(model_paths[0]   )
						tokenizer.save_pretrained(self.tmpdirname   )

					def       lowercase_				(							self   ):







						'''simple docstring'''




						pass

					def       lowercase_				(							self   ):







						'''simple docstring'''




						pass

					def       lowercase_				(							self   ):







						'''simple docstring'''




						pass

					def       lowercase_				(							self   ):







						'''simple docstring'''




						pass

					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Tuple				  =				self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
								try:
									lowerCamelCase_				:						Any				  =				tempfile.mkdtemp()
									lowerCamelCase_				:						Optional[Any]				  =				self.rust_tokenizer_class.from_pretrained(__lowercase					,			**__lowercase   )

									tokenizer.save_pretrained(self.tmpdirname   )
									super().test_training_new_tokenizer()
								finally:
									# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
									# is restored
									shutil.rmtree(self.tmpdirname   )
									lowerCamelCase_				:						Union[str, Any]				  =				tmpdirname_orig

					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						List[str]				  =				self.tmpdirname
						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
								try:
									lowerCamelCase_				:						List[Any]				  =				tempfile.mkdtemp()
									lowerCamelCase_				:						Optional[Any]				  =				self.rust_tokenizer_class.from_pretrained(__lowercase					,			**__lowercase   )

									tokenizer.save_pretrained(self.tmpdirname   )
									super().test_training_new_tokenizer_with_special_tokens_change()
								finally:
									# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
									# is restored
									shutil.rmtree(self.tmpdirname   )
									lowerCamelCase_				:						str				  =				tmpdirname_orig





					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Dict				  =				self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name   )

						lowerCamelCase_				:						str				  =				('a' for _ in range(1_000   ))
						lowerCamelCase_				:						int				  =				tokenizer.train_new_from_iterator(text_iterator=__lowercase					,			length=1_000					,			vocab_size=50   )

						lowerCamelCase_				:						Dict				  =				new_tokenizer.encode('a🤗'   )
						self.assertEqual(__lowercase					,			[64, 172, 253, 97, 245]   )


@require_tokenizers
class 							__SCREAMING_SNAKE_CASE				(						unittest.TestCase    ):

					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[Any]				  =				AutoTokenizer.from_pretrained('bert-base-cased'   )
						lowerCamelCase_				:						Dict				  =				json.loads(tokenizer._tokenizer.to_str()   )
						lowerCamelCase_				:						Tuple				  =				len(__lowercase   )

						with tempfile.TemporaryDirectory() as tmp_dir:
							# Hack to save this in the tokenizer_config.json
							lowerCamelCase_				:						Tuple				  =				['tokenizer.4.0.0.json']
							tokenizer.save_pretrained(__lowercase   )
							json.dump(__lowercase					,			open(os.path.join(__lowercase					,			'tokenizer.4.0.0.json'   )					,			'w'   )   )

							# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
							lowerCamelCase_				:						List[str]				  =				AutoTokenizer.from_pretrained(__lowercase   )
							self.assertEqual(len(__lowercase   )					,			len(__lowercase   ) + 1   )
							lowerCamelCase_				:						Tuple				  =				json.loads(new_tokenizer._tokenizer.to_str()   )
							self.assertIn('huggingface'					,			json_tokenizer['model']['vocab']   )

							# Will need to be adjusted if we reach v42 and this test is still here.
							# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
							shutil.move(os.path.join(__lowercase					,			'tokenizer.4.0.0.json'   )					,			os.path.join(__lowercase					,			'tokenizer.42.0.0.json'   )   )
							lowerCamelCase_				:						List[Any]				  =				['tokenizer.42.0.0.json']
							tokenizer.save_pretrained(__lowercase   )
							lowerCamelCase_				:						str				  =				AutoTokenizer.from_pretrained(__lowercase   )
							self.assertEqual(len(__lowercase   )					,			len(__lowercase   )   )
							lowerCamelCase_				:						int				  =				json.loads(new_tokenizer._tokenizer.to_str()   )
							self.assertNotIn('huggingface'					,			json_tokenizer['model']['vocab']   )





					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[int]				  =				'hf-internal-testing/test-two-tokenizers'

						# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
						lowerCamelCase_				:						int				  =				AutoTokenizer.from_pretrained(__lowercase   )
						self.assertEqual(len(__lowercase   )					,			28_997   )
						lowerCamelCase_				:						str				  =				json.loads(tokenizer._tokenizer.to_str()   )
						self.assertIn('huggingface'					,			json_tokenizer['model']['vocab']   )

						# Testing an older version by monkey-patching the version in the module it's used.
						import transformers as old_transformers

						lowerCamelCase_				:						Union[str, Any]				  =				'3.0.0'
						lowerCamelCase_				:						List[str]				  =				old_transformers.models.auto.AutoTokenizer.from_pretrained(__lowercase   )
						self.assertEqual(len(__lowercase   )					,			28_996   )
						lowerCamelCase_				:						Tuple				  =				json.loads(old_tokenizer._tokenizer.to_str()   )
						self.assertNotIn('huggingface'					,			json_tokenizer['model']['vocab']   )


@require_tokenizers
class 							__SCREAMING_SNAKE_CASE				(						unittest.TestCase    ):

					def       lowercase_				(							self   ):







						'''simple docstring'''




						lowerCamelCase_				:						Optional[int]				  =				PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'   )
						lowerCamelCase_				:						int				  =				'The Matrix is a 1999 science fiction action film.'

						with concurrent.futures.ThreadPoolExecutor() as executor:
							lowerCamelCase_				:						List[str]				  =				[executor.submit(self.fetch					,			__lowercase					,			__lowercase   ) for i in range(10   )]
							lowerCamelCase_				:						List[str]				  =				[future.result() for future in futures]
							self.assertEqual(__lowercase					,			[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10   )]   )





					def       lowercase_				(							self					,			__lowercase					,			__lowercase   ):







						'''simple docstring'''




						return tokenizer.encode(__lowercase					,			truncation='longest_first'					,			padding='longest'   )





"""simple docstring"""


def 					_lowerCAmelCase (  lowercase_		):
						return " ".join(sorted(set(sentence.split()		)		)		)


if __name__ == "__main__":
				import doctest

				doctest.testmod()





