






import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class        snake_case_					(						lowerCamelCase__		,       unittest.TestCase			):

						"""simple docstring"""



						snake_case__   					=			PreTrainedTokenizerFast
						snake_case__   					=			False
						snake_case__   					=			True
						snake_case__   					=			"""tokenizer_file"""



						def 					UpperCAmelCase__				(self:				Dict      )						->    str:




											'''simple docstring'''



											__a				:	List[Any]			=				False  # because we don't have pretrained_vocab_files_map
											super().setUp()
											__a				:	Any			=				True

											__a				:	Any			=				["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
											__a				:	Optional[Any]			=				"SaulLu/dummy-tokenizer-bytelevel-bpe"

											# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
											__a				:	Optional[Any]			=				[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

											__a				:	Tuple			=				PreTrainedTokenizerFast.from_pretrained(model_paths[0]      )
											tokenizer.save_pretrained(self.tmpdirname      )



						def 					UpperCAmelCase__				(self:				List[str]      )						->    Optional[Any]:




											'''simple docstring'''



											pass



						def 					UpperCAmelCase__				(self:				List[str]      )						->    str:




											'''simple docstring'''



											pass



						def 					UpperCAmelCase__				(self:				Dict      )						->    Optional[Any]:




											'''simple docstring'''



											pass



						def 					UpperCAmelCase__				(self:				str      )						->    Any:




											'''simple docstring'''



											pass



						def 					UpperCAmelCase__				(self:				List[str]      )						->    str:




											'''simple docstring'''



											__a				:	str			=				self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																					try:
																										__a				:	Any			=				tempfile.mkdtemp()
																										__a				:	str			=				self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_   ,				**SCREAMING_SNAKE_CASE_      )

																										tokenizer.save_pretrained(self.tmpdirname      )
																										super().test_training_new_tokenizer()
																					finally:
																										# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																										# is restored
																										shutil.rmtree(self.tmpdirname      )
																										__a				:	int			=				tmpdirname_orig



						def 					UpperCAmelCase__				(self:				Tuple      )						->    int:




											'''simple docstring'''



											__a				:	str			=				self.tmpdirname
											# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
											for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																					try:
																										__a				:	Dict			=				tempfile.mkdtemp()
																										__a				:	List[str]			=				self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_   ,				**SCREAMING_SNAKE_CASE_      )

																										tokenizer.save_pretrained(self.tmpdirname      )
																										super().test_training_new_tokenizer_with_special_tokens_change()
																					finally:
																										# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
																										# is restored
																										shutil.rmtree(self.tmpdirname      )
																										__a				:	int			=				tmpdirname_orig



						def 					UpperCAmelCase__				(self:				List[str]      )						->    Union[str, Any]:




											'''simple docstring'''



											__a				:	Dict			=				self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name      )

											__a				:	List[str]			=				("a" for _ in range(1000      ))
											__a				:	int			=				tokenizer.train_new_from_iterator(text_iterator=SCREAMING_SNAKE_CASE_   ,				length=1000   ,				vocab_size=50      )

											__a				:	Any			=				new_tokenizer.encode("a🤗"      )
											self.assertEqual(SCREAMING_SNAKE_CASE_   ,				[64, 172, 253, 97, 245]      )




@require_tokenizers
class        snake_case_					(						unittest.TestCase			):

						"""simple docstring"""



						def 					UpperCAmelCase__				(self:				Optional[int]      )						->    Optional[int]:




											'''simple docstring'''



											__a				:	List[str]			=				AutoTokenizer.from_pretrained("bert-base-cased"      )
											__a				:	Tuple			=				json.loads(tokenizer._tokenizer.to_str()      )
											__a				:	List[Any]			=				len(SCREAMING_SNAKE_CASE_      )

											with tempfile.TemporaryDirectory() as tmp_dir:
																# Hack to save this in the tokenizer_config.json
																__a				:	List[Any]			=				["tokenizer.4.0.0.json"]
																tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_      )
																json.dump(SCREAMING_SNAKE_CASE_   ,				open(os.path.join(SCREAMING_SNAKE_CASE_   ,				"tokenizer.4.0.0.json"      )   ,				"w"      )      )

																# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
																__a				:	Dict			=				AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
																self.assertEqual(len(SCREAMING_SNAKE_CASE_      )   ,				len(SCREAMING_SNAKE_CASE_      ) + 1      )
																__a				:	Optional[int]			=				json.loads(new_tokenizer._tokenizer.to_str()      )
																self.assertIn("huggingface"   ,				json_tokenizer["model"]["vocab"]      )

																# Will need to be adjusted if we reach v42 and this test is still here.
																# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
																shutil.move(os.path.join(SCREAMING_SNAKE_CASE_   ,				"tokenizer.4.0.0.json"      )   ,				os.path.join(SCREAMING_SNAKE_CASE_   ,				"tokenizer.42.0.0.json"      )      )
																__a				:	Optional[int]			=				["tokenizer.42.0.0.json"]
																tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_      )
																__a				:	Union[str, Any]			=				AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
																self.assertEqual(len(SCREAMING_SNAKE_CASE_      )   ,				len(SCREAMING_SNAKE_CASE_      )      )
																__a				:	Optional[int]			=				json.loads(new_tokenizer._tokenizer.to_str()      )
																self.assertNotIn("huggingface"   ,				json_tokenizer["model"]["vocab"]      )



						def 					UpperCAmelCase__				(self:				str      )						->    str:




											'''simple docstring'''



											__a				:	Optional[int]			=				"hf-internal-testing/test-two-tokenizers"

											# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
											__a				:	Tuple			=				AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
											self.assertEqual(len(SCREAMING_SNAKE_CASE_      )   ,				28997      )
											__a				:	Optional[int]			=				json.loads(tokenizer._tokenizer.to_str()      )
											self.assertIn("huggingface"   ,				json_tokenizer["model"]["vocab"]      )

											# Testing an older version by monkey-patching the version in the module it's used.
											import transformers as old_transformers

											__a				:	str			=				"3.0.0"
											__a				:	Optional[Any]			=				old_transformers.models.auto.AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
											self.assertEqual(len(SCREAMING_SNAKE_CASE_      )   ,				28996      )
											__a				:	List[Any]			=				json.loads(old_tokenizer._tokenizer.to_str()      )
											self.assertNotIn("huggingface"   ,				json_tokenizer["model"]["vocab"]      )




@require_tokenizers
class        snake_case_					(						unittest.TestCase			):

						"""simple docstring"""



						def 					UpperCAmelCase__				(self:				List[Any]      )						->    List[str]:




											'''simple docstring'''



											__a				:	List[Any]			=				PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"      )
											__a				:	Dict			=				"The Matrix is a 1999 science fiction action film."

											with concurrent.futures.ThreadPoolExecutor() as executor:
																__a				:	Tuple			=				[executor.submit(self.fetch   ,				SCREAMING_SNAKE_CASE_   ,				SCREAMING_SNAKE_CASE_      ) for i in range(10      )]
																__a				:	List[str]			=				[future.result() for future in futures]
																self.assertEqual(SCREAMING_SNAKE_CASE_   ,				[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10      )]      )



						def 					UpperCAmelCase__				(self:				Tuple   ,				__UpperCAmelCase:				Any   ,				__UpperCAmelCase:				Any      )						->    Dict:




											'''simple docstring'''



											return tokenizer.encode(SCREAMING_SNAKE_CASE_   ,				truncation="longest_first"   ,				padding="longest"      )













import unittest
from dataclasses import dataclass
from typing import Optional

from transformers.utils import ModelOutput




@dataclass
class        snake_case_					(						__UpperCamelCase			):

						"""simple docstring"""



						snake_case__   					=			42
						snake_case__   					=			None
						snake_case__   					=			None




class        snake_case_					(						unittest.TestCase			):

						"""simple docstring"""



						def 					UpperCAmelCase__				(self:				Optional[Any]      )						->    Tuple:




											'''simple docstring'''



											__a				:	List[str]			=				ModelOutputTest(a=30      )
											self.assertEqual(x.a   ,				30      )
											self.assertIsNone(x.b      )
											self.assertIsNone(x.c      )
											with self.assertRaises(__UpperCAmelCase      ):
																__a				:	Tuple			=				x.d



						def 					UpperCAmelCase__				(self:				Dict      )						->    Union[str, Any]:




											'''simple docstring'''



											__a				:	Union[str, Any]			=				ModelOutputTest(a=30   ,				b=10      )
											self.assertEqual(x[0]   ,				30      )
											self.assertEqual(x[1]   ,				10      )
											self.assertEqual(x[:2]   ,				(30, 10)      )
											self.assertEqual(x[:]   ,				(30, 10)      )

											__a				:	int			=				ModelOutputTest(a=30   ,				c=10      )
											self.assertEqual(x[0]   ,				30      )
											self.assertEqual(x[1]   ,				10      )
											self.assertEqual(x[:2]   ,				(30, 10)      )
											self.assertEqual(x[:]   ,				(30, 10)      )



						def 					UpperCAmelCase__				(self:				Optional[Any]      )						->    List[Any]:




											'''simple docstring'''



											__a				:	Optional[Any]			=				ModelOutputTest(a=30   ,				b=10      )
											self.assertEqual(x["a"]   ,				30      )
											self.assertEqual(x["b"]   ,				10      )
											with self.assertRaises(__UpperCAmelCase      ):
																__a				:	List[str]			=				x["c"]

											__a				:	Union[str, Any]			=				ModelOutputTest(a=30   ,				c=10      )
											self.assertEqual(x["a"]   ,				30      )
											self.assertEqual(x["c"]   ,				10      )
											with self.assertRaises(__UpperCAmelCase      ):
																__a				:	int			=				x["b"]



						def 					UpperCAmelCase__				(self:				Union[str, Any]      )						->    Optional[int]:




											'''simple docstring'''



											__a				:	Union[str, Any]			=				ModelOutputTest(a=30      )
											self.assertEqual(list(x.keys()      )   ,				["a"]      )
											self.assertEqual(list(x.values()      )   ,				[30]      )
											self.assertEqual(list(x.items()      )   ,				[("a", 30)]      )
											self.assertEqual(list(__UpperCAmelCase      )   ,				["a"]      )

											__a				:	List[str]			=				ModelOutputTest(a=30   ,				b=10      )
											self.assertEqual(list(x.keys()      )   ,				["a", "b"]      )
											self.assertEqual(list(x.values()      )   ,				[30, 10]      )
											self.assertEqual(list(x.items()      )   ,				[("a", 30), ("b", 10)]      )
											self.assertEqual(list(__UpperCAmelCase      )   ,				["a", "b"]      )

											__a				:	str			=				ModelOutputTest(a=30   ,				c=10      )
											self.assertEqual(list(x.keys()      )   ,				["a", "c"]      )
											self.assertEqual(list(x.values()      )   ,				[30, 10]      )
											self.assertEqual(list(x.items()      )   ,				[("a", 30), ("c", 10)]      )
											self.assertEqual(list(__UpperCAmelCase      )   ,				["a", "c"]      )

											with self.assertRaises(__UpperCAmelCase      ):
																__a				:	Dict			=				x.update({"d": 20}      )
											with self.assertRaises(__UpperCAmelCase      ):
																del x["a"]
											with self.assertRaises(__UpperCAmelCase      ):
																__a				:	Any			=				x.pop("a"      )
											with self.assertRaises(__UpperCAmelCase      ):
																__a				:	Any			=				x.setdefault("d"   ,				32      )



						def 					UpperCAmelCase__				(self:				Optional[Any]      )						->    Optional[Any]:




											'''simple docstring'''



											__a				:	Optional[Any]			=				ModelOutputTest(a=30      )
											__a				:	int			=				10
											self.assertEqual(x.a   ,				10      )
											self.assertEqual(x["a"]   ,				10      )



						def 					UpperCAmelCase__				(self:				Optional[Any]      )						->    Any:




											'''simple docstring'''



											__a				:	Optional[int]			=				ModelOutputTest(a=30      )
											__a				:	int			=				10
											self.assertEqual(x.a   ,				10      )
											self.assertEqual(x["a"]   ,				10      )



						def 					UpperCAmelCase__				(self:				List[str]      )						->    Any:




											'''simple docstring'''



											__a				:	Dict			=				ModelOutputTest({"a": 30, "b": 10}      )
											self.assertEqual(list(x.keys()      )   ,				["a", "b"]      )
											self.assertEqual(x.a   ,				30      )
											self.assertEqual(x.b   ,				10      )



						def 					UpperCAmelCase__				(self:				Optional[Any]      )						->    List[str]:




											'''simple docstring'''



											__a				:	List[Any]			=				ModelOutputTest([("a", 30), ("b", 10)]      )
											self.assertEqual(list(x.keys()      )   ,				["a", "b"]      )
											self.assertEqual(x.a   ,				30      )
											self.assertEqual(x.b   ,				10      )

											with self.assertRaises(__UpperCAmelCase      ):
																__a				:	List[Any]			=				ModelOutputTest([("a", 30), (10, 10)]      )

											__a				:	Optional[Any]			=				ModelOutputTest(a=(30, 30)      )
											self.assertEqual(list(x.keys()      )   ,				["a"]      )
											self.assertEqual(x.a   ,				(30, 30)      )





