code: 
  '''simple docstring'''
  
  
  
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class 						lowerCamelCase			(					__lowerCamelCase   ,		unittest.TestCase				):
  	UpperCamelCase_     :				Dict								=  PreTrainedTokenizerFast
  	UpperCamelCase_     :				List[Any]								=  False
  	UpperCamelCase_     :				Any								=  True
  	UpperCamelCase_     :				int								=  'tokenizer_file'
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[Any] )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   False  # because we don't have pretrained_vocab_files_map
  				super().setUp()
  				SCREAMING_SNAKE_CASE    	=   True
  
  				SCREAMING_SNAKE_CASE    	=   ['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
  				SCREAMING_SNAKE_CASE    	=   '''SaulLu/dummy-tokenizer-bytelevel-bpe'''
  
  				# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
  				SCREAMING_SNAKE_CASE    	=   [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
  				SCREAMING_SNAKE_CASE    	=   PreTrainedTokenizerFast.from_pretrained(model_paths[0] )
  				tokenizer.save_pretrained(self.tmpdirname )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:str )		->			Any:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Dict )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str] )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str] )		->			Optional[Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Union[str, Any] )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.tmpdirname
  				# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
  										try:
  													SCREAMING_SNAKE_CASE    	=   tempfile.mkdtemp()
  													SCREAMING_SNAKE_CASE    	=   self.rust_tokenizer_class.from_pretrained(lowercase      ,    **lowercase )
  
  													tokenizer.save_pretrained(self.tmpdirname )
  													super().test_training_new_tokenizer()
  										finally:
  													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  													# is restored
  													shutil.rmtree(self.tmpdirname )
  													SCREAMING_SNAKE_CASE    	=   tmpdirname_orig
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str] )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.tmpdirname
  				# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  							with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
  										try:
  													SCREAMING_SNAKE_CASE    	=   tempfile.mkdtemp()
  													SCREAMING_SNAKE_CASE    	=   self.rust_tokenizer_class.from_pretrained(lowercase      ,    **lowercase )
  
  													tokenizer.save_pretrained(self.tmpdirname )
  													super().test_training_new_tokenizer_with_special_tokens_change()
  										finally:
  													# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  													# is restored
  													shutil.rmtree(self.tmpdirname )
  													SCREAMING_SNAKE_CASE    	=   tmpdirname_orig
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:str )		->			Any:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name )
  
  				SCREAMING_SNAKE_CASE    	=   ('''a''' for _ in range(1_0_0_0 ))
  				SCREAMING_SNAKE_CASE    	=   tokenizer.train_new_from_iterator(text_iterator=lowercase      ,    length=1_0_0_0      ,    vocab_size=5_0 )
  
  				SCREAMING_SNAKE_CASE    	=   new_tokenizer.encode('''a🤗''' )
  				self.assertEqual(lowercase      ,    [6_4, 1_7_2, 2_5_3, 9_7, 2_4_5] )
  
  
  
  
  
  @require_tokenizers
  class 						lowerCamelCase			(					unittest.TestCase				):
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Any )		->			Optional[int]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   AutoTokenizer.from_pretrained('''bert-base-cased''' )
  				SCREAMING_SNAKE_CASE    	=   json.loads(tokenizer._tokenizer.to_str() )
  				SCREAMING_SNAKE_CASE    	=   len(lowercase )
  
  				with tempfile.TemporaryDirectory() as tmp_dir:
  							# Hack to save this in the tokenizer_config.json
  							SCREAMING_SNAKE_CASE    	=   ['''tokenizer.4.0.0.json''']
  							tokenizer.save_pretrained(lowercase )
  							json.dump(lowercase      ,    open(os.path.join(lowercase      ,    '''tokenizer.4.0.0.json''' )      ,    '''w''' ) )
  
  							# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  							SCREAMING_SNAKE_CASE    	=   AutoTokenizer.from_pretrained(lowercase )
  							self.assertEqual(len(lowercase )      ,    len(lowercase ) + 1 )
  							SCREAMING_SNAKE_CASE    	=   json.loads(new_tokenizer._tokenizer.to_str() )
  							self.assertIn('''huggingface'''      ,    json_tokenizer['''model''']['''vocab'''] )
  
  							# Will need to be adjusted if we reach v42 and this test is still here.
  							# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
  							shutil.move(os.path.join(lowercase      ,    '''tokenizer.4.0.0.json''' )      ,    os.path.join(lowercase      ,    '''tokenizer.42.0.0.json''' ) )
  							SCREAMING_SNAKE_CASE    	=   ['''tokenizer.42.0.0.json''']
  							tokenizer.save_pretrained(lowercase )
  							SCREAMING_SNAKE_CASE    	=   AutoTokenizer.from_pretrained(lowercase )
  							self.assertEqual(len(lowercase )      ,    len(lowercase ) )
  							SCREAMING_SNAKE_CASE    	=   json.loads(new_tokenizer._tokenizer.to_str() )
  							self.assertNotIn('''huggingface'''      ,    json_tokenizer['''model''']['''vocab'''] )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any] )		->			Tuple:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   '''hf-internal-testing/test-two-tokenizers'''
  
  				# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  				SCREAMING_SNAKE_CASE    	=   AutoTokenizer.from_pretrained(lowercase )
  				self.assertEqual(len(lowercase )      ,    2_8_9_9_7 )
  				SCREAMING_SNAKE_CASE    	=   json.loads(tokenizer._tokenizer.to_str() )
  				self.assertIn('''huggingface'''      ,    json_tokenizer['''model''']['''vocab'''] )
  
  				# Testing an older version by monkey-patching the version in the module it's used.
  				import transformers as old_transformers
  
  				SCREAMING_SNAKE_CASE    	=   '''3.0.0'''
  				SCREAMING_SNAKE_CASE    	=   old_transformers.models.auto.AutoTokenizer.from_pretrained(lowercase )
  				self.assertEqual(len(lowercase )      ,    2_8_9_9_6 )
  				SCREAMING_SNAKE_CASE    	=   json.loads(old_tokenizer._tokenizer.to_str() )
  				self.assertNotIn('''huggingface'''      ,    json_tokenizer['''model''']['''vocab'''] )
  
  
  
  
  
  @require_tokenizers
  class 						lowerCamelCase			(					unittest.TestCase				):
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any] )		->			Any:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel''' )
  				SCREAMING_SNAKE_CASE    	=   '''The Matrix is a 1999 science fiction action film.'''
  
  				with concurrent.futures.ThreadPoolExecutor() as executor:
  							SCREAMING_SNAKE_CASE    	=   [executor.submit(self.fetch      ,    lowercase      ,    lowercase ) for i in range(1_0 )]
  							SCREAMING_SNAKE_CASE    	=   [future.result() for future in futures]
  							self.assertEqual(lowercase      ,    [[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0 )] )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:str      ,    lowercase			:int      ,    lowercase			:str )		->			Optional[int]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				return tokenizer.encode(lowercase      ,    truncation='''longest_first'''      ,    padding='''longest''' )
style_context: 
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class 						lowerCamelCase			(					__lowerCamelCase   ,		unittest.TestCase				):
  	UpperCamelCase_     :				List[str]								=  GPTNeoXJapaneseTokenizer
  	UpperCamelCase_     :				List[Any]								=  False
  	UpperCamelCase_     :				int								=  {'do_clean_text': False, 'add_prefix_space': False}
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[int] )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				super().setUp()
  
  				SCREAMING_SNAKE_CASE    	=   [
  				    '''こん''',
  				    '''こんに''',
  				    '''にちは''',
  				    '''ばんは''',
  				    '''世界,㔺界''',
  				    '''、''',
  				    '''。''',
  				    '''<BR>''',
  				    '''<SP>''',
  				    '''<TAB>''',
  				    '''<URL>''',
  				    '''<EMAIL>''',
  				    '''<TEL>''',
  				    '''<DATE>''',
  				    '''<PRICE>''',
  				    '''<BLOCK>''',
  				    '''<KIGOU>''',
  				    '''<U2000U2BFF>''',
  				    '''<|emoji1|>''',
  				    '''<unk>''',
  				    '''<|startoftext|>''',
  				    '''<|endoftext|>''',
  				]
  				SCREAMING_SNAKE_CASE    	=   {'''emoji''': {'''\ud83d\ude00''': '''<|emoji1|>'''}, '''emoji_inv''': {'''<|emoji1|>''': '''\ud83d\ude00'''}}  # 😀
  				SCREAMING_SNAKE_CASE    	=   {'''unk_token''': '''<unk>'''}
  
  				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''vocab_file'''] )
  				SCREAMING_SNAKE_CASE    	=   os.path.join(self.tmpdirname      ,    VOCAB_FILES_NAMES['''emoji_file'''] )
  				with open(self.vocab_file      ,    '''w'''      ,    encoding='''utf-8''' ) as vocab_writer:
  							vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )
  				with open(self.emoji_file      ,    '''w''' ) as emoji_writer:
  							emoji_writer.write(json.dumps(lowercase ) )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Dict      ,    **lowercase			:Optional[int] )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				kwargs.update(self.special_tokens_map )
  				return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,    **lowercase )
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Dict      ,    lowercase			:Union[str, Any] )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、㔺界。😀'''
  				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。 \nこんばんは、世界。😀'''
  				return input_text, output_text
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[int]      ,    lowercase			:Union[str, Any] )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE	,       SCREAMING_SNAKE_CASE    	=   self.get_input_output_texts(lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.encode(lowercase      ,    add_special_tokens=lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.decode(lowercase      ,    clean_up_tokenization_spaces=lowercase )
  				return text, ids
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Dict )		->			int:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass  # TODO add if relevant
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[str] )		->			List[str]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass  # TODO add if relevant
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any] )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass  # TODO add if relevant
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Optional[Any] )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.get_tokenizer()
  
  				# Testing tokenization
  				SCREAMING_SNAKE_CASE    	=   '''こんにちは、世界。　こんばんは、㔺界。'''
  				SCREAMING_SNAKE_CASE    	=   ['''こん''', '''にちは''', '''、''', '''世界''', '''。''', '''<SP>''', '''こん''', '''ばんは''', '''、''', '''㔺界''', '''。''']
  				SCREAMING_SNAKE_CASE    	=   tokenizer.tokenize(lowercase )
  				self.assertListEqual(lowercase      ,    lowercase )
  
  				# Testing conversion to ids without special tokens
  				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
  				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
  				self.assertListEqual(lowercase      ,    lowercase )
  
  				# Testing conversion to ids with special tokens
  				SCREAMING_SNAKE_CASE    	=   tokens + [tokenizer.unk_token]
  				SCREAMING_SNAKE_CASE    	=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
  				SCREAMING_SNAKE_CASE    	=   tokenizer.convert_tokens_to_ids(lowercase )
  				self.assertListEqual(lowercase      ,    lowercase )
  
  
  
  
  
  
  
  	@slow
  	def    snake_case__    (    self			:str )		->			Union[str, Any]:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				SCREAMING_SNAKE_CASE    	=   self.tokenizer_class.from_pretrained('''abeja/gpt-neox-japanese-2.7b''' )
  
  				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''ありがとう。'''      ,    add_special_tokens=lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.encode('''どういたしまして。'''      ,    add_special_tokens=lowercase )
  
  				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase )
  				SCREAMING_SNAKE_CASE    	=   tokenizer.build_inputs_with_special_tokens(lowercase      ,    lowercase )
  
  				assert encoded_sentence == ids_a
  				assert encoded_pair == ids_a + ids_a
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:List[Any] )		->			Dict:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass
  
  
  
  
  
  
  
  	def    snake_case__    (    self			:Any )		->			str:
  
  
  
  
  
  
  
  				"""simple docstring"""
  
  
  
  
  
  
  				pass