code: 
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  @require_tokenizers
  class 					_UpperCamelCase (      lowerCamelCase__			,							unittest.TestCase	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  							_A  				=  PreTrainedTokenizerFast
  							_A  				=  False
  							_A  				=  True
  							_A  				=  "tokenizer_file"
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Dict		):
  													_a     =			False  # because we don't have pretrained_vocab_files_map
  													super().setUp()
  													_a     =			True
  
  													_a     =			['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
  													_a     =			'SaulLu/dummy-tokenizer-bytelevel-bpe'
  
  													# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
  													_a     =			[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
  													_a     =			PreTrainedTokenizerFast.from_pretrained(model_paths[0]		)
  													tokenizer.save_pretrained(self.tmpdirname		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		):
  													# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
  													# model
  													pass
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		):
  													# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
  													# model
  													pass
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Dict		):
  													# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
  													# model
  													pass
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				str		):
  													# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
  													pass
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		):
  													_a     =			self.tmpdirname
  													# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  																			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
  																									try:
  																															_a     =			tempfile.mkdtemp()
  																															_a     =			self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_		,				**SCREAMING_SNAKE_CASE_		)
  
  																															tokenizer.save_pretrained(self.tmpdirname		)
  																															super().test_training_new_tokenizer()
  																									finally:
  																															# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  																															# is restored
  																															shutil.rmtree(self.tmpdirname		)
  																															_a     =			tmpdirname_orig
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Tuple		):
  													_a     =			self.tmpdirname
  													# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  																			with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
  																									try:
  																															_a     =			tempfile.mkdtemp()
  																															_a     =			self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_		,				**SCREAMING_SNAKE_CASE_		)
  
  																															tokenizer.save_pretrained(self.tmpdirname		)
  																															super().test_training_new_tokenizer_with_special_tokens_change()
  																									finally:
  																															# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  																															# is restored
  																															shutil.rmtree(self.tmpdirname		)
  																															_a     =			tmpdirname_orig
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[str]		):
  													_a     =			self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name		)
  
  													_a     =			('a' for _ in range(1_0_0_0		))
  													_a     =			tokenizer.train_new_from_iterator(text_iterator=SCREAMING_SNAKE_CASE_		,				length=1_0_0_0		,				vocab_size=5_0		)
  
  													_a     =			new_tokenizer.encode('a🤗'		)
  													self.assertEqual(SCREAMING_SNAKE_CASE_		,				[6_4, 1_7_2, 2_5_3, 9_7, 2_4_5]		)
  
  
  
  
  @require_tokenizers
  class 					_UpperCamelCase (      unittest.TestCase	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Optional[int]		):
  													_a     =			AutoTokenizer.from_pretrained('bert-base-cased'		)
  													_a     =			json.loads(tokenizer._tokenizer.to_str()		)
  													_a     =			len(SCREAMING_SNAKE_CASE_		)
  
  													with tempfile.TemporaryDirectory() as tmp_dir:
  																			# Hack to save this in the tokenizer_config.json
  																			_a     =			['tokenizer.4.0.0.json']
  																			tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_		)
  																			json.dump(SCREAMING_SNAKE_CASE_		,				open(os.path.join(SCREAMING_SNAKE_CASE_		,				'tokenizer.4.0.0.json'		)		,				'w'		)		)
  
  																			# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  																			_a     =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_		)
  																			self.assertEqual(len(SCREAMING_SNAKE_CASE_		)		,				len(SCREAMING_SNAKE_CASE_		) + 1		)
  																			_a     =			json.loads(new_tokenizer._tokenizer.to_str()		)
  																			self.assertIn('huggingface'		,				json_tokenizer['model']['vocab']		)
  
  																			# Will need to be adjusted if we reach v42 and this test is still here.
  																			# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
  																			shutil.move(os.path.join(SCREAMING_SNAKE_CASE_		,				'tokenizer.4.0.0.json'		)		,				os.path.join(SCREAMING_SNAKE_CASE_		,				'tokenizer.42.0.0.json'		)		)
  																			_a     =			['tokenizer.42.0.0.json']
  																			tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_		)
  																			_a     =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_		)
  																			self.assertEqual(len(SCREAMING_SNAKE_CASE_		)		,				len(SCREAMING_SNAKE_CASE_		)		)
  																			_a     =			json.loads(new_tokenizer._tokenizer.to_str()		)
  																			self.assertNotIn('huggingface'		,				json_tokenizer['model']['vocab']		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				str		):
  													# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
  													_a     =			'hf-internal-testing/test-two-tokenizers'
  
  													# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  													_a     =			AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_		)
  													self.assertEqual(len(SCREAMING_SNAKE_CASE_		)		,				2_8_9_9_7		)
  													_a     =			json.loads(tokenizer._tokenizer.to_str()		)
  													self.assertIn('huggingface'		,				json_tokenizer['model']['vocab']		)
  
  													# Testing an older version by monkey-patching the version in the module it's used.
  													import transformers as old_transformers
  
  													_a     =			'3.0.0'
  													_a     =			old_transformers.models.auto.AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_		)
  													self.assertEqual(len(SCREAMING_SNAKE_CASE_		)		,				2_8_9_9_6		)
  													_a     =			json.loads(old_tokenizer._tokenizer.to_str()		)
  													self.assertNotIn('huggingface'		,				json_tokenizer['model']['vocab']		)
  
  
  
  
  @require_tokenizers
  class 					_UpperCamelCase (      unittest.TestCase	):
  
  
  
  
  
  
  
  							'''simple docstring'''
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				List[Any]		):
  													# See https://github.com/huggingface/transformers/pull/12550
  													# and https://github.com/huggingface/tokenizers/issues/537
  													_a     =			PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'		)
  													_a     =			'The Matrix is a 1999 science fiction action film.'
  
  													with concurrent.futures.ThreadPoolExecutor() as executor:
  																			_a     =			[executor.submit(self.fetch		,				SCREAMING_SNAKE_CASE_		,				SCREAMING_SNAKE_CASE_		) for i in range(1_0		)]
  																			_a     =			[future.result() for future in futures]
  																			self.assertEqual(SCREAMING_SNAKE_CASE_		,				[[1, 1_0, 0, 8, 0, 1_8, 0, 0, 0, 2] for i in range(1_0		)]		)
  
  
  
  
  
  
  
  							def 		_UpperCAmelCase	(		self  :				Tuple		,				SCREAMING_SNAKE_CASE_  :				Any		,				SCREAMING_SNAKE_CASE_  :				Any		):
  													return tokenizer.encode(SCREAMING_SNAKE_CASE_		,				truncation='longest_first'		,				padding='longest'		)
  
  
  
style_context: 
  
  
  import numpy as np
  import PIL
  import torch
  import torchvision.transforms as T
  import torchvision.transforms.functional as TF
  from PIL import Image
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase							,     _UpperCAmelCase=256				)    ->  Dict:
  						_a     =			min(img.size				)
  
  						if s < target_image_size:
  												raise ValueError(f"""min dim for image {s} < {target_image_size}"""				)
  
  						_a     =			target_image_size / s
  						_a     =			(round(r * img.size[1]				), round(r * img.size[0]				))
  						_a     =			TF.resize(_UpperCAmelCase							,     _UpperCAmelCase							,     interpolation=PIL.Image.LANCZOS				)
  						_a     =			TF.center_crop(_UpperCAmelCase							,     output_size=2 * [target_image_size]				)
  						_a     =			torch.unsqueeze(T.ToTensor()(_UpperCAmelCase				)							,     0				)
  						return img
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  List[Any]:
  						_a     =			2.0 * x - 1.0
  						return x
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase							,     _UpperCAmelCase=True							,     _UpperCAmelCase="RGB"				)    ->  Optional[Any]:
  						_a     =			x.detach().cpu()
  						if process:
  												_a     =			post_process_tensor(_UpperCAmelCase				)
  						_a     =			x.numpy()
  						if process:
  												_a     =			(255 * x).astype(np.uinta				)
  						_a     =			Image.fromarray(_UpperCAmelCase				)
  						if not x.mode == mode:
  												_a     =			x.convert(_UpperCAmelCase				)
  						return x
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  Optional[Any]:
  						_a     =			torch.clamp(_UpperCAmelCase							,     -1.0							,     1.0				)
  						_a     =			(x + 1.0) / 2.0
  						_a     =			x.permute(1							,     2							,     0				)
  						return x
  
  
  
  
  
  
  def 							SCREAMING_SNAKE_CASE					(  _UpperCAmelCase				)    ->  str:
  						_a     =			post_process_tensor(x.squeeze()				)
  						return x.permute(2							,     0							,     1				).unsqueeze(0				)
  
  
  