code: 
  
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  import concurrent.futures
  import json
  import os
  import shutil
  import tempfile
  import unittest
  
  from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from transformers.testing_utils import require_tokenizers
  
  from ..test_tokenization_common import TokenizerTesterMixin
  @require_tokenizers
  class 	a__				(				lowerCamelCase__			,     unittest.TestCase      ):
  
  
  
  
  
  			'''simple docstring'''
  
  			A	:							int							= PreTrainedTokenizerFast
  			A	:							Tuple							= False
  			A	:							Tuple							= True
  			A	:							Optional[int]							= '''tokenizer_file'''
  			def 				lowerCAmelCase    (     self       : Dict  )			->		str:
  						__A=  False  # because we don't have pretrained_vocab_files_map
  						super().setUp()
  						__A=  True
  
  						__A=  ['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
  						__A=  'SaulLu/dummy-tokenizer-bytelevel-bpe'
  
  						# Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
  						__A=  [(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]
  
  						__A=  PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
  						tokenizer.save_pretrained(self.tmpdirname  )
  			def 				lowerCAmelCase    (     self       : List[str]  )			->		int:
  						# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
  						# model
  						pass
  			def 				lowerCAmelCase    (     self       : List[str]  )			->		Tuple:
  						# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
  						# model
  						pass
  			def 				lowerCAmelCase    (     self       : Dict  )			->		int:
  						# We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
  						# model
  						pass
  			def 				lowerCAmelCase    (     self       : str  )			->		int:
  						# PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
  						pass
  			def 				lowerCAmelCase    (     self       : List[str]  )			->		Tuple:
  						__A=  self.tmpdirname
  						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
  												try:
  															__A=  tempfile.mkdtemp()
  															__A=  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,						**SCREAMING_SNAKE_CASE_  )
  
  															tokenizer.save_pretrained(self.tmpdirname  )
  															super().test_training_new_tokenizer()
  												finally:
  															# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  															# is restored
  															shutil.rmtree(self.tmpdirname  )
  															__A=  tmpdirname_orig
  			def 				lowerCAmelCase    (     self       : Tuple  )			->		Any:
  						__A=  self.tmpdirname
  						# Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
  						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  									with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
  												try:
  															__A=  tempfile.mkdtemp()
  															__A=  self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_ ,						**SCREAMING_SNAKE_CASE_  )
  
  															tokenizer.save_pretrained(self.tmpdirname  )
  															super().test_training_new_tokenizer_with_special_tokens_change()
  												finally:
  															# Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
  															# is restored
  															shutil.rmtree(self.tmpdirname  )
  															__A=  tmpdirname_orig
  
  			def 				lowerCAmelCase    (     self       : List[str]  )			->		Optional[Any]:
  						__A=  self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )
  
  						__A=  ('a' for _ in range(1_000  ))
  						__A=  tokenizer.train_new_from_iterator(text_iterator=SCREAMING_SNAKE_CASE_ ,						length=1_000 ,						vocab_size=50  )
  
  						__A=  new_tokenizer.encode('a🤗'  )
  						self.assertEqual(SCREAMING_SNAKE_CASE_ ,						[64, 172, 253, 97, 245]  )
  
  
  @require_tokenizers
  class 	a__				(				unittest.TestCase      ):
  
  
  
  
  
  			'''simple docstring'''
  
  			def 				lowerCAmelCase    (     self       : Optional[int]  )			->		Tuple:
  						__A=  AutoTokenizer.from_pretrained('bert-base-cased'  )
  						__A=  json.loads(tokenizer._tokenizer.to_str()  )
  						__A=  len(SCREAMING_SNAKE_CASE_  )
  
  						with tempfile.TemporaryDirectory() as tmp_dir:
  									# Hack to save this in the tokenizer_config.json
  									__A=  ['tokenizer.4.0.0.json']
  									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_  )
  									json.dump(SCREAMING_SNAKE_CASE_ ,						open(os.path.join(SCREAMING_SNAKE_CASE_ ,						'tokenizer.4.0.0.json'  ) ,						'w'  )  )
  
  									# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  									__A=  AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_  )
  									self.assertEqual(len(SCREAMING_SNAKE_CASE_  ) ,						len(SCREAMING_SNAKE_CASE_  ) + 1  )
  									__A=  json.loads(new_tokenizer._tokenizer.to_str()  )
  									self.assertIn('huggingface' ,						json_tokenizer['model']['vocab']  )
  
  									# Will need to be adjusted if we reach v42 and this test is still here.
  									# Should pick the old tokenizer file as the version of Transformers is < 4.0.0
  									shutil.move(os.path.join(SCREAMING_SNAKE_CASE_ ,						'tokenizer.4.0.0.json'  ) ,						os.path.join(SCREAMING_SNAKE_CASE_ ,						'tokenizer.42.0.0.json'  )  )
  									__A=  ['tokenizer.42.0.0.json']
  									tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_  )
  									__A=  AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_  )
  									self.assertEqual(len(SCREAMING_SNAKE_CASE_  ) ,						len(SCREAMING_SNAKE_CASE_  )  )
  									__A=  json.loads(new_tokenizer._tokenizer.to_str()  )
  									self.assertNotIn('huggingface' ,						json_tokenizer['model']['vocab']  )
  
  			def 				lowerCAmelCase    (     self       : str  )			->		List[str]:
  						# This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
  						__A=  'hf-internal-testing/test-two-tokenizers'
  
  						# This should pick the new tokenizer file as the version of Transformers is > 4.0.0
  						__A=  AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_  )
  						self.assertEqual(len(SCREAMING_SNAKE_CASE_  ) ,						28_997  )
  						__A=  json.loads(tokenizer._tokenizer.to_str()  )
  						self.assertIn('huggingface' ,						json_tokenizer['model']['vocab']  )
  
  						# Testing an older version by monkey-patching the version in the module it's used.
  						import transformers as old_transformers
  
  						__A=  '3.0.0'
  						__A=  old_transformers.models.auto.AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_  )
  						self.assertEqual(len(SCREAMING_SNAKE_CASE_  ) ,						28_996  )
  						__A=  json.loads(old_tokenizer._tokenizer.to_str()  )
  						self.assertNotIn('huggingface' ,						json_tokenizer['model']['vocab']  )
  
  
  @require_tokenizers
  class 	a__				(				unittest.TestCase      ):
  
  
  
  
  
  			'''simple docstring'''
  
  			def 				lowerCAmelCase    (     self       : List[Any]  )			->		Optional[Any]:
  						# See https://github.com/huggingface/transformers/pull/12550
  						# and https://github.com/huggingface/tokenizers/issues/537
  						__A=  PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'  )
  						__A=  'The Matrix is a 1999 science fiction action film.'
  
  						with concurrent.futures.ThreadPoolExecutor() as executor:
  									__A=  [executor.submit(self.fetch ,						SCREAMING_SNAKE_CASE_ ,						SCREAMING_SNAKE_CASE_  ) for i in range(10  )]
  									__A=  [future.result() for future in futures]
  									self.assertEqual(SCREAMING_SNAKE_CASE_ ,						[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )
  
  			def 				lowerCAmelCase    (     self       : Tuple ,						lowerCAmelCase_       : Any ,						lowerCAmelCase_       : Any  )			->		List[str]:
  						return tokenizer.encode(SCREAMING_SNAKE_CASE_ ,						truncation='longest_first' ,						padding='longest'  )
  
  
  
  
style_context: 
  
  
  '''simple docstring'''
  
  
  
  
  
  import numpy as np
  from cva import destroyAllWindows, imread, imshow, waitKey
  class 	a__				:
  
  
  
  
  
  			'''simple docstring'''
  
  			def __init__(     self       : Dict ,						lowerCAmelCase_       : Optional[Any] ,						lowerCAmelCase_       : int  )			->		Optional[int]:
  						__A=  0
  						# max greyscale value for #FFFFFF
  						__A=  int(self.get_greyscale(255 ,						255 ,						255  )  )
  
  						if not self.min_threshold < threshold < self.max_threshold:
  									__A=  F"""Factor value should be from 0 to {self.max_threshold}"""
  									raise ValueError(lowerCAmelCase_  )
  
  						__A=  input_img
  						__A=  threshold
  						__A,  __A=  self.input_img.shape[1], self.input_img.shape[0]
  
  						# error table size (+4 columns and +1 row) greater than input image because of
  						# lack of if statements
  						__A=  [
  						    [0 for _ in range(self.height + 4  )] for __ in range(self.width + 1  )
  						]
  						__A=  np.ones((self.width, self.height, 3) ,						np.uinta  ) * 255
  			@classmethod
  			def 				lowerCAmelCase    (     cls       : Tuple ,						lowerCAmelCase_       : int ,						lowerCAmelCase_       : int ,						lowerCAmelCase_       : int  )			->		float:
  						return 0.1_14 * blue + 0.5_87 * green + 0.21_26 * red
  
  			def 				lowerCAmelCase    (     self       : str  )			->		None:
  						for y in range(self.height  ):
  									for x in range(self.width  ):
  												__A=  int(self.get_greyscale(*self.input_img[y][x]  )  )
  												if self.threshold > greyscale + self.error_table[y][x]:
  															__A=  (0, 0, 0)
  															__A=  greyscale + self.error_table[x][y]
  												else:
  															__A=  (255, 255, 255)
  															__A=  greyscale + self.error_table[x][y] - 255
  												self.error_table[y][x + 1] += int(8 / 32 * current_error  )
  												self.error_table[y][x + 2] += int(4 / 32 * current_error  )
  												self.error_table[y + 1][x] += int(8 / 32 * current_error  )
  												self.error_table[y + 1][x + 1] += int(4 / 32 * current_error  )
  												self.error_table[y + 1][x + 2] += int(2 / 32 * current_error  )
  												self.error_table[y + 1][x - 1] += int(4 / 32 * current_error  )
  												self.error_table[y + 1][x - 2] += int(2 / 32 * current_error  )
  
  
  if __name__ == "__main__":
  					# create Burke's instances with original images in greyscale
  					UpperCAmelCase__      =							[
  					    Burkes(imread('''image_data/lena.jpg''', 1), threshold)
  					    for threshold in (1, 1_2_6, 1_3_0, 1_4_0)
  					]
  
  					for burkes in burkes_instances:
  										burkes.process()
  
  					for burkes in burkes_instances:
  										imshow(
  										    F"""Original image with dithering threshold: {burkes.threshold}""",
  										    burkes.output_img,
  										)
  
  					waitKey(0)
  					destroyAllWindows()
  
  
  
  