



import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
    import torch

if is_vision_available():
    from PIL import Image

    from transformers import CLIPImageProcessor





class 		UpperCAmelCase    (    unittest.TestCase					):





   def __init__(						self							,  _lowerCAmelCase							,  _lowerCAmelCase=7							,  _lowerCAmelCase=3							,  _lowerCAmelCase=18							,  _lowerCAmelCase=30							,  _lowerCAmelCase=400							,  _lowerCAmelCase=True							,  _lowerCAmelCase=None							,  _lowerCAmelCase=True							,  _lowerCAmelCase=None							,  _lowerCAmelCase=True							,  _lowerCAmelCase=[0.48_145_466, 0.4_578_275, 0.40_821_073]							,  _lowerCAmelCase=[0.26_862_954, 0.26_130_258, 0.27_577_711]							,  _lowerCAmelCase=True							,  ):
         _lowerCAmelCase			      =	size if size is not None else {'''shortest_edge''': 20}
         _lowerCAmelCase			      =	crop_size if crop_size is not None else {'''height''': 18, '''width''': 18}
         _lowerCAmelCase			      =	parent
         _lowerCAmelCase			      =	batch_size
         _lowerCAmelCase			      =	num_channels
         _lowerCAmelCase			      =	image_size
         _lowerCAmelCase			      =	min_resolution
         _lowerCAmelCase			      =	max_resolution
         _lowerCAmelCase			      =	do_resize
         _lowerCAmelCase			      =	size
         _lowerCAmelCase			      =	do_center_crop
         _lowerCAmelCase			      =	crop_size
         _lowerCAmelCase			      =	do_normalize
         _lowerCAmelCase			      =	image_mean
         _lowerCAmelCase			      =	image_std
         _lowerCAmelCase			      =	do_convert_rgb





   def 						__lowerCAmelCase				(						self      ):
         return {
             "do_resize": self.do_resize,
             "size": self.size,
             "do_center_crop": self.do_center_crop,
             "crop_size": self.crop_size,
             "do_normalize": self.do_normalize,
             "image_mean": self.image_mean,
             "image_std": self.image_std,
             "do_convert_rgb": self.do_convert_rgb,
         }





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase=False							,  _lowerCAmelCase=False							,  _lowerCAmelCase=False      ):
         assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

         if equal_resolution:
               _lowerCAmelCase			      =	[]
               for i in range(self.batch_size      ):
                     image_inputs.append(
                         np.random.randint(
                             255							,  size=(self.num_channels, self.max_resolution, self.max_resolution)							,  dtype=np.uinta      )      )
         else:
               _lowerCAmelCase			      =	[]
               for i in range(self.batch_size      ):
                     _lowerCAmelCase      , _lowerCAmelCase			      =	np.random.choice(np.arange(self.min_resolution							,  self.max_resolution      )							,  2      )
                     image_inputs.append(np.random.randint(255							,  size=(self.num_channels, width, height)							,  dtype=np.uinta      )      )

         if not numpify and not torchify:
               # PIL expects the channel dimension as last dimension
               _lowerCAmelCase			      =	[Image.fromarray(np.moveaxis(_lowerCAmelCase							,  0							,  -1      )      ) for x in image_inputs]

         if torchify:
               _lowerCAmelCase			      =	[torch.from_numpy(_lowerCAmelCase      ) for x in image_inputs]

         return image_inputs






@require_torch
@require_vision
class 		UpperCAmelCase    (    snake_case_							,unittest.TestCase					):
   SCREAMING_SNAKE_CASE__					      =  CLIPImageProcessor if is_vision_available() else None





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	CLIPImageProcessingTester(self      )





   @property
   def 						__lowerCAmelCase				(						self      ):
         return self.image_processor_tester.prepare_image_processor_dict()





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.image_processing_class(**self.image_processor_dict      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_resize'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''size'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_center_crop'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''center_crop'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_normalize'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''image_mean'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''image_std'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_convert_rgb'''      )      )





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.image_processing_class.from_dict(self.image_processor_dict      )
         self.assertEqual(image_processor.size							,  {'''shortest_edge''': 20}      )
         self.assertEqual(image_processor.crop_size							,  {'''height''': 18, '''width''': 18}      )

         _lowerCAmelCase			      =	self.image_processing_class.from_dict(self.image_processor_dict							,  size=42							,  crop_size=84      )
         self.assertEqual(image_processor.size							,  {'''shortest_edge''': 42}      )
         self.assertEqual(image_processor.crop_size							,  {'''height''': 84, '''width''': 84}      )





   def 						__lowerCAmelCase				(						self      ):
         pass





   def 						__lowerCAmelCase				(						self      ):
         # Initialize image_processing
         _lowerCAmelCase			      =	self.image_processing_class(**self.image_processor_dict      )
         # create random PIL images
         _lowerCAmelCase			      =	self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCAmelCase      )
         for image in image_inputs:
               self.assertIsInstance(_lowerCAmelCase							,  Image.Image      )

         # Test not batched input
         _lowerCAmelCase			      =	image_processing(image_inputs[0]							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )

         # Test batched
         _lowerCAmelCase			      =	image_processing(_lowerCAmelCase							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )





   def 						__lowerCAmelCase				(						self      ):
         # Initialize image_processing
         _lowerCAmelCase			      =	self.image_processing_class(**self.image_processor_dict      )
         # create random numpy tensors
         _lowerCAmelCase			      =	self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCAmelCase							,  numpify=_lowerCAmelCase      )
         for image in image_inputs:
               self.assertIsInstance(_lowerCAmelCase							,  np.ndarray      )

         # Test not batched input
         _lowerCAmelCase			      =	image_processing(image_inputs[0]							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )

         # Test batched
         _lowerCAmelCase			      =	image_processing(_lowerCAmelCase							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )





   def 						__lowerCAmelCase				(						self      ):
         # Initialize image_processing
         _lowerCAmelCase			      =	self.image_processing_class(**self.image_processor_dict      )
         # create random PyTorch tensors
         _lowerCAmelCase			      =	self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCAmelCase							,  torchify=_lowerCAmelCase      )
         for image in image_inputs:
               self.assertIsInstance(_lowerCAmelCase							,  torch.Tensor      )

         # Test not batched input
         _lowerCAmelCase			      =	image_processing(image_inputs[0]							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 1,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )

         # Test batched
         _lowerCAmelCase			      =	image_processing(_lowerCAmelCase							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 self.image_processor_tester.batch_size,
                 self.image_processor_tester.num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )






@require_torch
@require_vision
class 		UpperCAmelCase    (    snake_case_							,unittest.TestCase					):
   SCREAMING_SNAKE_CASE__					      =  CLIPImageProcessor if is_vision_available() else None





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	CLIPImageProcessingTester(self							,  num_channels=4      )
         _lowerCAmelCase			      =	3





   @property
   def 						__lowerCAmelCase				(						self      ):
         return self.image_processor_tester.prepare_image_processor_dict()





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.image_processing_class(**self.image_processor_dict      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_resize'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''size'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_center_crop'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''center_crop'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_normalize'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''image_mean'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''image_std'''      )      )
         self.assertTrue(hasattr(_lowerCAmelCase							,  '''do_convert_rgb'''      )      )





   def 						__lowerCAmelCase				(						self      ):
         pass





   def 						__lowerCAmelCase				(						self      ):
         # Initialize image_processing
         _lowerCAmelCase			      =	self.image_processing_class(**self.image_processor_dict      )
         # create random PIL images
         _lowerCAmelCase			      =	self.image_processor_tester.prepare_inputs(equal_resolution=_lowerCAmelCase      )
         for image in image_inputs:
               self.assertIsInstance(_lowerCAmelCase							,  Image.Image      )

         # Test not batched input
         _lowerCAmelCase			      =	image_processing(image_inputs[0]							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 1,
                 self.expected_encoded_image_num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )

         # Test batched
         _lowerCAmelCase			      =	image_processing(_lowerCAmelCase							,  return_tensors='''pt'''      ).pixel_values
         self.assertEqual(
             encoded_images.shape							,  (
                 self.image_processor_tester.batch_size,
                 self.expected_encoded_image_num_channels,
                 self.image_processor_tester.crop_size['''height'''],
                 self.image_processor_tester.crop_size['''width'''],
             )							,  )




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class 		UpperCAmelCase    (    snake_case_							,unittest.TestCase					):
   SCREAMING_SNAKE_CASE__					      =  PreTrainedTokenizerFast
   SCREAMING_SNAKE_CASE__					      =  False
   SCREAMING_SNAKE_CASE__					      =  True
   SCREAMING_SNAKE_CASE__					      =  '''tokenizer_file'''





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	False  # because we don't have pretrained_vocab_files_map
         super().setUp()
         _lowerCAmelCase			      =	True

         _lowerCAmelCase			      =	['''robot-test/dummy-tokenizer-fast''', '''robot-test/dummy-tokenizer-wordlevel''']
         _lowerCAmelCase			      =	'''SaulLu/dummy-tokenizer-bytelevel-bpe'''

         # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
         _lowerCAmelCase			      =	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

         _lowerCAmelCase			      =	PreTrainedTokenizerFast.from_pretrained(model_paths[0]      )
         tokenizer.save_pretrained(self.tmpdirname      )





   def 						__lowerCAmelCase				(						self      ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass





   def 						__lowerCAmelCase				(						self      ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass





   def 						__lowerCAmelCase				(						self      ):
         # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
         # model
         pass





   def 						__lowerCAmelCase				(						self      ):
         # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
         pass





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                     try:
                           _lowerCAmelCase			      =	tempfile.mkdtemp()
                           _lowerCAmelCase			      =	self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase							,  **_lowerCAmelCase      )

                           tokenizer.save_pretrained(self.tmpdirname      )
                           super().test_training_new_tokenizer()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname      )
                           _lowerCAmelCase			      =	tmpdirname_orig





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.tmpdirname
         # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
               with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''      ):
                     try:
                           _lowerCAmelCase			      =	tempfile.mkdtemp()
                           _lowerCAmelCase			      =	self.rust_tokenizer_class.from_pretrained(_lowerCAmelCase							,  **_lowerCAmelCase      )

                           tokenizer.save_pretrained(self.tmpdirname      )
                           super().test_training_new_tokenizer_with_special_tokens_change()
                     finally:
                           # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                           # is restored
                           shutil.rmtree(self.tmpdirname      )
                           _lowerCAmelCase			      =	tmpdirname_orig





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name      )

         _lowerCAmelCase			      =	('''a''' for _ in range(1_000      ))
         _lowerCAmelCase			      =	tokenizer.train_new_from_iterator(text_iterator=_lowerCAmelCase							,  length=1_000							,  vocab_size=50      )

         _lowerCAmelCase			      =	new_tokenizer.encode('''a🤗'''      )
         self.assertEqual(_lowerCAmelCase							,  [64, 172, 253, 97, 245]      )






@require_tokenizers
class 		UpperCAmelCase    (    unittest.TestCase					):





   def 						__lowerCAmelCase				(						self      ):
         _lowerCAmelCase			      =	AutoTokenizer.from_pretrained('''bert-base-cased'''      )
         _lowerCAmelCase			      =	json.loads(tokenizer._tokenizer.to_str()      )
         _lowerCAmelCase			      =	len(_lowerCAmelCase      )

         with tempfile.TemporaryDirectory() as tmp_dir:
               # Hack to save this in the tokenizer_config.json
               _lowerCAmelCase			      =	['''tokenizer.4.0.0.json''']
               tokenizer.save_pretrained(_lowerCAmelCase      )
               json.dump(_lowerCAmelCase							,  open(os.path.join(_lowerCAmelCase							,  '''tokenizer.4.0.0.json'''      )							,  '''w'''      )      )

               # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
               _lowerCAmelCase			      =	AutoTokenizer.from_pretrained(_lowerCAmelCase      )
               self.assertEqual(len(_lowerCAmelCase      )							,  len(_lowerCAmelCase      ) + 1      )
               _lowerCAmelCase			      =	json.loads(new_tokenizer._tokenizer.to_str()      )
               self.assertIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )

               # Will need to be adjusted if we reach v42 and this test is still here.
               # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
               shutil.move(os.path.join(_lowerCAmelCase							,  '''tokenizer.4.0.0.json'''      )							,  os.path.join(_lowerCAmelCase							,  '''tokenizer.42.0.0.json'''      )      )
               _lowerCAmelCase			      =	['''tokenizer.42.0.0.json''']
               tokenizer.save_pretrained(_lowerCAmelCase      )
               _lowerCAmelCase			      =	AutoTokenizer.from_pretrained(_lowerCAmelCase      )
               self.assertEqual(len(_lowerCAmelCase      )							,  len(_lowerCAmelCase      )      )
               _lowerCAmelCase			      =	json.loads(new_tokenizer._tokenizer.to_str()      )
               self.assertNotIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )





   def 						__lowerCAmelCase				(						self      ):
         # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
         _lowerCAmelCase			      =	'''hf-internal-testing/test-two-tokenizers'''

         # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
         _lowerCAmelCase			      =	AutoTokenizer.from_pretrained(_lowerCAmelCase      )
         self.assertEqual(len(_lowerCAmelCase      )							,  28_997      )
         _lowerCAmelCase			      =	json.loads(tokenizer._tokenizer.to_str()      )
         self.assertIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )

         # Testing an older version by monkey-patching the version in the module it's used.
         import transformers as old_transformers

         _lowerCAmelCase			      =	'''3.0.0'''
         _lowerCAmelCase			      =	old_transformers.models.auto.AutoTokenizer.from_pretrained(_lowerCAmelCase      )
         self.assertEqual(len(_lowerCAmelCase      )							,  28_996      )
         _lowerCAmelCase			      =	json.loads(old_tokenizer._tokenizer.to_str()      )
         self.assertNotIn('''huggingface'''							,  json_tokenizer['''model''']['''vocab''']      )






@require_tokenizers
class 		UpperCAmelCase    (    unittest.TestCase					):





   def 						__lowerCAmelCase				(						self      ):
         # See https://github.com/huggingface/transformers/pull/12550
         # and https://github.com/huggingface/tokenizers/issues/537
         _lowerCAmelCase			      =	PreTrainedTokenizerFast.from_pretrained('''robot-test/dummy-tokenizer-wordlevel'''      )
         _lowerCAmelCase			      =	'''The Matrix is a 1999 science fiction action film.'''

         with concurrent.futures.ThreadPoolExecutor() as executor:
               _lowerCAmelCase			      =	[executor.submit(self.fetch							,  _lowerCAmelCase							,  _lowerCAmelCase      ) for i in range(10      )]
               _lowerCAmelCase			      =	[future.result() for future in futures]
               self.assertEqual(_lowerCAmelCase							,  [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10      )]      )





   def 						__lowerCAmelCase				(						self							,  _lowerCAmelCase							,  _lowerCAmelCase      ):
         return tokenizer.encode(_lowerCAmelCase							,  truncation='''longest_first'''							,  padding='''longest'''      )