

"""simple docstring"""





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
							from tokenizers import Tokenizer
							from tokenizers.models import WordPiece







class        SCREAMING_SNAKE_CASE      (	unittest.TestCase ):






					"""simple docstring"""



					def  __lowerCAmelCase				( self    :       Any     ,lowercase_    :       int	):
						lowerCAmelCase__			:		List[str]						       =			list(tokenizer_class.max_model_input_sizes.keys()	)
						for model_name in sa_models[:1]:
							lowerCAmelCase__			:		Any						       =			tokenizer_class.from_pretrained(lowerCamelCase__	)
							self.assertIsNotNone(lowerCamelCase__	)
							self.assertIsInstance(lowerCamelCase__     ,lowerCamelCase__	)
							self.assertIsInstance(lowerCamelCase__     ,lowerCamelCase__	)

							for special_tok in tokenizer.all_special_tokens:
								self.assertIsInstance(lowerCamelCase__     ,lowerCamelCase__	)
								lowerCAmelCase__			:		Optional[int]						       =			tokenizer.convert_tokens_to_ids(lowerCamelCase__	)
								self.assertIsInstance(lowerCamelCase__     ,lowerCamelCase__	)







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       Union[str, Any]     ,lowercase_    :       Any = None	):
						lowerCAmelCase__			:		Tuple						       =			pickle.dumps(lowerCamelCase__	)
						self.assertIsNotNone(lowerCamelCase__	)

						lowerCAmelCase__			:		Any						       =			pickle.loads(lowerCamelCase__	)

						# Ensure is_fast is correctly restored
						self.assertEqual(be_restored.is_fast     ,be_original.is_fast	)

						# Ensure encodings are potentially correctly restored
						if be_original.is_fast:
							self.assertIsNotNone(be_restored.encodings	)
						else:
							self.assertIsNone(be_restored.encodings	)

						# Ensure the keys are the same
						for original_v, restored_v in zip(be_original.values()     ,be_restored.values()	):
							if equal_op:
								self.assertTrue(equal_op(lowerCamelCase__     ,lowerCamelCase__	)	)
							else:
								self.assertEqual(lowerCamelCase__     ,lowerCamelCase__	)







					@slow
					def  __lowerCAmelCase				( self    :       List[str]	):
						self.check_tokenizer_from_pretrained(lowerCamelCase__	)







					def  __lowerCAmelCase				( self    :       str	):
						self.assertEqual(TensorType('''tf'''	)     ,TensorType.TENSORFLOW	)
						self.assertEqual(TensorType('''pt'''	)     ,TensorType.PYTORCH	)
						self.assertEqual(TensorType('''np'''	)     ,TensorType.NUMPY	)







					@require_tokenizers
					def  __lowerCAmelCase				( self    :       Optional[int]	):
						import numpy as np

						lowerCAmelCase__			:		int						       =			BertTokenizer.from_pretrained('''bert-base-cased'''	)
						lowerCAmelCase__			:		Dict						       =			BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

						# Python no tensor
						with self.subTest('''BatchEncoding (Python, return_tensors=None)'''	):
							self.assert_dump_and_restore(tokenizer_p('''Small example to encode'''	)	)

						with self.subTest('''BatchEncoding (Python, return_tensors=NUMPY)'''	):
							self.assert_dump_and_restore(
							    tokenizer_p('''Small example to encode'''     ,return_tensors=TensorType.NUMPY	)     ,np.array_equal	)

						with self.subTest('''BatchEncoding (Rust, return_tensors=None)'''	):
							self.assert_dump_and_restore(tokenizer_r('''Small example to encode'''	)	)

						with self.subTest('''BatchEncoding (Rust, return_tensors=NUMPY)'''	):
							self.assert_dump_and_restore(
							    tokenizer_r('''Small example to encode'''     ,return_tensors=TensorType.NUMPY	)     ,np.array_equal	)







					@require_tf
					@require_tokenizers
					def  __lowerCAmelCase				( self    :       Optional[Any]	):
						import tensorflow as tf

						def tf_array_equals(lowercase_    :       List[str]     ,lowercase_    :       Dict	):
							return tf.reduce_all(tf.equal(lowerCamelCase__     ,lowerCamelCase__	)	)

						lowerCAmelCase__			:		Optional[int]						       =			BertTokenizer.from_pretrained('''bert-base-cased'''	)
						lowerCAmelCase__			:		str						       =			BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

						with self.subTest('''BatchEncoding (Python, return_tensors=TENSORFLOW)'''	):
							self.assert_dump_and_restore(
							    tokenizer_p('''Small example to encode'''     ,return_tensors=TensorType.TENSORFLOW	)     ,lowerCamelCase__	)

						with self.subTest('''BatchEncoding (Rust, return_tensors=TENSORFLOW)'''	):
							self.assert_dump_and_restore(
							    tokenizer_r('''Small example to encode'''     ,return_tensors=TensorType.TENSORFLOW	)     ,lowerCamelCase__	)







					@require_torch
					@require_tokenizers
					def  __lowerCAmelCase				( self    :       int	):
						import torch

						lowerCAmelCase__			:		List[str]						       =			BertTokenizer.from_pretrained('''bert-base-cased'''	)
						lowerCAmelCase__			:		Dict						       =			BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

						with self.subTest('''BatchEncoding (Python, return_tensors=PYTORCH)'''	):
							self.assert_dump_and_restore(
							    tokenizer_p('''Small example to encode'''     ,return_tensors=TensorType.PYTORCH	)     ,torch.equal	)

						with self.subTest('''BatchEncoding (Rust, return_tensors=PYTORCH)'''	):
							self.assert_dump_and_restore(
							    tokenizer_r('''Small example to encode'''     ,return_tensors=TensorType.PYTORCH	)     ,torch.equal	)







					@require_tokenizers
					def  __lowerCAmelCase				( self    :       Optional[Any]	):
						lowerCAmelCase__			:		List[Any]						       =			BertTokenizer.from_pretrained('''bert-base-cased'''	)
						lowerCAmelCase__			:		List[str]						       =			BertTokenizerFast.from_pretrained('''bert-base-cased'''	)

						with self.subTest('''Python Tokenizer'''	):
							self.assertFalse(tokenizer_p('''Small example to_encode'''	).is_fast	)

						with self.subTest('''Rust Tokenizer'''	):
							self.assertTrue(tokenizer_r('''Small example to_encode'''	).is_fast	)







					@require_tokenizers
					def  __lowerCAmelCase				( self    :       List[Any]	):
						lowerCAmelCase__			:		Tuple						       =			BertTokenizerFast.from_pretrained('''bert-base-cased'''	)
						lowerCAmelCase__			:		Any						       =			tokenizer_r(['''Test''', '''\xad''', '''test''']     ,is_split_into_words=lowerCamelCase__	)

						self.assertEqual(encoded.word_to_tokens(0	)     ,TokenSpan(start=1     ,end=2	)	)
						self.assertEqual(encoded.word_to_tokens(1	)     ,lowerCamelCase__	)
						self.assertEqual(encoded.word_to_tokens(2	)     ,TokenSpan(start=2     ,end=3	)	)







					def  __lowerCAmelCase				( self    :       Dict	):
						lowerCAmelCase__			:		Optional[int]						       =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
						lowerCAmelCase__			:		List[Any]						       =			batch.convert_to_tensors(tensor_type='''np'''	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(2, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(2,)	)
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCAmelCase__			:		List[Any]						       =			batch.convert_to_tensors(tensor_type='''np'''	)
						self.assertFalse(len(cs.err	)     ,msg=F'should have no warning, but got {cs.err}'	)

						lowerCAmelCase__			:		List[Any]						       =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
						lowerCAmelCase__			:		Tuple						       =			batch.convert_to_tensors(tensor_type='''np'''     ,prepend_batch_axis=lowerCamelCase__	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(1, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(1,)	)







					@require_torch
					def  __lowerCAmelCase				( self    :       Any	):
						lowerCAmelCase__			:		Tuple						       =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
						lowerCAmelCase__			:		Union[str, Any]						       =			batch.convert_to_tensors(tensor_type='''pt'''	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(2, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(2,)	)
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCAmelCase__			:		Optional[int]						       =			batch.convert_to_tensors(tensor_type='''pt'''	)
						self.assertFalse(len(cs.err	)     ,msg=F'should have no warning, but got {cs.err}'	)

						lowerCAmelCase__			:		Union[str, Any]						       =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
						lowerCAmelCase__			:		Union[str, Any]						       =			batch.convert_to_tensors(tensor_type='''pt'''     ,prepend_batch_axis=lowerCamelCase__	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(1, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(1,)	)







					@require_tf
					def  __lowerCAmelCase				( self    :       List[Any]	):
						lowerCAmelCase__			:		Optional[Any]						       =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
						lowerCAmelCase__			:		Optional[int]						       =			batch.convert_to_tensors(tensor_type='''tf'''	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(2, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(2,)	)
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCAmelCase__			:		List[str]						       =			batch.convert_to_tensors(tensor_type='''tf'''	)
						self.assertFalse(len(cs.err	)     ,msg=F'should have no warning, but got {cs.err}'	)

						lowerCAmelCase__			:		Any						       =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
						lowerCAmelCase__			:		Optional[int]						       =			batch.convert_to_tensors(tensor_type='''tf'''     ,prepend_batch_axis=lowerCamelCase__	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(1, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(1,)	)







					@require_flax
					def  __lowerCAmelCase				( self    :       Dict	):
						lowerCAmelCase__			:		Optional[Any]						       =			BatchEncoding({'''inputs''': [[1, 2, 3], [4, 5, 6]], '''labels''': [0, 1]}	)
						lowerCAmelCase__			:		Any						       =			batch.convert_to_tensors(tensor_type='''jax'''	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(2, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(2,)	)
						# test converting the converted
						with CaptureStderr() as cs:
							lowerCAmelCase__			:		Union[str, Any]						       =			batch.convert_to_tensors(tensor_type='''jax'''	)
						self.assertFalse(len(cs.err	)     ,msg=F'should have no warning, but got {cs.err}'	)

						lowerCAmelCase__			:		Union[str, Any]						       =			BatchEncoding({'''inputs''': [1, 2, 3], '''labels''': 0}	)
						lowerCAmelCase__			:		List[Any]						       =			batch.convert_to_tensors(tensor_type='''jax'''     ,prepend_batch_axis=lowerCamelCase__	)
						self.assertEqual(tensor_batch['''inputs'''].shape     ,(1, 3)	)
						self.assertEqual(tensor_batch['''labels'''].shape     ,(1,)	)







					def  __lowerCAmelCase				( self    :       Dict	):
						lowerCAmelCase__			:		str						       =			[{"input_ids": np.array([0, 1, 2]	)}, {"input_ids": np.array([0, 1, 2, 3]	)}]
						lowerCAmelCase__			:		List[str]						       =			BertTokenizer.from_pretrained('''bert-base-cased'''	)

						lowerCAmelCase__			:		List[Any]						       =			tokenizer.pad(lowerCamelCase__     ,padding=lowerCamelCase__	)
						self.assertTrue(isinstance(batch['''input_ids''']     ,np.ndarray	)	)
						self.assertEqual(batch['''input_ids'''].tolist()     ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
						lowerCAmelCase__			:		int						       =			tokenizer.pad(lowerCamelCase__     ,padding=lowerCamelCase__     ,return_tensors='''np'''	)
						self.assertTrue(isinstance(batch['''input_ids''']     ,np.ndarray	)	)
						self.assertEqual(batch['''input_ids'''].tolist()     ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)







					@require_torch
					def  __lowerCAmelCase				( self    :       Optional[int]	):
						import torch

						lowerCAmelCase__			:		Tuple						       =			[{"input_ids": torch.tensor([0, 1, 2]	)}, {"input_ids": torch.tensor([0, 1, 2, 3]	)}]
						lowerCAmelCase__			:		int						       =			BertTokenizer.from_pretrained('''bert-base-cased'''	)

						lowerCAmelCase__			:		str						       =			tokenizer.pad(lowerCamelCase__     ,padding=lowerCamelCase__	)
						self.assertTrue(isinstance(batch['''input_ids''']     ,torch.Tensor	)	)
						self.assertEqual(batch['''input_ids'''].tolist()     ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
						lowerCAmelCase__			:		Optional[Any]						       =			tokenizer.pad(lowerCamelCase__     ,padding=lowerCamelCase__     ,return_tensors='''pt'''	)
						self.assertTrue(isinstance(batch['''input_ids''']     ,torch.Tensor	)	)
						self.assertEqual(batch['''input_ids'''].tolist()     ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)







					@require_tf
					def  __lowerCAmelCase				( self    :       Dict	):
						import tensorflow as tf

						lowerCAmelCase__			:		Any						       =			[{"input_ids": tf.constant([0, 1, 2]	)}, {"input_ids": tf.constant([0, 1, 2, 3]	)}]
						lowerCAmelCase__			:		Dict						       =			BertTokenizer.from_pretrained('''bert-base-cased'''	)

						lowerCAmelCase__			:		Dict						       =			tokenizer.pad(lowerCamelCase__     ,padding=lowerCamelCase__	)
						self.assertTrue(isinstance(batch['''input_ids''']     ,tf.Tensor	)	)
						self.assertEqual(batch['''input_ids'''].numpy().tolist()     ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
						lowerCAmelCase__			:		str						       =			tokenizer.pad(lowerCamelCase__     ,padding=lowerCamelCase__     ,return_tensors='''tf'''	)
						self.assertTrue(isinstance(batch['''input_ids''']     ,tf.Tensor	)	)
						self.assertEqual(batch['''input_ids'''].numpy().tolist()     ,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)







					@require_tokenizers
					def  __lowerCAmelCase				( self    :       Dict	):
						lowerCAmelCase__			:		Union[str, Any]						       =			Tokenizer(WordPiece(unk_token='''[UNK]'''	)	)
						PreTrainedTokenizerFast(tokenizer_object=lowerCamelCase__	)







					@require_tokenizers
					def  __lowerCAmelCase				( self    :       List[Any]	):
						lowerCAmelCase__			:		List[str]						       =			Tokenizer(WordPiece(unk_token='''[UNK]'''	)	)
						with tempfile.TemporaryDirectory() as tmpdirname:
							bert_tokenizer.save(os.path.join(lowerCamelCase__     ,'''tokenizer.json'''	)	)
							PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowerCamelCase__     ,'''tokenizer.json'''	)	)







"""simple docstring"""





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import normalize, rescale, resize, to_channel_dimension_format
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging


__UpperCamelCase						:   List[Any]							   =	logging.get_logger(__name__)







class        SCREAMING_SNAKE_CASE      (	a_ ):






					"""simple docstring"""



					lowercase__       				= ["pixel_values"]







					def __init__( self    :       Tuple     ,lowercase_    :       bool = True     ,lowercase_    :       Optional[Dict[str, int]] = None     ,lowercase_    :       PILImageResampling = PILImageResampling.BILINEAR     ,lowercase_    :       bool = True     ,lowercase_    :       Union[int, float] = 1 / 2_5_5     ,lowercase_    :       bool = True     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,**lowercase_    :       Tuple     ,):
						super().__init__(**lowercase_	)
						lowerCAmelCase__			:		Any						       =			size if size is not None else {'''height''': 2_2_4, '''width''': 2_2_4}
						lowerCAmelCase__			:		Tuple						       =			get_size_dict(lowercase_	)
						lowerCAmelCase__			:		Tuple						       =			do_resize
						lowerCAmelCase__			:		Tuple						       =			do_rescale
						lowerCAmelCase__			:		Union[str, Any]						       =			do_normalize
						lowerCAmelCase__			:		Optional[int]						       =			size
						lowerCAmelCase__			:		List[Any]						       =			resample
						lowerCAmelCase__			:		int						       =			rescale_factor
						lowerCAmelCase__			:		int						       =			image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
						lowerCAmelCase__			:		Any						       =			image_std if image_std is not None else IMAGENET_STANDARD_STD







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       np.ndarray     ,lowercase_    :       Dict[str, int]     ,lowercase_    :       PILImageResampling = PILImageResampling.BILINEAR     ,lowercase_    :       Optional[Union[str, ChannelDimension]] = None     ,**lowercase_    :       str     ,):
						lowerCAmelCase__			:		Union[str, Any]						       =			get_size_dict(lowercase_	)
						if "height" not in size or "width" not in size:
							raise ValueError(F'The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}'	)
						return resize(
						    lowercase_     ,size=(size['''height'''], size['''width'''])     ,resample=lowercase_     ,data_format=lowercase_     ,**lowercase_	)







					def  __lowerCAmelCase				( self    :       List[Any]     ,lowercase_    :       np.ndarray     ,lowercase_    :       float     ,lowercase_    :       Optional[Union[str, ChannelDimension]] = None     ,**lowercase_    :       Tuple	):
						return rescale(lowercase_     ,scale=lowercase_     ,data_format=lowercase_     ,**lowercase_	)







					def  __lowerCAmelCase				( self    :       Dict     ,lowercase_    :       np.ndarray     ,lowercase_    :       Union[float, List[float]]     ,lowercase_    :       Union[float, List[float]]     ,lowercase_    :       Optional[Union[str, ChannelDimension]] = None     ,**lowercase_    :       Union[str, Any]     ,):
						return normalize(lowercase_     ,mean=lowercase_     ,std=lowercase_     ,data_format=lowercase_     ,**lowercase_	)







					def  __lowerCAmelCase				( self    :       List[str]     ,lowercase_    :       ImageInput     ,lowercase_    :       Optional[bool] = None     ,lowercase_    :       Dict[str, int] = None     ,lowercase_    :       PILImageResampling = None     ,lowercase_    :       Optional[bool] = None     ,lowercase_    :       Optional[float] = None     ,lowercase_    :       Optional[bool] = None     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,lowercase_    :       Optional[Union[float, List[float]]] = None     ,lowercase_    :       Optional[Union[str, TensorType]] = None     ,lowercase_    :       Union[str, ChannelDimension] = ChannelDimension.FIRST     ,**lowercase_    :       List[Any]     ,):
						lowerCAmelCase__			:		Optional[int]						       =			do_resize if do_resize is not None else self.do_resize
						lowerCAmelCase__			:		Any						       =			do_rescale if do_rescale is not None else self.do_rescale
						lowerCAmelCase__			:		Dict						       =			do_normalize if do_normalize is not None else self.do_normalize
						lowerCAmelCase__			:		Dict						       =			resample if resample is not None else self.resample
						lowerCAmelCase__			:		Dict						       =			rescale_factor if rescale_factor is not None else self.rescale_factor
						lowerCAmelCase__			:		Optional[Any]						       =			image_mean if image_mean is not None else self.image_mean
						lowerCAmelCase__			:		str						       =			image_std if image_std is not None else self.image_std

						lowerCAmelCase__			:		Any						       =			size if size is not None else self.size
						lowerCAmelCase__			:		Dict						       =			get_size_dict(lowercase_	)

						lowerCAmelCase__			:		Union[str, Any]						       =			make_list_of_images(lowercase_	)

						if not valid_images(lowercase_	):
							raise ValueError(
							    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
							    '''torch.Tensor, tf.Tensor or jax.ndarray.'''	)

						if do_resize and size is None:
							raise ValueError('''Size must be specified if do_resize is True.'''	)

						if do_rescale and rescale_factor is None:
							raise ValueError('''Rescale factor must be specified if do_rescale is True.'''	)

						# All transformations expect numpy arrays.
						lowerCAmelCase__			:		Dict						       =			[to_numpy_array(lowercase_	) for image in images]

						if do_resize:
							lowerCAmelCase__			:		Optional[Any]						       =			[self.resize(image=lowercase_     ,size=lowercase_     ,resample=lowercase_	) for image in images]

						if do_rescale:
							lowerCAmelCase__			:		List[Any]						       =			[self.rescale(image=lowercase_     ,scale=lowercase_	) for image in images]

						if do_normalize:
							lowerCAmelCase__			:		Dict						       =			[self.normalize(image=lowercase_     ,mean=lowercase_     ,std=lowercase_	) for image in images]

						lowerCAmelCase__			:		Tuple						       =			[to_channel_dimension_format(lowercase_     ,lowercase_	) for image in images]

						lowerCAmelCase__			:		int						       =			{'''pixel_values''': images}
						return BatchFeature(data=lowercase_     ,tensor_type=lowercase_	)




