




import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class       A      (		_lowerCAmelCase      ,    unittest.TestCase		):
     """simple docstring"""

     _snake_case:						List[str]  =ElectraTokenizer
     _snake_case:						Dict  =ElectraTokenizerFast
     _snake_case:						Dict  =True
     _snake_case:						List[Any]  =True
     _snake_case:						Dict  =filter_non_english




     def 			__snake_case (     self	: List[Any]      )					->							Dict:
           super().setUp()

           lowerCamelCase__   	=					[
               '[UNK]',
               '[CLS]',
               '[SEP]',
               '[PAD]',
               '[MASK]',
               'want',
               '##want',
               '##ed',
               'wa',
               'un',
               'runn',
               '##ing',
               ',',
               'low',
               'lowest',
           ]
           lowerCamelCase__   	=					os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES['vocab_file']      )
           with open(self.vocab_file						,	'w'						,	encoding='utf-8'      ) as vocab_writer:
                 vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )




     def 			__snake_case (     self	: Union[str, Any]						,	A__	: List[Any]      )					->							int:
           lowerCamelCase__   	=					'UNwant\u00E9d,running'
           lowerCamelCase__   	=					'unwanted, running'
           return input_text, output_text




     def 			__snake_case (     self	: Tuple      )					->							str:
           lowerCamelCase__   	=					self.tokenizer_class(self.vocab_file      )

           lowerCamelCase__   	=					tokenizer.tokenize('UNwant\u00E9d,running'      )
           self.assertListEqual(A__						,	['un', '##want', '##ed', ',', 'runn', '##ing']      )
           self.assertListEqual(tokenizer.convert_tokens_to_ids(A__      )						,	[9, 6, 7, 1_2, 1_0, 1_1]      )




     def 			__snake_case (     self	: Union[str, Any]      )					->							List[str]:
           if not self.test_rust_tokenizer:
                 return

           lowerCamelCase__   	=					self.get_tokenizer()
           lowerCamelCase__   	=					self.get_rust_tokenizer()

           lowerCamelCase__   	=					'UNwant\u00E9d,running'

           lowerCamelCase__   	=					tokenizer.tokenize(A__      )
           lowerCamelCase__   	=					rust_tokenizer.tokenize(A__      )
           self.assertListEqual(A__						,	A__      )

           lowerCamelCase__   	=					tokenizer.encode(A__						,	add_special_tokens=A__      )
           lowerCamelCase__   	=					rust_tokenizer.encode(A__						,	add_special_tokens=A__      )
           self.assertListEqual(A__						,	A__      )

           lowerCamelCase__   	=					self.get_rust_tokenizer()
           lowerCamelCase__   	=					tokenizer.encode(A__      )
           lowerCamelCase__   	=					rust_tokenizer.encode(A__      )
           self.assertListEqual(A__						,	A__      )

           # With lower casing
           lowerCamelCase__   	=					self.get_tokenizer(do_lower_case=A__      )
           lowerCamelCase__   	=					self.get_rust_tokenizer(do_lower_case=A__      )

           lowerCamelCase__   	=					'UNwant\u00E9d,running'

           lowerCamelCase__   	=					tokenizer.tokenize(A__      )
           lowerCamelCase__   	=					rust_tokenizer.tokenize(A__      )
           self.assertListEqual(A__						,	A__      )

           lowerCamelCase__   	=					tokenizer.encode(A__						,	add_special_tokens=A__      )
           lowerCamelCase__   	=					rust_tokenizer.encode(A__						,	add_special_tokens=A__      )
           self.assertListEqual(A__						,	A__      )

           lowerCamelCase__   	=					self.get_rust_tokenizer()
           lowerCamelCase__   	=					tokenizer.encode(A__      )
           lowerCamelCase__   	=					rust_tokenizer.encode(A__      )
           self.assertListEqual(A__						,	A__      )




     def 			__snake_case (     self	: int      )					->							Union[str, Any]:
           lowerCamelCase__   	=					BasicTokenizer()

           self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'      )						,	['ah', '\u535A', '\u63A8', 'zz']      )




     def 			__snake_case (     self	: Any      )					->							int:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )						,	['hello', '!', 'how', 'are', 'you', '?']      )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,	['hello']      )




     def 			__snake_case (     self	: Union[str, Any]      )					->							Tuple:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__						,	strip_accents=A__      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,	['hällo', '!', 'how', 'are', 'you', '?']      )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,	['h\u00E9llo']      )




     def 			__snake_case (     self	: Dict      )					->							Any:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__						,	strip_accents=A__      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,	['hallo', '!', 'how', 'are', 'you', '?']      )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,	['hello']      )




     def 			__snake_case (     self	: Any      )					->							str:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,	['hallo', '!', 'how', 'are', 'you', '?']      )
           self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )						,	['hello']      )




     def 			__snake_case (     self	: List[Any]      )					->							str:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )						,	['HeLLo', '!', 'how', 'Are', 'yoU', '?']      )




     def 			__snake_case (     self	: int      )					->							List[Any]:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__						,	strip_accents=A__      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,	['HäLLo', '!', 'how', 'Are', 'yoU', '?']      )




     def 			__snake_case (     self	: List[str]      )					->							Any:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__						,	strip_accents=A__      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )						,	['HaLLo', '!', 'how', 'Are', 'yoU', '?']      )




     def 			__snake_case (     self	: Tuple      )					->							Union[str, Any]:
           lowerCamelCase__   	=					BasicTokenizer(do_lower_case=A__						,	never_split=['[UNK]']      )

           self.assertListEqual(
               tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'      )						,	['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']      )




     def 			__snake_case (     self	: Union[str, Any]      )					->							List[str]:
           lowerCamelCase__   	=					['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

           lowerCamelCase__   	=					{}
           for i, token in enumerate(A__      ):
                 lowerCamelCase__   	=					i
           lowerCamelCase__   	=					WordpieceTokenizer(vocab=A__						,	unk_token='[UNK]'      )

           self.assertListEqual(tokenizer.tokenize(''      )						,	[]      )

           self.assertListEqual(tokenizer.tokenize('unwanted running'      )						,	['un', '##want', '##ed', 'runn', '##ing']      )

           self.assertListEqual(tokenizer.tokenize('unwantedX running'      )						,	['[UNK]', 'runn', '##ing']      )




     def 			__snake_case (     self	: Dict      )					->							str:
           self.assertTrue(_is_whitespace(' '      )      )
           self.assertTrue(_is_whitespace('\t'      )      )
           self.assertTrue(_is_whitespace('\r'      )      )
           self.assertTrue(_is_whitespace('\n'      )      )
           self.assertTrue(_is_whitespace('\u00A0'      )      )

           self.assertFalse(_is_whitespace('A'      )      )
           self.assertFalse(_is_whitespace('-'      )      )




     def 			__snake_case (     self	: List[Any]      )					->							Union[str, Any]:
           self.assertTrue(_is_control('\u0005'      )      )

           self.assertFalse(_is_control('A'      )      )
           self.assertFalse(_is_control(' '      )      )
           self.assertFalse(_is_control('\t'      )      )
           self.assertFalse(_is_control('\r'      )      )




     def 			__snake_case (     self	: Dict      )					->							Dict:
           self.assertTrue(_is_punctuation('-'      )      )
           self.assertTrue(_is_punctuation('$'      )      )
           self.assertTrue(_is_punctuation('`'      )      )
           self.assertTrue(_is_punctuation('.'      )      )

           self.assertFalse(_is_punctuation('A'      )      )
           self.assertFalse(_is_punctuation(' '      )      )




     def 			__snake_case (     self	: Tuple      )					->							Optional[Any]:
           lowerCamelCase__   	=					self.get_tokenizer()
           lowerCamelCase__   	=					self.get_rust_tokenizer()

           # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
           self.assertListEqual([tokenizer.tokenize(A__      ) for t in ['Test', '\xad', 'test']]						,	[['[UNK]'], [], ['[UNK]']]      )

           self.assertListEqual(
               [rust_tokenizer.tokenize(A__      ) for t in ['Test', '\xad', 'test']]						,	[['[UNK]'], [], ['[UNK]']]      )




     @slow
     def 			__snake_case (     self	: List[str]      )					->							Union[str, Any]:
           lowerCamelCase__   	=					self.tokenizer_class.from_pretrained('google/electra-base-discriminator'      )

           lowerCamelCase__   	=					tokenizer.encode('sequence builders'						,	add_special_tokens=A__      )
           lowerCamelCase__   	=					tokenizer.encode('multi-sequence build'						,	add_special_tokens=A__      )

           lowerCamelCase__   	=					tokenizer.build_inputs_with_special_tokens(A__      )
           lowerCamelCase__   	=					tokenizer.build_inputs_with_special_tokens(A__						,	A__      )

           assert encoded_sentence == [1_0_1] + text + [1_0_2]
           assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]




     def 			__snake_case (     self	: Dict      )					->							Union[str, Any]:
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
                       lowerCamelCase__   	=					self.rust_tokenizer_class.from_pretrained(A__						,	**A__      )

                       lowerCamelCase__   	=					F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
                       lowerCamelCase__   	=					tokenizer_r.encode_plus(
                           A__						,	return_attention_mask=A__						,	return_token_type_ids=A__						,	return_offsets_mapping=A__						,	add_special_tokens=A__						,	)

                       lowerCamelCase__   	=					tokenizer_r.do_lower_case if hasattr(A__						,	'do_lower_case'      ) else False
                       lowerCamelCase__   	=					(
                           [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), 'A'),
                               ((1, 2), ','),
                               ((3, 5), 'na'),
                               ((5, 6), '##ï'),
                               ((6, 8), '##ve'),
                               ((9, 1_5), tokenizer_r.mask_token),
                               ((1_6, 2_1), 'Allen'),
                               ((2_1, 2_3), '##NL'),
                               ((2_3, 2_4), '##P'),
                               ((2_5, 3_3), 'sentence'),
                               ((3_3, 3_4), '.'),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                           if not do_lower_case
                           else [
                               ((0, 0), tokenizer_r.cls_token),
                               ((0, 1), 'a'),
                               ((1, 2), ','),
                               ((3, 8), 'naive'),
                               ((9, 1_5), tokenizer_r.mask_token),
                               ((1_6, 2_1), 'allen'),
                               ((2_1, 2_3), '##nl'),
                               ((2_3, 2_4), '##p'),
                               ((2_5, 3_3), 'sentence'),
                               ((3_3, 3_4), '.'),
                               ((0, 0), tokenizer_r.sep_token),
                           ]
                       )

                       self.assertEqual(
                           [e[1] for e in expected_results]						,	tokenizer_r.convert_ids_to_tokens(tokens['input_ids']      )      )
                       self.assertEqual([e[0] for e in expected_results]						,	tokens['offset_mapping']      )




     def 			__snake_case (     self	: Optional[int]      )					->							List[str]:
           lowerCamelCase__   	=					['的', '人', '有']
           lowerCamelCase__   	=					''.join(A__      )
           for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""      ):
                       lowerCamelCase__   	=					True
                       lowerCamelCase__   	=					self.tokenizer_class.from_pretrained(A__						,	**A__      )
                       lowerCamelCase__   	=					self.rust_tokenizer_class.from_pretrained(A__						,	**A__      )

                       lowerCamelCase__   	=					tokenizer_p.encode(A__						,	add_special_tokens=A__      )
                       lowerCamelCase__   	=					tokenizer_r.encode(A__						,	add_special_tokens=A__      )

                       lowerCamelCase__   	=					tokenizer_r.convert_ids_to_tokens(A__      )
                       lowerCamelCase__   	=					tokenizer_p.convert_ids_to_tokens(A__      )

                       # it is expected that each Chinese character is not preceded by "##"
                       self.assertListEqual(A__						,	A__      )
                       self.assertListEqual(A__						,	A__      )

                       lowerCamelCase__   	=					False
                       lowerCamelCase__   	=					self.rust_tokenizer_class.from_pretrained(A__						,	**A__      )
                       lowerCamelCase__   	=					self.tokenizer_class.from_pretrained(A__						,	**A__      )

                       lowerCamelCase__   	=					tokenizer_r.encode(A__						,	add_special_tokens=A__      )
                       lowerCamelCase__   	=					tokenizer_p.encode(A__						,	add_special_tokens=A__      )

                       lowerCamelCase__   	=					tokenizer_r.convert_ids_to_tokens(A__      )
                       lowerCamelCase__   	=					tokenizer_p.convert_ids_to_tokens(A__      )

                       # it is expected that only the first Chinese character is not preceded by "##".
                       lowerCamelCase__   	=					[
                           F"""##{token}""" if idx != 0 else token for idx, token in enumerate(A__      )
                       ]
                       self.assertListEqual(A__						,	A__      )
                       self.assertListEqual(A__						,	A__      )





from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


__SCREAMING_SNAKE_CASE     : Dict      =logging.get_logger(__name__)


if is_vision_available():
      import PIL






class       A      (		_lowerCAmelCase		):
     """simple docstring"""

     _snake_case:						Any  =["pixel_values"]




     def __init__(     self	: Union[str, Any]						,	A__	: bool = True						,	A__	: Dict[str, int] = None						,	A__	: PILImageResampling = PILImageResampling.BICUBIC						,	A__	: bool = True						,	A__	: Dict[str, int] = None						,	A__	: bool = True						,	A__	: Union[int, float] = 1 / 2_5_5						,	A__	: bool = True						,	A__	: Optional[Union[float, List[float]]] = None						,	A__	: Optional[Union[float, List[float]]] = None						,	A__	: bool = True						,	**A__	: Optional[int]						,	)					->							None:
           super().__init__(**A__      )
           lowerCamelCase__   	=					size if size is not None else {'shortest_edge': 2_2_4}
           lowerCamelCase__   	=					get_size_dict(A__						,	default_to_square=A__      )
           lowerCamelCase__   	=					crop_size if crop_size is not None else {'height': 2_2_4, 'width': 2_2_4}
           lowerCamelCase__   	=					get_size_dict(A__      )

           lowerCamelCase__   	=					do_resize
           lowerCamelCase__   	=					size
           lowerCamelCase__   	=					resample
           lowerCamelCase__   	=					do_center_crop
           lowerCamelCase__   	=					crop_size
           lowerCamelCase__   	=					do_rescale
           lowerCamelCase__   	=					rescale_factor
           lowerCamelCase__   	=					do_normalize
           lowerCamelCase__   	=					image_mean if image_mean is not None else OPENAI_CLIP_MEAN
           lowerCamelCase__   	=					image_std if image_std is not None else OPENAI_CLIP_STD
           lowerCamelCase__   	=					do_convert_rgb




     def 			__snake_case (     self	: Union[str, Any]						,	A__	: np.ndarray						,	A__	: Dict[str, int]						,	A__	: PILImageResampling = PILImageResampling.BICUBIC						,	A__	: Optional[Union[str, ChannelDimension]] = None						,	**A__	: Optional[int]						,	)					->							np.ndarray:
           lowerCamelCase__   	=					get_size_dict(A__						,	default_to_square=A__      )
           lowerCamelCase__   	=					get_resize_output_image_size(
               A__						,	size=(size['height'], size['width'])						,	default_to_square=A__      )
           return resize(A__						,	size=A__						,	resample=A__						,	data_format=A__						,	**A__      )




     def 			__snake_case (     self	: int						,	A__	: np.ndarray						,	A__	: Dict[str, int]						,	A__	: Optional[Union[str, ChannelDimension]] = None						,	**A__	: List[Any]						,	)					->							np.ndarray:
           lowerCamelCase__   	=					get_size_dict(A__      )
           return center_crop(A__						,	size=(size['height'], size['width'])						,	data_format=A__						,	**A__      )




     def 			__snake_case (     self	: Dict						,	A__	: np.ndarray						,	A__	: Union[int, float]						,	A__	: Optional[Union[str, ChannelDimension]] = None						,	**A__	: List[Any]						,	)					->							Tuple:
           return rescale(A__						,	scale=A__						,	data_format=A__						,	**A__      )




     def 			__snake_case (     self	: int						,	A__	: np.ndarray						,	A__	: Union[float, List[float]]						,	A__	: Union[float, List[float]]						,	A__	: Optional[Union[str, ChannelDimension]] = None						,	**A__	: Tuple						,	)					->							np.ndarray:
           return normalize(A__						,	mean=A__						,	std=A__						,	data_format=A__						,	**A__      )




     def 			__snake_case (     self	: Tuple						,	A__	: ImageInput						,	A__	: bool = None						,	A__	: Dict[str, int] = None						,	A__	: PILImageResampling = None						,	A__	: bool = None						,	A__	: int = None						,	A__	: bool = None						,	A__	: float = None						,	A__	: bool = None						,	A__	: Optional[Union[float, List[float]]] = None						,	A__	: Optional[Union[float, List[float]]] = None						,	A__	: bool = None						,	A__	: Optional[Union[str, TensorType]] = None						,	A__	: Optional[ChannelDimension] = ChannelDimension.FIRST						,	**A__	: List[str]						,	)					->							PIL.Image.Image:
           lowerCamelCase__   	=					do_resize if do_resize is not None else self.do_resize
           lowerCamelCase__   	=					size if size is not None else self.size
           lowerCamelCase__   	=					get_size_dict(A__						,	default_to_square=A__      )
           lowerCamelCase__   	=					resample if resample is not None else self.resample
           lowerCamelCase__   	=					do_center_crop if do_center_crop is not None else self.do_center_crop
           lowerCamelCase__   	=					crop_size if crop_size is not None else self.crop_size
           lowerCamelCase__   	=					get_size_dict(A__      )
           lowerCamelCase__   	=					do_rescale if do_rescale is not None else self.do_rescale
           lowerCamelCase__   	=					rescale_factor if rescale_factor is not None else self.rescale_factor
           lowerCamelCase__   	=					do_normalize if do_normalize is not None else self.do_normalize
           lowerCamelCase__   	=					image_mean if image_mean is not None else self.image_mean
           lowerCamelCase__   	=					image_std if image_std is not None else self.image_std
           lowerCamelCase__   	=					do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

           lowerCamelCase__   	=					make_list_of_images(A__      )

           if not valid_images(A__      ):
                 raise ValueError(
                     'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
                     'torch.Tensor, tf.Tensor or jax.ndarray.'      )

           if do_resize and size is None:
                 raise ValueError('Size must be specified if do_resize is True.'      )

           if do_center_crop and crop_size is None:
                 raise ValueError('Crop size must be specified if do_center_crop is True.'      )

           if do_rescale and rescale_factor is None:
                 raise ValueError('Rescale factor must be specified if do_rescale is True.'      )

           if do_normalize and (image_mean is None or image_std is None):
                 raise ValueError('Image mean and std must be specified if do_normalize is True.'      )

           # PIL RGBA images are converted to RGB
           if do_convert_rgb:
                 lowerCamelCase__   	=					[convert_to_rgb(A__      ) for image in images]

           # All transformations expect numpy arrays.
           lowerCamelCase__   	=					[to_numpy_array(A__      ) for image in images]

           if do_resize:
                 lowerCamelCase__   	=					[self.resize(image=A__						,	size=A__						,	resample=A__      ) for image in images]

           if do_center_crop:
                 lowerCamelCase__   	=					[self.center_crop(image=A__						,	size=A__      ) for image in images]

           if do_rescale:
                 lowerCamelCase__   	=					[self.rescale(image=A__						,	scale=A__      ) for image in images]

           if do_normalize:
                 lowerCamelCase__   	=					[self.normalize(image=A__						,	mean=A__						,	std=A__      ) for image in images]

           lowerCamelCase__   	=					[to_channel_dimension_format(A__						,	A__      ) for image in images]

           lowerCamelCase__   	=					{'pixel_values': images}
           return BatchFeature(data=A__						,	tensor_type=A__      )