



"""simple docstring"""







import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class 			_a     (   SCREAMING_SNAKE_CASE__    ,  unittest.TestCase):
			__magic_name__    				=    ElectraTokenizer
			__magic_name__    				=    ElectraTokenizerFast
			__magic_name__    				=    True
			__magic_name__    				=    True
			__magic_name__    				=    filter_non_english



			def 							__lowercase				( self       :							Optional[Any]		)       ->				Tuple:
									super().setUp()

									snake_case      : Dict							=   [
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "[PAD]",
									    "[MASK]",
									    "want",
									    "##want",
									    "##ed",
									    "wa",
									    "un",
									    "runn",
									    "##ing",
									    ",",
									    "low",
									    "lowest",
									]
									snake_case      : Optional[Any]							=   os.path.join(self.tmpdirname      ,  VOCAB_FILES_NAMES["vocab_file"]		)
									with open(self.vocab_file      ,  "w"      ,  encoding="utf-8"		) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]		)		)



			def 							__lowercase				( self       :							Optional[Any]      ,  _lowercase       :							int		)       ->				Tuple:
									snake_case      : List[Any]							=   "UNwant\u00E9d,running"
									snake_case      : Any							=   "unwanted, running"
									return input_text, output_text



			def 							__lowercase				( self       :							List[str]		)       ->				Tuple:
									snake_case      : str							=   self.tokenizer_class(self.vocab_file		)

									snake_case      : Any							=   tokenizer.tokenize("UNwant\u00E9d,running"		)
									self.assertListEqual(_lowercase      ,  ["un", "##want", "##ed", ",", "runn", "##ing"]		)
									self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowercase		)      ,  [9, 6, 7, 12, 10, 11]		)



			def 							__lowercase				( self       :							str		)       ->				List[Any]:
									if not self.test_rust_tokenizer:
															return

									snake_case      : Any							=   self.get_tokenizer()
									snake_case      : str							=   self.get_rust_tokenizer()

									snake_case      : Optional[int]							=   "UNwant\u00E9d,running"

									snake_case      : Dict							=   tokenizer.tokenize(_lowercase		)
									snake_case      : Any							=   rust_tokenizer.tokenize(_lowercase		)
									self.assertListEqual(_lowercase      ,  _lowercase		)

									snake_case      : List[str]							=   tokenizer.encode(_lowercase      ,  add_special_tokens=_lowercase		)
									snake_case      : Optional[Any]							=   rust_tokenizer.encode(_lowercase      ,  add_special_tokens=_lowercase		)
									self.assertListEqual(_lowercase      ,  _lowercase		)

									snake_case      : Union[str, Any]							=   self.get_rust_tokenizer()
									snake_case      : Optional[Any]							=   tokenizer.encode(_lowercase		)
									snake_case      : Tuple							=   rust_tokenizer.encode(_lowercase		)
									self.assertListEqual(_lowercase      ,  _lowercase		)

									# With lower casing
									snake_case      : int							=   self.get_tokenizer(do_lower_case=_lowercase		)
									snake_case      : List[str]							=   self.get_rust_tokenizer(do_lower_case=_lowercase		)

									snake_case      : Any							=   "UNwant\u00E9d,running"

									snake_case      : Union[str, Any]							=   tokenizer.tokenize(_lowercase		)
									snake_case      : Optional[Any]							=   rust_tokenizer.tokenize(_lowercase		)
									self.assertListEqual(_lowercase      ,  _lowercase		)

									snake_case      : Union[str, Any]							=   tokenizer.encode(_lowercase      ,  add_special_tokens=_lowercase		)
									snake_case      : Tuple							=   rust_tokenizer.encode(_lowercase      ,  add_special_tokens=_lowercase		)
									self.assertListEqual(_lowercase      ,  _lowercase		)

									snake_case      : str							=   self.get_rust_tokenizer()
									snake_case      : List[str]							=   tokenizer.encode(_lowercase		)
									snake_case      : str							=   rust_tokenizer.encode(_lowercase		)
									self.assertListEqual(_lowercase      ,  _lowercase		)



			def 							__lowercase				( self       :							Optional[int]		)       ->				Optional[int]:
									snake_case      : Any							=   BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"		)      ,  ["ah", "\u535A", "\u63A8", "zz"]		)



			def 							__lowercase				( self       :							Dict		)       ->				int:
									snake_case      : Tuple							=   BasicTokenizer(do_lower_case=_lowercase		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,  ["hello", "!", "how", "are", "you", "?"]		)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,  ["hello"]		)



			def 							__lowercase				( self       :							List[Any]		)       ->				int:
									snake_case      : Dict							=   BasicTokenizer(do_lower_case=_lowercase      ,  strip_accents=_lowercase		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,  ["hällo", "!", "how", "are", "you", "?"]		)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,  ["h\u00E9llo"]		)



			def 							__lowercase				( self       :							List[Any]		)       ->				Tuple:
									snake_case      : str							=   BasicTokenizer(do_lower_case=_lowercase      ,  strip_accents=_lowercase		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,  ["hallo", "!", "how", "are", "you", "?"]		)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,  ["hello"]		)



			def 							__lowercase				( self       :							Union[str, Any]		)       ->				str:
									snake_case      : List[str]							=   BasicTokenizer(do_lower_case=_lowercase		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,  ["hallo", "!", "how", "are", "you", "?"]		)
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)      ,  ["hello"]		)



			def 							__lowercase				( self       :							Union[str, Any]		)       ->				Tuple:
									snake_case      : List[str]							=   BasicTokenizer(do_lower_case=_lowercase		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "		)      ,  ["HeLLo", "!", "how", "Are", "yoU", "?"]		)



			def 							__lowercase				( self       :							int		)       ->				Tuple:
									snake_case      : Union[str, Any]							=   BasicTokenizer(do_lower_case=_lowercase      ,  strip_accents=_lowercase		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,  ["HäLLo", "!", "how", "Are", "yoU", "?"]		)



			def 							__lowercase				( self       :							int		)       ->				Union[str, Any]:
									snake_case      : List[str]							=   BasicTokenizer(do_lower_case=_lowercase      ,  strip_accents=_lowercase		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "		)      ,  ["HaLLo", "!", "how", "Are", "yoU", "?"]		)



			def 							__lowercase				( self       :							Optional[Any]		)       ->				Optional[int]:
									snake_case      : List[Any]							=   BasicTokenizer(do_lower_case=_lowercase      ,  never_split=["[UNK]"]		)

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"		)      ,  ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]		)



			def 							__lowercase				( self       :							str		)       ->				List[str]:
									snake_case      : Dict							=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

									snake_case      : Any							=   {}
									for i, token in enumerate(_lowercase		):
															snake_case      : int							=   i
									snake_case      : Any							=   WordpieceTokenizer(vocab=_lowercase      ,  unk_token="[UNK]"		)

									self.assertListEqual(tokenizer.tokenize(""		)      ,  []		)

									self.assertListEqual(tokenizer.tokenize("unwanted running"		)      ,  ["un", "##want", "##ed", "runn", "##ing"]		)

									self.assertListEqual(tokenizer.tokenize("unwantedX running"		)      ,  ["[UNK]", "runn", "##ing"]		)



			def 							__lowercase				( self       :							int		)       ->				Any:
									self.assertTrue(_is_whitespace(" "		)		)
									self.assertTrue(_is_whitespace("\t"		)		)
									self.assertTrue(_is_whitespace("\r"		)		)
									self.assertTrue(_is_whitespace("\n"		)		)
									self.assertTrue(_is_whitespace("\u00A0"		)		)

									self.assertFalse(_is_whitespace("A"		)		)
									self.assertFalse(_is_whitespace("-"		)		)



			def 							__lowercase				( self       :							Optional[int]		)       ->				int:
									self.assertTrue(_is_control("\u0005"		)		)

									self.assertFalse(_is_control("A"		)		)
									self.assertFalse(_is_control(" "		)		)
									self.assertFalse(_is_control("\t"		)		)
									self.assertFalse(_is_control("\r"		)		)



			def 							__lowercase				( self       :							Optional[int]		)       ->				Any:
									self.assertTrue(_is_punctuation("-"		)		)
									self.assertTrue(_is_punctuation("$"		)		)
									self.assertTrue(_is_punctuation("`"		)		)
									self.assertTrue(_is_punctuation("."		)		)

									self.assertFalse(_is_punctuation("A"		)		)
									self.assertFalse(_is_punctuation(" "		)		)



			def 							__lowercase				( self       :							Any		)       ->				Any:
									snake_case      : str							=   self.get_tokenizer()
									snake_case      : Optional[int]							=   self.get_rust_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(_lowercase		) for t in ["Test", "\xad", "test"]]      ,  [["[UNK]"], [], ["[UNK]"]]		)

									self.assertListEqual(
									    [rust_tokenizer.tokenize(_lowercase		) for t in ["Test", "\xad", "test"]]      ,  [["[UNK]"], [], ["[UNK]"]]		)



			@slow
			def 							__lowercase				( self       :							Optional[Any]		)       ->				Optional[Any]:
									snake_case      : Optional[Any]							=   self.tokenizer_class.from_pretrained("google/electra-base-discriminator"		)

									snake_case      : int							=   tokenizer.encode("sequence builders"      ,  add_special_tokens=_lowercase		)
									snake_case      : Optional[Any]							=   tokenizer.encode("multi-sequence build"      ,  add_special_tokens=_lowercase		)

									snake_case      : str							=   tokenizer.build_inputs_with_special_tokens(_lowercase		)
									snake_case      : Any							=   tokenizer.build_inputs_with_special_tokens(_lowercase      ,  _lowercase		)

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]



			def 							__lowercase				( self       :							Any		)       ->				List[str]:
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case      : Dict							=   self.rust_tokenizer_class.from_pretrained(_lowercase      ,  **_lowercase		)

																					snake_case      : List[str]							=   F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																					snake_case      : Optional[int]							=   tokenizer_r.encode_plus(
																					    _lowercase      ,  return_attention_mask=_lowercase      ,  return_token_type_ids=_lowercase      ,  return_offsets_mapping=_lowercase      ,  add_special_tokens=_lowercase      ,  )

																					snake_case      : int							=   tokenizer_r.do_lower_case if hasattr(_lowercase      ,  "do_lower_case"		) else False
																					snake_case      : Any							=   (
																					    [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "A"),
																					        ((1, 2), ","),
																					        ((3, 5), "na"),
																					        ((5, 6), "##ï"),
																					        ((6, 8), "##ve"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "Allen"),
																					        ((21, 23), "##NL"),
																					        ((23, 24), "##P"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					    if not do_lower_case
																					    else [
																					        ((0, 0), tokenizer_r.cls_token),
																					        ((0, 1), "a"),
																					        ((1, 2), ","),
																					        ((3, 8), "naive"),
																					        ((9, 15), tokenizer_r.mask_token),
																					        ((16, 21), "allen"),
																					        ((21, 23), "##nl"),
																					        ((23, 24), "##p"),
																					        ((25, 33), "sentence"),
																					        ((33, 34), "."),
																					        ((0, 0), tokenizer_r.sep_token),
																					    ]
																					)

																					self.assertEqual(
																					    [e[1] for e in expected_results]      ,  tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]		)		)
																					self.assertEqual([e[0] for e in expected_results]      ,  tokens["offset_mapping"]		)



			def 							__lowercase				( self       :							Tuple		)       ->				Optional[Any]:
									snake_case      : Dict							=   ["的", "人", "有"]
									snake_case      : Tuple							=   "".join(_lowercase		)
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
																					snake_case      : Optional[int]							=   True
																					snake_case      : Any							=   self.tokenizer_class.from_pretrained(_lowercase      ,  **_lowercase		)
																					snake_case      : Dict							=   self.rust_tokenizer_class.from_pretrained(_lowercase      ,  **_lowercase		)

																					snake_case      : Tuple							=   tokenizer_p.encode(_lowercase      ,  add_special_tokens=_lowercase		)
																					snake_case      : Tuple							=   tokenizer_r.encode(_lowercase      ,  add_special_tokens=_lowercase		)

																					snake_case      : List[Any]							=   tokenizer_r.convert_ids_to_tokens(_lowercase		)
																					snake_case      : List[Any]							=   tokenizer_p.convert_ids_to_tokens(_lowercase		)

																					# it is expected that each Chinese character is not preceded by "##"
																					self.assertListEqual(_lowercase      ,  _lowercase		)
																					self.assertListEqual(_lowercase      ,  _lowercase		)

																					snake_case      : str							=   False
																					snake_case      : Optional[int]							=   self.rust_tokenizer_class.from_pretrained(_lowercase      ,  **_lowercase		)
																					snake_case      : str							=   self.tokenizer_class.from_pretrained(_lowercase      ,  **_lowercase		)

																					snake_case      : List[str]							=   tokenizer_r.encode(_lowercase      ,  add_special_tokens=_lowercase		)
																					snake_case      : int							=   tokenizer_p.encode(_lowercase      ,  add_special_tokens=_lowercase		)

																					snake_case      : Any							=   tokenizer_r.convert_ids_to_tokens(_lowercase		)
																					snake_case      : List[str]							=   tokenizer_p.convert_ids_to_tokens(_lowercase		)

																					# it is expected that only the first Chinese character is not preceded by "##".
																					snake_case      : List[Any]							=   [
																					    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(_lowercase		)
																					]
																					self.assertListEqual(_lowercase      ,  _lowercase		)
																					self.assertListEqual(_lowercase      ,  _lowercase		)








"""simple docstring"""







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
			import torch

if is_vision_available():
			from PIL import Image

			from transformers import OwlViTImageProcessor

class 			_a     (   unittest.TestCase):



			def __init__( self       :							Optional[int]      ,  _lowercase       :							int      ,  _lowercase       :							Union[str, Any]=7      ,  _lowercase       :							Union[str, Any]=3      ,  _lowercase       :							int=18      ,  _lowercase       :							Any=30      ,  _lowercase       :							str=400      ,  _lowercase       :							str=True      ,  _lowercase       :							Dict=None      ,  _lowercase       :							int=True      ,  _lowercase       :							Union[str, Any]=None      ,  _lowercase       :							str=True      ,  _lowercase       :							Union[str, Any]=[0.48145466, 0.4578275, 0.40821073]      ,  _lowercase       :							str=[0.26862954, 0.26130258, 0.27577711]      ,  _lowercase       :							Optional[int]=True      ,  )       ->				Optional[int]:
									snake_case      : str							=   parent
									snake_case      : Any							=   batch_size
									snake_case      : Optional[int]							=   num_channels
									snake_case      : Any							=   image_size
									snake_case      : Optional[Any]							=   min_resolution
									snake_case      : int							=   max_resolution
									snake_case      : List[Any]							=   do_resize
									snake_case      : Dict							=   size if size is not None else {"height": 18, "width": 18}
									snake_case      : str							=   do_center_crop
									snake_case      : List[str]							=   crop_size if crop_size is not None else {"height": 18, "width": 18}
									snake_case      : str							=   do_normalize
									snake_case      : str							=   image_mean
									snake_case      : Optional[Any]							=   image_std
									snake_case      : str							=   do_convert_rgb



			def 							__lowercase				( self       :							Tuple		)       ->				Union[str, Any]:
									return {
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "do_center_crop": self.do_center_crop,
									    "crop_size": self.crop_size,
									    "do_normalize": self.do_normalize,
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									    "do_convert_rgb": self.do_convert_rgb,
									}



@require_torch
@require_vision
class 			_a     (   SCREAMING_SNAKE_CASE__    ,  unittest.TestCase):
			__magic_name__    				=    OwlViTImageProcessor if is_vision_available() else None



			def 							__lowercase				( self       :							Any		)       ->				List[Any]:
									snake_case      : Dict							=   OwlViTImageProcessingTester(self		)



			@property
			def 							__lowercase				( self       :							Optional[int]		)       ->				List[Any]:
									return self.image_processor_tester.prepare_image_processor_dict()



			def 							__lowercase				( self       :							List[Any]		)       ->				Union[str, Any]:
									snake_case      : List[Any]							=   self.image_processing_class(**self.image_processor_dict		)
									self.assertTrue(hasattr(_lowercase      ,  "do_resize"		)		)
									self.assertTrue(hasattr(_lowercase      ,  "size"		)		)
									self.assertTrue(hasattr(_lowercase      ,  "do_center_crop"		)		)
									self.assertTrue(hasattr(_lowercase      ,  "center_crop"		)		)
									self.assertTrue(hasattr(_lowercase      ,  "do_normalize"		)		)
									self.assertTrue(hasattr(_lowercase      ,  "image_mean"		)		)
									self.assertTrue(hasattr(_lowercase      ,  "image_std"		)		)
									self.assertTrue(hasattr(_lowercase      ,  "do_convert_rgb"		)		)



			def 							__lowercase				( self       :							Any		)       ->				Any:
									snake_case      : List[Any]							=   self.image_processing_class.from_dict(self.image_processor_dict		)
									self.assertEqual(image_processor.size      ,  {"height": 18, "width": 18}		)
									self.assertEqual(image_processor.crop_size      ,  {"height": 18, "width": 18}		)

									snake_case      : int							=   self.image_processing_class.from_dict(self.image_processor_dict      ,  size=42      ,  crop_size=84		)
									self.assertEqual(image_processor.size      ,  {"height": 42, "width": 42}		)
									self.assertEqual(image_processor.crop_size      ,  {"height": 84, "width": 84}		)



			def 							__lowercase				( self       :							Any		)       ->				Optional[int]:
									# Initialize image_processing
									snake_case      : List[Any]							=   self.image_processing_class(**self.image_processor_dict		)
									# create random PIL images
									snake_case      : Tuple							=   prepare_image_inputs(self.image_processor_tester      ,  equal_resolution=_lowercase		)

									for image in image_inputs:
															self.assertIsInstance(_lowercase      ,  Image.Image		)

									# Test not batched input
									snake_case      : Dict							=   image_processing(image_inputs[0]      ,  return_tensors="pt"		).pixel_values
									self.assertEqual(
									    encoded_images.shape      ,  (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size["height"],
									        self.image_processor_tester.crop_size["width"],
									    )      ,  )

									# Test batched
									snake_case      : Optional[Any]							=   image_processing(_lowercase      ,  return_tensors="pt"		).pixel_values
									self.assertEqual(
									    encoded_images.shape      ,  (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size["height"],
									        self.image_processor_tester.crop_size["width"],
									    )      ,  )



			def 							__lowercase				( self       :							Optional[Any]		)       ->				Dict:
									# Initialize image_processing
									snake_case      : Optional[int]							=   self.image_processing_class(**self.image_processor_dict		)
									# create random numpy tensors
									snake_case      : int							=   prepare_image_inputs(self.image_processor_tester      ,  equal_resolution=_lowercase      ,  numpify=_lowercase		)
									for image in image_inputs:
															self.assertIsInstance(_lowercase      ,  np.ndarray		)

									# Test not batched input
									snake_case      : str							=   image_processing(image_inputs[0]      ,  return_tensors="pt"		).pixel_values
									self.assertEqual(
									    encoded_images.shape      ,  (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size["height"],
									        self.image_processor_tester.crop_size["width"],
									    )      ,  )

									# Test batched
									snake_case      : str							=   image_processing(_lowercase      ,  return_tensors="pt"		).pixel_values
									self.assertEqual(
									    encoded_images.shape      ,  (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size["height"],
									        self.image_processor_tester.crop_size["width"],
									    )      ,  )



			def 							__lowercase				( self       :							List[Any]		)       ->				List[str]:
									# Initialize image_processing
									snake_case      : Any							=   self.image_processing_class(**self.image_processor_dict		)
									# create random PyTorch tensors
									snake_case      : Dict							=   prepare_image_inputs(self.image_processor_tester      ,  equal_resolution=_lowercase      ,  torchify=_lowercase		)
									for image in image_inputs:
															self.assertIsInstance(_lowercase      ,  torch.Tensor		)

									# Test not batched input
									snake_case      : Union[str, Any]							=   image_processing(image_inputs[0]      ,  return_tensors="pt"		).pixel_values
									self.assertEqual(
									    encoded_images.shape      ,  (
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size["height"],
									        self.image_processor_tester.crop_size["width"],
									    )      ,  )

									# Test batched
									snake_case      : Any							=   image_processing(_lowercase      ,  return_tensors="pt"		).pixel_values
									self.assertEqual(
									    encoded_images.shape      ,  (
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.crop_size["height"],
									        self.image_processor_tester.crop_size["width"],
									    )      ,  )



