


'''simple docstring'''


from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import symbol_database as _symbol_database
from google.protobuf.internal import builder as _builder


# @@protoc_insertion_point(imports)

UpperCAmelCase_		:   List[Any]							   =		_symbol_database.Default()


UpperCAmelCase_		:   Optional[Any]							   =		_descriptor_pool.Default().AddSerializedFile(
    B'\n\x19sentencepiece_model.proto\x12\rsentencepiece"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18  \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05<unk>\x12\x16\n\tbos_piece\x18. \x01(\t:\x03<s>\x12\x17\n\teos_piece\x18/ \x01(\t:\x04</s>\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05<pad>\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03'
)

UpperCAmelCase_		:   str							   =		globals()
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sentencepiece_model_pb2', _globals)
if _descriptor._USE_C_DESCRIPTORS is False:
				UpperCAmelCase_		:   Union[str, Any]							   =		None
				UpperCAmelCase_		:   int							   =		B'H\003'
				# (generated by protobuf compiler, but `_TRAINERSPEC` is not defined)
				# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._options = None
				# _TRAINERSPEC.fields_by_name["mining_sentence_size"]._serialized_options = b"\030\001"
				# _TRAINERSPEC.fields_by_name["training_sentence_size"]._options = None
				# _TRAINERSPEC.fields_by_name["training_sentence_size"]._serialized_options = b"\030\001"
				UpperCAmelCase_		:   Optional[int]							   =		45
				UpperCAmelCase_		:   int							   =		1581
				UpperCAmelCase_		:   List[Any]							   =		1517
				UpperCAmelCase_		:   Optional[int]							   =		1570
				UpperCAmelCase_		:   Any							   =		1584
				UpperCAmelCase_		:   Union[str, Any]							   =		1793
				UpperCAmelCase_		:   List[str]							   =		1795
				UpperCAmelCase_		:   str							   =		1916
				UpperCAmelCase_		:   Any							   =		1864
				UpperCAmelCase_		:   str							   =		1905
				UpperCAmelCase_		:   Dict							   =		1919
				UpperCAmelCase_		:   str							   =		2429
				UpperCAmelCase_		:   Union[str, Any]							   =		2208
				UpperCAmelCase_		:   Union[str, Any]							   =		2418
				UpperCAmelCase_		:   Union[str, Any]							   =		2323
				UpperCAmelCase_		:   Union[str, Any]							   =		2407
# @@protoc_insertion_point(module_scope)








'''simple docstring'''


import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image

				from transformers import MobileNetVaImageProcessor





class     lowercase__  (						unittest.TestCase					):





							'''simple docstring'''


							def __init__(    self    ,					__snake_case    ,					__snake_case=7    ,					__snake_case=3    ,					__snake_case=18    ,					__snake_case=30    ,					__snake_case=400    ,					__snake_case=True    ,					__snake_case=None    ,					__snake_case=True    ,					__snake_case=None    ,					):
											_SCREAMING_SNAKE_CASE :       Any     		=      size if size is not None else {"""shortest_edge""": 20}
											_SCREAMING_SNAKE_CASE :       int     		=      crop_size if crop_size is not None else {"""height""": 18, """width""": 18}
											_SCREAMING_SNAKE_CASE :       Dict     		=      parent
											_SCREAMING_SNAKE_CASE :       Any     		=      batch_size
											_SCREAMING_SNAKE_CASE :       Any     		=      num_channels
											_SCREAMING_SNAKE_CASE :       Dict     		=      image_size
											_SCREAMING_SNAKE_CASE :       Any     		=      min_resolution
											_SCREAMING_SNAKE_CASE :       Any     		=      max_resolution
											_SCREAMING_SNAKE_CASE :       Union[str, Any]     		=      do_resize
											_SCREAMING_SNAKE_CASE :       Tuple     		=      size
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      do_center_crop
											_SCREAMING_SNAKE_CASE :       str     		=      crop_size






							def UpperCAmelCase_	(    self  ):
											return {
											    "do_resize": self.do_resize,
											    "size": self.size,
											    "do_center_crop": self.do_center_crop,
											    "crop_size": self.crop_size,
											}





@require_torch
@require_vision
class     lowercase__  (						_snake_case     ,     unittest.TestCase					):





							'''simple docstring'''


							A_							:      Union[str, Any]      	=  MobileNetVaImageProcessor if is_vision_available() else None






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       int     		=      MobileNetVaImageProcessingTester(self  )






							@property
							def UpperCAmelCase_	(    self  ):
											return self.image_processor_tester.prepare_image_processor_dict()






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       str     		=      self.image_processing_class(**self.image_processor_dict  )
											self.assertTrue(hasattr(__snake_case    ,					"""do_resize"""  )  )
											self.assertTrue(hasattr(__snake_case    ,					"""size"""  )  )
											self.assertTrue(hasattr(__snake_case    ,					"""do_center_crop"""  )  )
											self.assertTrue(hasattr(__snake_case    ,					"""crop_size"""  )  )






							def UpperCAmelCase_	(    self  ):
											_SCREAMING_SNAKE_CASE :       str     		=      self.image_processing_class.from_dict(self.image_processor_dict  )
											self.assertEqual(image_processor.size    ,					{"""shortest_edge""": 20}  )
											self.assertEqual(image_processor.crop_size    ,					{"""height""": 18, """width""": 18}  )

											_SCREAMING_SNAKE_CASE :       Dict     		=      self.image_processing_class.from_dict(self.image_processor_dict    ,					size=42    ,					crop_size=84  )
											self.assertEqual(image_processor.size    ,					{"""shortest_edge""": 42}  )
											self.assertEqual(image_processor.crop_size    ,					{"""height""": 84, """width""": 84}  )






							def UpperCAmelCase_	(    self  ):
											pass






							def UpperCAmelCase_	(    self  ):
											# Initialize image_processing
											_SCREAMING_SNAKE_CASE :       str     		=      self.image_processing_class(**self.image_processor_dict  )
											# create random PIL images
											_SCREAMING_SNAKE_CASE :       List[str]     		=      prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=__snake_case  )
											for image in image_inputs:
															self.assertIsInstance(__snake_case    ,					Image.Image  )

											# Test not batched input
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      image_processing(image_inputs[0]    ,					return_tensors="""pt"""  ).pixel_values
											self.assertEqual(
											    encoded_images.shape    ,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )    ,					)

											# Test batched
											_SCREAMING_SNAKE_CASE :       int     		=      image_processing(__snake_case    ,					return_tensors="""pt"""  ).pixel_values
											self.assertEqual(
											    encoded_images.shape    ,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )    ,					)






							def UpperCAmelCase_	(    self  ):
											# Initialize image_processing
											_SCREAMING_SNAKE_CASE :       Optional[int]     		=      self.image_processing_class(**self.image_processor_dict  )
											# create random numpy tensors
											_SCREAMING_SNAKE_CASE :       Tuple     		=      prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=__snake_case    ,					numpify=__snake_case  )
											for image in image_inputs:
															self.assertIsInstance(__snake_case    ,					np.ndarray  )

											# Test not batched input
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      image_processing(image_inputs[0]    ,					return_tensors="""pt"""  ).pixel_values
											self.assertEqual(
											    encoded_images.shape    ,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )    ,					)

											# Test batched
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      image_processing(__snake_case    ,					return_tensors="""pt"""  ).pixel_values
											self.assertEqual(
											    encoded_images.shape    ,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )    ,					)






							def UpperCAmelCase_	(    self  ):
											# Initialize image_processing
											_SCREAMING_SNAKE_CASE :       Optional[Any]     		=      self.image_processing_class(**self.image_processor_dict  )
											# create random PyTorch tensors
											_SCREAMING_SNAKE_CASE :       str     		=      prepare_image_inputs(self.image_processor_tester    ,					equal_resolution=__snake_case    ,					torchify=__snake_case  )
											for image in image_inputs:
															self.assertIsInstance(__snake_case    ,					torch.Tensor  )

											# Test not batched input
											_SCREAMING_SNAKE_CASE :       List[Any]     		=      image_processing(image_inputs[0]    ,					return_tensors="""pt"""  ).pixel_values
											self.assertEqual(
											    encoded_images.shape    ,					(
											        1,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )    ,					)

											# Test batched
											_SCREAMING_SNAKE_CASE :       str     		=      image_processing(__snake_case    ,					return_tensors="""pt"""  ).pixel_values
											self.assertEqual(
											    encoded_images.shape    ,					(
											        self.image_processor_tester.batch_size,
											        self.image_processor_tester.num_channels,
											        self.image_processor_tester.crop_size["""height"""],
											        self.image_processor_tester.crop_size["""width"""],
											    )    ,					)




