





'''simple docstring'''




import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 							__snake_case    (				__magic_name__					,					unittest.TestCase       ):
	__lowerCAmelCase						=     RealmTokenizer
	__lowerCAmelCase						=     RealmTokenizerFast
	__lowerCAmelCase						=     True
	__lowerCAmelCase						=     True
	__lowerCAmelCase						=     filter_non_english





	def      _snake_case      (		self		)			->		Any:
		super().setUp()

		snake_case__  					=      [
		    '[UNK]',
		    '[CLS]',
		    '[SEP]',
		    '[PAD]',
		    '[MASK]',
		    'want',
		    '##want',
		    '##ed',
		    'wa',
		    'un',
		    'runn',
		    '##ing',
		    ',',
		    'low',
		    'lowest',
		]
		snake_case__  					=      os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']		)
		with open(self.vocab_file			,					'w'			,					encoding='utf-8'		) as vocab_writer:
			vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)





	def      _snake_case      (		self			,					UpperCamelCase_		)			->		Tuple:
		snake_case__  					=      'UNwant\u00E9d,running'
		snake_case__  					=      'unwanted, running'
		return input_text, output_text





	def      _snake_case      (		self		)			->		Any:
		snake_case__  					=      self.tokenizer_class(self.vocab_file		)

		snake_case__  					=      tokenizer.tokenize('UNwant\u00E9d,running'		)
		self.assertListEqual(UpperCamelCase_			,					['un', '##want', '##ed', ',', 'runn', '##ing']		)
		self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_		)			,					[9, 6, 7, 12, 10, 11]		)





	def      _snake_case      (		self		)			->		Optional[int]:







		if not self.test_rust_tokenizer:
			return

		snake_case__  					=      self.get_tokenizer()
		snake_case__  					=      self.get_rust_tokenizer()

		snake_case__  					=      'UNwant\u00E9d,running'

		snake_case__  					=      tokenizer.tokenize(UpperCamelCase_		)
		snake_case__  					=      rust_tokenizer.tokenize(UpperCamelCase_		)
		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)

		snake_case__  					=      tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)

		snake_case__  					=      self.get_rust_tokenizer()
		snake_case__  					=      tokenizer.encode(UpperCamelCase_		)
		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_		)
		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)

		# With lower casing
		snake_case__  					=      self.get_tokenizer(do_lower_case=UpperCamelCase_		)
		snake_case__  					=      self.get_rust_tokenizer(do_lower_case=UpperCamelCase_		)

		snake_case__  					=      'UNwant\u00E9d,running'

		snake_case__  					=      tokenizer.tokenize(UpperCamelCase_		)
		snake_case__  					=      rust_tokenizer.tokenize(UpperCamelCase_		)
		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)

		snake_case__  					=      tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)

		snake_case__  					=      self.get_rust_tokenizer()
		snake_case__  					=      tokenizer.encode(UpperCamelCase_		)
		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_		)
		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)





	def      _snake_case      (		self		)			->		int:
		snake_case__  					=      BasicTokenizer()

		self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'		)			,					['ah', '\u535A', '\u63A8', 'zz']		)





	def      _snake_case      (		self		)			->		Optional[Any]:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)			,					['hello', '!', 'how', 'are', 'you', '?']		)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['hello']		)





	def      _snake_case      (		self		)			->		List[str]:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['hällo', '!', 'how', 'are', 'you', '?']		)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['h\u00E9llo']		)





	def      _snake_case      (		self		)			->		Dict:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['hallo', '!', 'how', 'are', 'you', '?']		)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['hello']		)





	def      _snake_case      (		self		)			->		List[Any]:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['hallo', '!', 'how', 'are', 'you', '?']		)
		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['hello']		)





	def      _snake_case      (		self		)			->		Optional[int]:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']		)





	def      _snake_case      (		self		)			->		str:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']		)





	def      _snake_case      (		self		)			->		Tuple:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']		)





	def      _snake_case      (		self		)			->		Dict:
		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					never_split=['[UNK]']		)

		self.assertListEqual(
		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'		)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']		)





	def      _snake_case      (		self		)			->		List[str]:
		snake_case__  					=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

		snake_case__  					=      {}
		for i, token in enumerate(UpperCamelCase_		):
			snake_case__  					=      i
		snake_case__  					=      WordpieceTokenizer(vocab=UpperCamelCase_			,					unk_token='[UNK]'		)

		self.assertListEqual(tokenizer.tokenize(''		)			,					[]		)

		self.assertListEqual(tokenizer.tokenize('unwanted running'		)			,					['un', '##want', '##ed', 'runn', '##ing']		)

		self.assertListEqual(tokenizer.tokenize('unwantedX running'		)			,					['[UNK]', 'runn', '##ing']		)





	def      _snake_case      (		self		)			->		Optional[Any]:
		self.assertTrue(_is_whitespace(' '		)		)
		self.assertTrue(_is_whitespace('\t'		)		)
		self.assertTrue(_is_whitespace('\r'		)		)
		self.assertTrue(_is_whitespace('\n'		)		)
		self.assertTrue(_is_whitespace('\u00A0'		)		)

		self.assertFalse(_is_whitespace('A'		)		)
		self.assertFalse(_is_whitespace('-'		)		)





	def      _snake_case      (		self		)			->		Optional[Any]:
		self.assertTrue(_is_control('\u0005'		)		)

		self.assertFalse(_is_control('A'		)		)
		self.assertFalse(_is_control(' '		)		)
		self.assertFalse(_is_control('\t'		)		)
		self.assertFalse(_is_control('\r'		)		)





	def      _snake_case      (		self		)			->		Dict:
		self.assertTrue(_is_punctuation('-'		)		)
		self.assertTrue(_is_punctuation('$'		)		)
		self.assertTrue(_is_punctuation('`'		)		)
		self.assertTrue(_is_punctuation('.'		)		)

		self.assertFalse(_is_punctuation('A'		)		)
		self.assertFalse(_is_punctuation(' '		)		)





	def      _snake_case      (		self		)			->		Tuple:
		snake_case__  					=      self.get_tokenizer()

		# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
		self.assertListEqual([tokenizer.tokenize(UpperCamelCase_		) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]		)

		if self.test_rust_tokenizer:
			snake_case__  					=      self.get_rust_tokenizer()
			self.assertListEqual(
			    [rust_tokenizer.tokenize(UpperCamelCase_		) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]		)





	@slow
	def      _snake_case      (		self		)			->		Dict:
		snake_case__  					=      self.tokenizer_class.from_pretrained('bert-base-uncased'		)

		snake_case__  					=      tokenizer.encode('sequence builders'			,					add_special_tokens=UpperCamelCase_		)
		snake_case__  					=      tokenizer.encode('multi-sequence build'			,					add_special_tokens=UpperCamelCase_		)

		snake_case__  					=      tokenizer.build_inputs_with_special_tokens(UpperCamelCase_		)
		snake_case__  					=      tokenizer.build_inputs_with_special_tokens(UpperCamelCase_			,					UpperCamelCase_		)

		assert encoded_sentence == [101] + text + [102]
		assert encoded_pair == [101] + text + [102] + text_a + [102]





	def      _snake_case      (		self		)			->		Optional[Any]:







		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
				snake_case__  					=      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_			,					**UpperCamelCase_		)

				snake_case__  					=      F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
				snake_case__  					=      tokenizer_r.encode_plus(
				    UpperCamelCase_			,					return_attention_mask=UpperCamelCase_			,					return_token_type_ids=UpperCamelCase_			,					return_offsets_mapping=UpperCamelCase_			,					add_special_tokens=UpperCamelCase_			,					)

				snake_case__  					=      tokenizer_r.do_lower_case if hasattr(UpperCamelCase_			,					'do_lower_case'		) else False
				snake_case__  					=      (
				    [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'A'),
				        ((1, 2), ','),
				        ((3, 5), 'na'),
				        ((5, 6), '##ï'),
				        ((6, 8), '##ve'),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), 'Allen'),
				        ((21, 23), '##NL'),
				        ((23, 24), '##P'),
				        ((25, 33), 'sentence'),
				        ((33, 34), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				    if not do_lower_case
				    else [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), 'a'),
				        ((1, 2), ','),
				        ((3, 8), 'naive'),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), 'allen'),
				        ((21, 23), '##nl'),
				        ((23, 24), '##p'),
				        ((25, 33), 'sentence'),
				        ((33, 34), '.'),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				)

				self.assertEqual(
				    [e[1] for e in expected_results]			,					tokenizer_r.convert_ids_to_tokens(tokens['input_ids']		)		)
				self.assertEqual([e[0] for e in expected_results]			,					tokens['offset_mapping']		)





	@slow
	def      _snake_case      (		self		)			->		Optional[int]:







		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
				snake_case__  					=      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_			,					**UpperCamelCase_		)
				snake_case__  					=      self.tokenizer_class.from_pretrained(UpperCamelCase_			,					**UpperCamelCase_		)
				snake_case__  					=      [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

				snake_case__  					=      tokenizer_r.batch_encode_candidates(UpperCamelCase_			,					max_length=10			,					return_tensors='np'		)
				snake_case__  					=      tokenizer_p.batch_encode_candidates(UpperCamelCase_			,					max_length=10			,					return_tensors='np'		)

				snake_case__  					=      (2, 2, 10)

				self.assertEqual(encoded_sentence_r['input_ids'].shape			,					UpperCamelCase_		)
				self.assertEqual(encoded_sentence_r['attention_mask'].shape			,					UpperCamelCase_		)
				self.assertEqual(encoded_sentence_r['token_type_ids'].shape			,					UpperCamelCase_		)

				self.assertEqual(encoded_sentence_p['input_ids'].shape			,					UpperCamelCase_		)
				self.assertEqual(encoded_sentence_p['attention_mask'].shape			,					UpperCamelCase_		)
				self.assertEqual(encoded_sentence_p['token_type_ids'].shape			,					UpperCamelCase_		)











'''simple docstring'''




from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


a__				: Dict									=    logging.get_logger(__name__)

a__				: Tuple									=    {
    '''apple/mobilevitv2-1.0''': '''https://huggingface.co/apple/mobilevitv2-1.0/resolve/main/config.json''',
}






class 							__snake_case    (				__magic_name__       ):
	__lowerCAmelCase						=     '''mobilevitv2'''





	def __init__(		self			,					UpperCamelCase_=3			,					UpperCamelCase_=256			,					UpperCamelCase_=2			,					UpperCamelCase_=2.0			,					UpperCamelCase_="swish"			,					UpperCamelCase_=3			,					UpperCamelCase_=32			,					UpperCamelCase_=0.1			,					UpperCamelCase_=0.0_2			,					UpperCamelCase_=1E-5			,					UpperCamelCase_=512			,					UpperCamelCase_=[6, 12, 18]			,					UpperCamelCase_=0.1			,					UpperCamelCase_=255			,					UpperCamelCase_=[2, 4, 3]			,					UpperCamelCase_=[128, 192, 256]			,					UpperCamelCase_=1.0			,					UpperCamelCase_=2			,					UpperCamelCase_=0.0			,					UpperCamelCase_=0.0			,					**UpperCamelCase_			,					)			->		Union[str, Any]:
		super().__init__(**UpperCamelCase_		)

		snake_case__  					=      num_channels
		snake_case__  					=      image_size
		snake_case__  					=      patch_size
		snake_case__  					=      expand_ratio
		snake_case__  					=      hidden_act
		snake_case__  					=      conv_kernel_size
		snake_case__  					=      output_stride
		snake_case__  					=      initializer_range
		snake_case__  					=      layer_norm_eps
		snake_case__  					=      n_attn_blocks
		snake_case__  					=      base_attn_unit_dims
		snake_case__  					=      width_multiplier
		snake_case__  					=      ffn_multiplier
		snake_case__  					=      ffn_dropout
		snake_case__  					=      attn_dropout
		snake_case__  					=      classifier_dropout_prob

		# decode head attributes for semantic segmentation
		snake_case__  					=      aspp_out_channels
		snake_case__  					=      atrous_rates
		snake_case__  					=      aspp_dropout_prob
		snake_case__  					=      semantic_loss_ignore_index






class 							__snake_case    (				__magic_name__       ):
	__lowerCAmelCase						=     version.parse('''1.11'''       )





	@property
	def      _snake_case      (		self		)			->		Mapping[str, Mapping[int, str]]:
		return OrderedDict([('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'})]		)





	@property
	def      _snake_case      (		self		)			->		Mapping[str, Mapping[int, str]]:
		if self.task == "image-classification":
			return OrderedDict([('logits', {0: 'batch'})]		)
		else:
			return OrderedDict([('last_hidden_state', {0: 'batch'}), ('pooler_output', {0: 'batch'})]		)





	@property
	def      _snake_case      (		self		)			->		float:
		return 1E-4



