code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  
  @require_tokenizers
  class 							__snake_case    (				__magic_name__					,					unittest.TestCase       ):
  	__lowerCAmelCase						=     RealmTokenizer
  	__lowerCAmelCase						=     RealmTokenizerFast
  	__lowerCAmelCase						=     True
  	__lowerCAmelCase						=     True
  	__lowerCAmelCase						=     filter_non_english
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Any:
  		super().setUp()
  
  		snake_case__  					=      [
  		    '[UNK]',
  		    '[CLS]',
  		    '[SEP]',
  		    '[PAD]',
  		    '[MASK]',
  		    'want',
  		    '##want',
  		    '##ed',
  		    'wa',
  		    'un',
  		    'runn',
  		    '##ing',
  		    ',',
  		    'low',
  		    'lowest',
  		]
  		snake_case__  					=      os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['vocab_file']		)
  		with open(self.vocab_file			,					'w'			,					encoding='utf-8'		) as vocab_writer:
  			vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]		)		)
  
  
  
  
  
  	def      _snake_case      (		self			,					UpperCamelCase_		)			->		Tuple:
  		snake_case__  					=      'UNwant\u00E9d,running'
  		snake_case__  					=      'unwanted, running'
  		return input_text, output_text
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Any:
  		snake_case__  					=      self.tokenizer_class(self.vocab_file		)
  
  		snake_case__  					=      tokenizer.tokenize('UNwant\u00E9d,running'		)
  		self.assertListEqual(UpperCamelCase_			,					['un', '##want', '##ed', ',', 'runn', '##ing']		)
  		self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_		)			,					[9, 6, 7, 12, 10, 11]		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[int]:
  
  
  
  
  
  
  
  		if not self.test_rust_tokenizer:
  			return
  
  		snake_case__  					=      self.get_tokenizer()
  		snake_case__  					=      self.get_rust_tokenizer()
  
  		snake_case__  					=      'UNwant\u00E9d,running'
  
  		snake_case__  					=      tokenizer.tokenize(UpperCamelCase_		)
  		snake_case__  					=      rust_tokenizer.tokenize(UpperCamelCase_		)
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  		snake_case__  					=      tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
  		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  		snake_case__  					=      self.get_rust_tokenizer()
  		snake_case__  					=      tokenizer.encode(UpperCamelCase_		)
  		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_		)
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  		# With lower casing
  		snake_case__  					=      self.get_tokenizer(do_lower_case=UpperCamelCase_		)
  		snake_case__  					=      self.get_rust_tokenizer(do_lower_case=UpperCamelCase_		)
  
  		snake_case__  					=      'UNwant\u00E9d,running'
  
  		snake_case__  					=      tokenizer.tokenize(UpperCamelCase_		)
  		snake_case__  					=      rust_tokenizer.tokenize(UpperCamelCase_		)
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  		snake_case__  					=      tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
  		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_			,					add_special_tokens=UpperCamelCase_		)
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  		snake_case__  					=      self.get_rust_tokenizer()
  		snake_case__  					=      tokenizer.encode(UpperCamelCase_		)
  		snake_case__  					=      rust_tokenizer.encode(UpperCamelCase_		)
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		int:
  		snake_case__  					=      BasicTokenizer()
  
  		self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'		)			,					['ah', '\u535A', '\u63A8', 'zz']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)			,					['hello', '!', 'how', 'are', 'you', '?']		)
  		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['hello']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['hällo', '!', 'how', 'are', 'you', '?']		)
  		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['h\u00E9llo']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['hallo', '!', 'how', 'are', 'you', '?']		)
  		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['hello']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[Any]:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['hallo', '!', 'how', 'are', 'you', '?']		)
  		self.assertListEqual(tokenizer.tokenize('H\u00E9llo'		)			,					['hello']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[int]:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '		)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		str:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['HäLLo', '!', 'how', 'Are', 'yoU', '?']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Tuple:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					strip_accents=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '		)			,					['HaLLo', '!', 'how', 'Are', 'yoU', '?']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      BasicTokenizer(do_lower_case=UpperCamelCase_			,					never_split=['[UNK]']		)
  
  		self.assertListEqual(
  		    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'		)			,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']
  
  		snake_case__  					=      {}
  		for i, token in enumerate(UpperCamelCase_		):
  			snake_case__  					=      i
  		snake_case__  					=      WordpieceTokenizer(vocab=UpperCamelCase_			,					unk_token='[UNK]'		)
  
  		self.assertListEqual(tokenizer.tokenize(''		)			,					[]		)
  
  		self.assertListEqual(tokenizer.tokenize('unwanted running'		)			,					['un', '##want', '##ed', 'runn', '##ing']		)
  
  		self.assertListEqual(tokenizer.tokenize('unwantedX running'		)			,					['[UNK]', 'runn', '##ing']		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  		self.assertTrue(_is_whitespace(' '		)		)
  		self.assertTrue(_is_whitespace('\t'		)		)
  		self.assertTrue(_is_whitespace('\r'		)		)
  		self.assertTrue(_is_whitespace('\n'		)		)
  		self.assertTrue(_is_whitespace('\u00A0'		)		)
  
  		self.assertFalse(_is_whitespace('A'		)		)
  		self.assertFalse(_is_whitespace('-'		)		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  		self.assertTrue(_is_control('\u0005'		)		)
  
  		self.assertFalse(_is_control('A'		)		)
  		self.assertFalse(_is_control(' '		)		)
  		self.assertFalse(_is_control('\t'		)		)
  		self.assertFalse(_is_control('\r'		)		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		self.assertTrue(_is_punctuation('-'		)		)
  		self.assertTrue(_is_punctuation('$'		)		)
  		self.assertTrue(_is_punctuation('`'		)		)
  		self.assertTrue(_is_punctuation('.'		)		)
  
  		self.assertFalse(_is_punctuation('A'		)		)
  		self.assertFalse(_is_punctuation(' '		)		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Tuple:
  		snake_case__  					=      self.get_tokenizer()
  
  		# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  		self.assertListEqual([tokenizer.tokenize(UpperCamelCase_		) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]		)
  
  		if self.test_rust_tokenizer:
  			snake_case__  					=      self.get_rust_tokenizer()
  			self.assertListEqual(
  			    [rust_tokenizer.tokenize(UpperCamelCase_		) for t in ['Test', '\xad', 'test']]			,					[['[UNK]'], [], ['[UNK]']]		)
  
  
  
  
  
  	@slow
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      self.tokenizer_class.from_pretrained('bert-base-uncased'		)
  
  		snake_case__  					=      tokenizer.encode('sequence builders'			,					add_special_tokens=UpperCamelCase_		)
  		snake_case__  					=      tokenizer.encode('multi-sequence build'			,					add_special_tokens=UpperCamelCase_		)
  
  		snake_case__  					=      tokenizer.build_inputs_with_special_tokens(UpperCamelCase_		)
  		snake_case__  					=      tokenizer.build_inputs_with_special_tokens(UpperCamelCase_			,					UpperCamelCase_		)
  
  		assert encoded_sentence == [101] + text + [102]
  		assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Optional[Any]:
  
  
  
  
  
  
  
  		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
  				snake_case__  					=      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_			,					**UpperCamelCase_		)
  
  				snake_case__  					=      F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
  				snake_case__  					=      tokenizer_r.encode_plus(
  				    UpperCamelCase_			,					return_attention_mask=UpperCamelCase_			,					return_token_type_ids=UpperCamelCase_			,					return_offsets_mapping=UpperCamelCase_			,					add_special_tokens=UpperCamelCase_			,					)
  
  				snake_case__  					=      tokenizer_r.do_lower_case if hasattr(UpperCamelCase_			,					'do_lower_case'		) else False
  				snake_case__  					=      (
  				    [
  				        ((0, 0), tokenizer_r.cls_token),
  				        ((0, 1), 'A'),
  				        ((1, 2), ','),
  				        ((3, 5), 'na'),
  				        ((5, 6), '##ï'),
  				        ((6, 8), '##ve'),
  				        ((9, 15), tokenizer_r.mask_token),
  				        ((16, 21), 'Allen'),
  				        ((21, 23), '##NL'),
  				        ((23, 24), '##P'),
  				        ((25, 33), 'sentence'),
  				        ((33, 34), '.'),
  				        ((0, 0), tokenizer_r.sep_token),
  				    ]
  				    if not do_lower_case
  				    else [
  				        ((0, 0), tokenizer_r.cls_token),
  				        ((0, 1), 'a'),
  				        ((1, 2), ','),
  				        ((3, 8), 'naive'),
  				        ((9, 15), tokenizer_r.mask_token),
  				        ((16, 21), 'allen'),
  				        ((21, 23), '##nl'),
  				        ((23, 24), '##p'),
  				        ((25, 33), 'sentence'),
  				        ((33, 34), '.'),
  				        ((0, 0), tokenizer_r.sep_token),
  				    ]
  				)
  
  				self.assertEqual(
  				    [e[1] for e in expected_results]			,					tokenizer_r.convert_ids_to_tokens(tokens['input_ids']		)		)
  				self.assertEqual([e[0] for e in expected_results]			,					tokens['offset_mapping']		)
  
  
  
  
  
  	@slow
  	def      _snake_case      (		self		)			->		Optional[int]:
  
  
  
  
  
  
  
  		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  			with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''		):
  				snake_case__  					=      self.rust_tokenizer_class.from_pretrained(UpperCamelCase_			,					**UpperCamelCase_		)
  				snake_case__  					=      self.tokenizer_class.from_pretrained(UpperCamelCase_			,					**UpperCamelCase_		)
  				snake_case__  					=      [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]
  
  				snake_case__  					=      tokenizer_r.batch_encode_candidates(UpperCamelCase_			,					max_length=10			,					return_tensors='np'		)
  				snake_case__  					=      tokenizer_p.batch_encode_candidates(UpperCamelCase_			,					max_length=10			,					return_tensors='np'		)
  
  				snake_case__  					=      (2, 2, 10)
  
  				self.assertEqual(encoded_sentence_r['input_ids'].shape			,					UpperCamelCase_		)
  				self.assertEqual(encoded_sentence_r['attention_mask'].shape			,					UpperCamelCase_		)
  				self.assertEqual(encoded_sentence_r['token_type_ids'].shape			,					UpperCamelCase_		)
  
  				self.assertEqual(encoded_sentence_p['input_ids'].shape			,					UpperCamelCase_		)
  				self.assertEqual(encoded_sentence_p['attention_mask'].shape			,					UpperCamelCase_		)
  				self.assertEqual(encoded_sentence_p['token_type_ids'].shape			,					UpperCamelCase_		)
  
  
  
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  from collections import OrderedDict
  from typing import Mapping
  
  from packaging import version
  
  from ...configuration_utils import PretrainedConfig
  from ...onnx import OnnxConfig
  from ...utils import logging
  
  
  a__				: Dict									=    logging.get_logger(__name__)
  
  a__				: Tuple									=    {
      '''apple/mobilevitv2-1.0''': '''https://huggingface.co/apple/mobilevitv2-1.0/resolve/main/config.json''',
  }
  
  
  
  
  
  
  class 							__snake_case    (				__magic_name__       ):
  	__lowerCAmelCase						=     '''mobilevitv2'''
  
  
  
  
  
  	def __init__(		self			,					UpperCamelCase_=3			,					UpperCamelCase_=256			,					UpperCamelCase_=2			,					UpperCamelCase_=2.0			,					UpperCamelCase_="swish"			,					UpperCamelCase_=3			,					UpperCamelCase_=32			,					UpperCamelCase_=0.1			,					UpperCamelCase_=0.0_2			,					UpperCamelCase_=1E-5			,					UpperCamelCase_=512			,					UpperCamelCase_=[6, 12, 18]			,					UpperCamelCase_=0.1			,					UpperCamelCase_=255			,					UpperCamelCase_=[2, 4, 3]			,					UpperCamelCase_=[128, 192, 256]			,					UpperCamelCase_=1.0			,					UpperCamelCase_=2			,					UpperCamelCase_=0.0			,					UpperCamelCase_=0.0			,					**UpperCamelCase_			,					)			->		Union[str, Any]:
  		super().__init__(**UpperCamelCase_		)
  
  		snake_case__  					=      num_channels
  		snake_case__  					=      image_size
  		snake_case__  					=      patch_size
  		snake_case__  					=      expand_ratio
  		snake_case__  					=      hidden_act
  		snake_case__  					=      conv_kernel_size
  		snake_case__  					=      output_stride
  		snake_case__  					=      initializer_range
  		snake_case__  					=      layer_norm_eps
  		snake_case__  					=      n_attn_blocks
  		snake_case__  					=      base_attn_unit_dims
  		snake_case__  					=      width_multiplier
  		snake_case__  					=      ffn_multiplier
  		snake_case__  					=      ffn_dropout
  		snake_case__  					=      attn_dropout
  		snake_case__  					=      classifier_dropout_prob
  
  		# decode head attributes for semantic segmentation
  		snake_case__  					=      aspp_out_channels
  		snake_case__  					=      atrous_rates
  		snake_case__  					=      aspp_dropout_prob
  		snake_case__  					=      semantic_loss_ignore_index
  
  
  
  
  
  
  class 							__snake_case    (				__magic_name__       ):
  	__lowerCAmelCase						=     version.parse('''1.11'''       )
  
  
  
  
  
  	@property
  	def      _snake_case      (		self		)			->		Mapping[str, Mapping[int, str]]:
  		return OrderedDict([('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'})]		)
  
  
  
  
  
  	@property
  	def      _snake_case      (		self		)			->		Mapping[str, Mapping[int, str]]:
  		if self.task == "image-classification":
  			return OrderedDict([('logits', {0: 'batch'})]		)
  		else:
  			return OrderedDict([('last_hidden_state', {0: 'batch'}), ('pooler_output', {0: 'batch'})]		)
  
  
  
  
  
  	@property
  	def      _snake_case      (		self		)			->		float:
  		return 1E-4
  
  
  
  