code: 
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  
  @require_tokenizers
  class    __A				(							lowerCAmelCase						,						unittest.TestCase ):
      lowerCAmelCase_							:		Tuple						=							RealmTokenizer
      lowerCAmelCase_							:		Optional[int]						=							RealmTokenizerFast
      lowerCAmelCase_							:		int						=							True
      lowerCAmelCase_							:		Tuple						=							True
      lowerCAmelCase_							:		Optional[int]						=							filter_non_english
  
  
  
  
  
  
      def 	lowercase__ (	self			:    str    ):
          super().setUp()
  
          lowerCAmelCase				:			Optional[Any]    			=      [
              '[UNK]',
              '[CLS]',
              '[SEP]',
              '[PAD]',
              '[MASK]',
              'want',
              '##want',
              '##ed',
              'wa',
              'un',
              'runn',
              '##ing',
              ',',
              'low',
              'lowest',
          ]
          lowerCAmelCase				:			int    			=      os.path.join(self.tmpdirname    ,				VOCAB_FILES_NAMES['vocab_file']    )
          with open(self.vocab_file    ,				'w'    ,				encoding='utf-8'    ) as vocab_writer:
              vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]    )    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[Any]    ,				UpperCAmelCase_			:    Dict    ):
          lowerCAmelCase				:			str    			=      'UNwant\u00E9d,running'
          lowerCAmelCase				:			Tuple    			=      'unwanted, running'
          return input_text, output_text
  
  
  
  
  
  
      def 	lowercase__ (	self			:    int    ):
          lowerCAmelCase				:			Optional[int]    			=      self.tokenizer_class(self.vocab_file    )
  
          lowerCAmelCase				:			Optional[Any]    			=      tokenizer.tokenize('UNwant\u00E9d,running'    )
          self.assertListEqual(UpperCAmelCase_    ,				['un', '##want', '##ed', ',', 'runn', '##ing']    )
          self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_    )    ,				[9, 6, 7, 12, 10, 11]    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[Any]    ):
          if not self.test_rust_tokenizer:
              return
  
          lowerCAmelCase				:			Tuple    			=      self.get_tokenizer()
          lowerCAmelCase				:			Tuple    			=      self.get_rust_tokenizer()
  
          lowerCAmelCase				:			Union[str, Any]    			=      'UNwant\u00E9d,running'
  
          lowerCAmelCase				:			Optional[Any]    			=      tokenizer.tokenize(UpperCAmelCase_    )
          lowerCAmelCase				:			int    			=      rust_tokenizer.tokenize(UpperCAmelCase_    )
          self.assertListEqual(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
          lowerCAmelCase				:			Tuple    			=      tokenizer.encode(UpperCAmelCase_    ,				add_special_tokens=UpperCAmelCase_    )
          lowerCAmelCase				:			Dict    			=      rust_tokenizer.encode(UpperCAmelCase_    ,				add_special_tokens=UpperCAmelCase_    )
          self.assertListEqual(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
          lowerCAmelCase				:			List[Any]    			=      self.get_rust_tokenizer()
          lowerCAmelCase				:			Tuple    			=      tokenizer.encode(UpperCAmelCase_    )
          lowerCAmelCase				:			Tuple    			=      rust_tokenizer.encode(UpperCAmelCase_    )
          self.assertListEqual(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
          # With lower casing
          lowerCAmelCase				:			Optional[int]    			=      self.get_tokenizer(do_lower_case=UpperCAmelCase_    )
          lowerCAmelCase				:			Any    			=      self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_    )
  
          lowerCAmelCase				:			Tuple    			=      'UNwant\u00E9d,running'
  
          lowerCAmelCase				:			str    			=      tokenizer.tokenize(UpperCAmelCase_    )
          lowerCAmelCase				:			Optional[int]    			=      rust_tokenizer.tokenize(UpperCAmelCase_    )
          self.assertListEqual(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
          lowerCAmelCase				:			List[str]    			=      tokenizer.encode(UpperCAmelCase_    ,				add_special_tokens=UpperCAmelCase_    )
          lowerCAmelCase				:			str    			=      rust_tokenizer.encode(UpperCAmelCase_    ,				add_special_tokens=UpperCAmelCase_    )
          self.assertListEqual(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
          lowerCAmelCase				:			Union[str, Any]    			=      self.get_rust_tokenizer()
          lowerCAmelCase				:			Union[str, Any]    			=      tokenizer.encode(UpperCAmelCase_    )
          lowerCAmelCase				:			Optional[int]    			=      rust_tokenizer.encode(UpperCAmelCase_    )
          self.assertListEqual(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[str]    ):
          lowerCAmelCase				:			Dict    			=      BasicTokenizer()
  
          self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'    )    ,				['ah', '\u535A', '\u63A8', 'zz']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Any    ):
          lowerCAmelCase				:			str    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )    ,				['hello', '!', 'how', 'are', 'you', '?']    )
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,				['hello']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    str    ):
          lowerCAmelCase				:			Optional[int]    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    ,				strip_accents=UpperCAmelCase_    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,				['hällo', '!', 'how', 'are', 'you', '?']    )
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,				['h\u00E9llo']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Tuple    ):
          lowerCAmelCase				:			Tuple    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    ,				strip_accents=UpperCAmelCase_    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,				['hallo', '!', 'how', 'are', 'you', '?']    )
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,				['hello']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Dict    ):
          lowerCAmelCase				:			int    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,				['hallo', '!', 'how', 'are', 'you', '?']    )
          self.assertListEqual(tokenizer.tokenize('H\u00E9llo'    )    ,				['hello']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    int    ):
          lowerCAmelCase				:			Any    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '    )    ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Dict    ):
          lowerCAmelCase				:			int    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    ,				strip_accents=UpperCAmelCase_    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,				['HäLLo', '!', 'how', 'Are', 'yoU', '?']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    int    ):
          lowerCAmelCase				:			List[str]    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    ,				strip_accents=UpperCAmelCase_    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '    )    ,				['HaLLo', '!', 'how', 'Are', 'yoU', '?']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Union[str, Any]    ):
          lowerCAmelCase				:			str    			=      BasicTokenizer(do_lower_case=UpperCAmelCase_    ,				never_split=['[UNK]']    )
  
          self.assertListEqual(
              tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'    )    ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[str]    ):
          lowerCAmelCase				:			Optional[Any]    			=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']
  
          lowerCAmelCase				:			int    			=      {}
          for i, token in enumerate(UpperCAmelCase_    ):
              lowerCAmelCase				:			Union[str, Any]    			=      i
          lowerCAmelCase				:			List[str]    			=      WordpieceTokenizer(vocab=UpperCAmelCase_    ,				unk_token='[UNK]'    )
  
          self.assertListEqual(tokenizer.tokenize(''    )    ,				[]    )
  
          self.assertListEqual(tokenizer.tokenize('unwanted running'    )    ,				['un', '##want', '##ed', 'runn', '##ing']    )
  
          self.assertListEqual(tokenizer.tokenize('unwantedX running'    )    ,				['[UNK]', 'runn', '##ing']    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Dict    ):
          self.assertTrue(_is_whitespace(' '    )    )
          self.assertTrue(_is_whitespace('\t'    )    )
          self.assertTrue(_is_whitespace('\r'    )    )
          self.assertTrue(_is_whitespace('\n'    )    )
          self.assertTrue(_is_whitespace('\u00A0'    )    )
  
          self.assertFalse(_is_whitespace('A'    )    )
          self.assertFalse(_is_whitespace('-'    )    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[str]    ):
          self.assertTrue(_is_control('\u0005'    )    )
  
          self.assertFalse(_is_control('A'    )    )
          self.assertFalse(_is_control(' '    )    )
          self.assertFalse(_is_control('\t'    )    )
          self.assertFalse(_is_control('\r'    )    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Union[str, Any]    ):
          self.assertTrue(_is_punctuation('-'    )    )
          self.assertTrue(_is_punctuation('$'    )    )
          self.assertTrue(_is_punctuation('`'    )    )
          self.assertTrue(_is_punctuation('.'    )    )
  
          self.assertFalse(_is_punctuation('A'    )    )
          self.assertFalse(_is_punctuation(' '    )    )
  
  
  
  
  
  
      def 	lowercase__ (	self			:    List[str]    ):
          lowerCAmelCase				:			Optional[int]    			=      self.get_tokenizer()
  
          # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
          self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_    ) for t in ['Test', '\xad', 'test']]    ,				[['[UNK]'], [], ['[UNK]']]    )
  
          if self.test_rust_tokenizer:
              lowerCAmelCase				:			List[str]    			=      self.get_rust_tokenizer()
              self.assertListEqual(
                  [rust_tokenizer.tokenize(UpperCAmelCase_    ) for t in ['Test', '\xad', 'test']]    ,				[['[UNK]'], [], ['[UNK]']]    )
  
  
  
  
  
  
      @slow
      def 	lowercase__ (	self			:    List[Any]    ):
          lowerCAmelCase				:			Optional[int]    			=      self.tokenizer_class.from_pretrained('bert-base-uncased'    )
  
          lowerCAmelCase				:			Optional[int]    			=      tokenizer.encode('sequence builders'    ,				add_special_tokens=UpperCAmelCase_    )
          lowerCAmelCase				:			Union[str, Any]    			=      tokenizer.encode('multi-sequence build'    ,				add_special_tokens=UpperCAmelCase_    )
  
          lowerCAmelCase				:			int    			=      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_    )
          lowerCAmelCase				:			str    			=      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_    ,				UpperCAmelCase_    )
  
          assert encoded_sentence == [101] + text + [102]
          assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  
      def 	lowercase__ (	self			:    Optional[Any]    ):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                  lowerCAmelCase				:			Optional[Any]    			=      self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_    ,				**UpperCAmelCase_    )
  
                  lowerCAmelCase				:			List[Any]    			=      f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                  lowerCAmelCase				:			List[Any]    			=      tokenizer_r.encode_plus(
                      UpperCAmelCase_    ,				return_attention_mask=UpperCAmelCase_    ,				return_token_type_ids=UpperCAmelCase_    ,				return_offsets_mapping=UpperCAmelCase_    ,				add_special_tokens=UpperCAmelCase_    ,				)
  
                  lowerCAmelCase				:			Optional[Any]    			=      tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_    ,				'do_lower_case'    ) else False
                  lowerCAmelCase				:			Tuple    			=      (
                      [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), 'A'),
                          ((1, 2), ','),
                          ((3, 5), 'na'),
                          ((5, 6), '##ï'),
                          ((6, 8), '##ve'),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), 'Allen'),
                          ((21, 23), '##NL'),
                          ((23, 24), '##P'),
                          ((25, 33), 'sentence'),
                          ((33, 34), '.'),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                      if not do_lower_case
                      else [
                          ((0, 0), tokenizer_r.cls_token),
                          ((0, 1), 'a'),
                          ((1, 2), ','),
                          ((3, 8), 'naive'),
                          ((9, 15), tokenizer_r.mask_token),
                          ((16, 21), 'allen'),
                          ((21, 23), '##nl'),
                          ((23, 24), '##p'),
                          ((25, 33), 'sentence'),
                          ((33, 34), '.'),
                          ((0, 0), tokenizer_r.sep_token),
                      ]
                  )
  
                  self.assertEqual(
                      [e[1] for e in expected_results]    ,				tokenizer_r.convert_ids_to_tokens(tokens['input_ids']    )    )
                  self.assertEqual([e[0] for e in expected_results]    ,				tokens['offset_mapping']    )
  
  
  
  
  
  
      @slow
      def 	lowercase__ (	self			:    Union[str, Any]    ):
          for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"    ):
                  lowerCAmelCase				:			int    			=      self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_    ,				**UpperCAmelCase_    )
                  lowerCAmelCase				:			Tuple    			=      self.tokenizer_class.from_pretrained(UpperCAmelCase_    ,				**UpperCAmelCase_    )
                  lowerCAmelCase				:			Optional[Any]    			=      [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]
  
                  lowerCAmelCase				:			Any    			=      tokenizer_r.batch_encode_candidates(UpperCAmelCase_    ,				max_length=10    ,				return_tensors='np'    )
                  lowerCAmelCase				:			int    			=      tokenizer_p.batch_encode_candidates(UpperCAmelCase_    ,				max_length=10    ,				return_tensors='np'    )
  
                  lowerCAmelCase				:			Tuple    			=      (2, 2, 10)
  
                  self.assertEqual(encoded_sentence_r['input_ids'].shape    ,				UpperCAmelCase_    )
                  self.assertEqual(encoded_sentence_r['attention_mask'].shape    ,				UpperCAmelCase_    )
                  self.assertEqual(encoded_sentence_r['token_type_ids'].shape    ,				UpperCAmelCase_    )
  
                  self.assertEqual(encoded_sentence_p['input_ids'].shape    ,				UpperCAmelCase_    )
                  self.assertEqual(encoded_sentence_p['attention_mask'].shape    ,				UpperCAmelCase_    )
                  self.assertEqual(encoded_sentence_p['token_type_ids'].shape    ,				UpperCAmelCase_    )
  
  
  
  
  
  
style_context: 
  
  from typing import TYPE_CHECKING
  
  from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available
  
  
  __A   :     List[Any]      =			{'''configuration_umt5''': ['''UMT5Config''', '''UMT5OnnxConfig''']}
  
  
  try:
   if not is_torch_available():
    raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
   pass
  else:
   __A   :     Tuple      =			[
       '''UMT5EncoderModel''',
       '''UMT5ForConditionalGeneration''',
       '''UMT5ForQuestionAnswering''',
       '''UMT5Model''',
       '''UMT5PreTrainedModel''',
   ]
  
  if TYPE_CHECKING:
   from .configuration_umta import UMTaConfig, UMTaOnnxConfig
  
   try:
    if not is_torch_available():
     raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
    pass
   else:
    from .modeling_umta import (
        UMTaEncoderModel,
        UMTaForConditionalGeneration,
        UMTaForQuestionAnswering,
        UMTaModel,
        UMTaPreTrainedModel,
    )
  else:
   import sys
  
   __A   :     Any      =			_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
  
  
  
  
  
  