code: 
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  @require_tokenizers
  class   __lowerCamelCase (_a							,			unittest.TestCase	):
         _lowercase       							=    RealmTokenizer
         _lowercase       							=    RealmTokenizerFast
         _lowercase       							=    True
         _lowercase       							=    True
         _lowercase       							=    filter_non_english
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Union[str, Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              super().setUp()
  
              __UpperCamelCase   = [
                  '[UNK]',
                  '[CLS]',
                  '[SEP]',
                  '[PAD]',
                  '[MASK]',
                  'want',
                  '##want',
                  '##ed',
                  'wa',
                  'un',
                  'runn',
                  '##ing',
                  ',',
                  'low',
                  'lowest',
              ]
              __UpperCamelCase   = os.path.join(self.tmpdirname,VOCAB_FILES_NAMES['vocab_file']				)
              with open(self.vocab_file,'w',encoding='utf-8'				) as vocab_writer:
                   vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Optional[int],A_:   Optional[Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = 'UNwant\u00E9d,running'
              __UpperCamelCase   = 'unwanted, running'
              return input_text, output_text
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   List[Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = self.tokenizer_class(self.vocab_file				)
  
              __UpperCamelCase   = tokenizer.tokenize('UNwant\u00E9d,running'				)
              self.assertListEqual(A_,['un', '##want', '##ed', ',', 'runn', '##ing']				)
              self.assertListEqual(tokenizer.convert_tokens_to_ids(A_				),[9, 6, 7, 12, 10, 11]				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Tuple				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              if not self.test_rust_tokenizer:
                   return
  
              __UpperCamelCase   = self.get_tokenizer()
              __UpperCamelCase   = self.get_rust_tokenizer()
  
              __UpperCamelCase   = 'UNwant\u00E9d,running'
  
              __UpperCamelCase   = tokenizer.tokenize(A_				)
              __UpperCamelCase   = rust_tokenizer.tokenize(A_				)
              self.assertListEqual(A_,A_				)
  
              __UpperCamelCase   = tokenizer.encode(A_,add_special_tokens=A_				)
              __UpperCamelCase   = rust_tokenizer.encode(A_,add_special_tokens=A_				)
              self.assertListEqual(A_,A_				)
  
              __UpperCamelCase   = self.get_rust_tokenizer()
              __UpperCamelCase   = tokenizer.encode(A_				)
              __UpperCamelCase   = rust_tokenizer.encode(A_				)
              self.assertListEqual(A_,A_				)
  
              # With lower casing
              __UpperCamelCase   = self.get_tokenizer(do_lower_case=A_				)
              __UpperCamelCase   = self.get_rust_tokenizer(do_lower_case=A_				)
  
              __UpperCamelCase   = 'UNwant\u00E9d,running'
  
              __UpperCamelCase   = tokenizer.tokenize(A_				)
              __UpperCamelCase   = rust_tokenizer.tokenize(A_				)
              self.assertListEqual(A_,A_				)
  
              __UpperCamelCase   = tokenizer.encode(A_,add_special_tokens=A_				)
              __UpperCamelCase   = rust_tokenizer.encode(A_,add_special_tokens=A_				)
              self.assertListEqual(A_,A_				)
  
              __UpperCamelCase   = self.get_rust_tokenizer()
              __UpperCamelCase   = tokenizer.encode(A_				)
              __UpperCamelCase   = rust_tokenizer.encode(A_				)
              self.assertListEqual(A_,A_				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Dict				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer()
  
              self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'				),['ah', '\u535A', '\u63A8', 'zz']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Union[str, Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				),['hello', '!', 'how', 'are', 'you', '?']				)
              self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['hello']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   int				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['hällo', '!', 'how', 'are', 'you', '?']				)
              self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['h\u00E9llo']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Optional[Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['hallo', '!', 'how', 'are', 'you', '?']				)
              self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['hello']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   List[str]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['hallo', '!', 'how', 'are', 'you', '?']				)
              self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				),['hello']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   str				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				),['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Optional[int]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['HäLLo', '!', 'how', 'Are', 'yoU', '?']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   str				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,strip_accents=A_				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				),['HaLLo', '!', 'how', 'Are', 'yoU', '?']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   List[str]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = BasicTokenizer(do_lower_case=A_,never_split=['[UNK]']				)
  
              self.assertListEqual(
                  tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'				),['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Union[str, Any]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']
  
              __UpperCamelCase   = {}
              for i, token in enumerate(A_				):
                   __UpperCamelCase   = i
              __UpperCamelCase   = WordpieceTokenizer(vocab=A_,unk_token='[UNK]'				)
  
              self.assertListEqual(tokenizer.tokenize(''				),[]				)
  
              self.assertListEqual(tokenizer.tokenize('unwanted running'				),['un', '##want', '##ed', 'runn', '##ing']				)
  
              self.assertListEqual(tokenizer.tokenize('unwantedX running'				),['[UNK]', 'runn', '##ing']				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Any				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              self.assertTrue(_is_whitespace(' '				)				)
              self.assertTrue(_is_whitespace('\t'				)				)
              self.assertTrue(_is_whitespace('\r'				)				)
              self.assertTrue(_is_whitespace('\n'				)				)
              self.assertTrue(_is_whitespace('\u00A0'				)				)
  
              self.assertFalse(_is_whitespace('A'				)				)
              self.assertFalse(_is_whitespace('-'				)				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Optional[int]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              self.assertTrue(_is_control('\u0005'				)				)
  
              self.assertFalse(_is_control('A'				)				)
              self.assertFalse(_is_control(' '				)				)
              self.assertFalse(_is_control('\t'				)				)
              self.assertFalse(_is_control('\r'				)				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   str				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              self.assertTrue(_is_punctuation('-'				)				)
              self.assertTrue(_is_punctuation('$'				)				)
              self.assertTrue(_is_punctuation('`'				)				)
              self.assertTrue(_is_punctuation('.'				)				)
  
              self.assertFalse(_is_punctuation('A'				)				)
              self.assertFalse(_is_punctuation(' '				)				)
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   List[str]				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = self.get_tokenizer()
  
              # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
              self.assertListEqual([tokenizer.tokenize(A_				) for t in ['Test', '\xad', 'test']],[['[UNK]'], [], ['[UNK]']]				)
  
              if self.test_rust_tokenizer:
                   __UpperCamelCase   = self.get_rust_tokenizer()
                   self.assertListEqual(
                       [rust_tokenizer.tokenize(A_				) for t in ['Test', '\xad', 'test']],[['[UNK]'], [], ['[UNK]']]				)
  
  
  
  
  
  
  
         @slow
         def 		snake_case_  ( self:   str				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              __UpperCamelCase   = self.tokenizer_class.from_pretrained('bert-base-uncased'				)
  
              __UpperCamelCase   = tokenizer.encode('sequence builders',add_special_tokens=A_				)
              __UpperCamelCase   = tokenizer.encode('multi-sequence build',add_special_tokens=A_				)
  
              __UpperCamelCase   = tokenizer.build_inputs_with_special_tokens(A_				)
              __UpperCamelCase   = tokenizer.build_inputs_with_special_tokens(A_,A_				)
  
              assert encoded_sentence == [101] + text + [102]
              assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
  
  
  
  
         def 		snake_case_  ( self:   Tuple				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
                        __UpperCamelCase   = self.rust_tokenizer_class.from_pretrained(A_,**A_				)
  
                        __UpperCamelCase   = F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
                        __UpperCamelCase   = tokenizer_r.encode_plus(
                            A_,return_attention_mask=A_,return_token_type_ids=A_,return_offsets_mapping=A_,add_special_tokens=A_,)
  
                        __UpperCamelCase   = tokenizer_r.do_lower_case if hasattr(A_,'do_lower_case'				) else False
                        __UpperCamelCase   = (
                            [
                                ((0, 0), tokenizer_r.cls_token),
                                ((0, 1), 'A'),
                                ((1, 2), ','),
                                ((3, 5), 'na'),
                                ((5, 6), '##ï'),
                                ((6, 8), '##ve'),
                                ((9, 15), tokenizer_r.mask_token),
                                ((16, 21), 'Allen'),
                                ((21, 23), '##NL'),
                                ((23, 24), '##P'),
                                ((25, 33), 'sentence'),
                                ((33, 34), '.'),
                                ((0, 0), tokenizer_r.sep_token),
                            ]
                            if not do_lower_case
                            else [
                                ((0, 0), tokenizer_r.cls_token),
                                ((0, 1), 'a'),
                                ((1, 2), ','),
                                ((3, 8), 'naive'),
                                ((9, 15), tokenizer_r.mask_token),
                                ((16, 21), 'allen'),
                                ((21, 23), '##nl'),
                                ((23, 24), '##p'),
                                ((25, 33), 'sentence'),
                                ((33, 34), '.'),
                                ((0, 0), tokenizer_r.sep_token),
                            ]
                        )
  
                        self.assertEqual(
                            [e[1] for e in expected_results],tokenizer_r.convert_ids_to_tokens(tokens['input_ids']				)				)
                        self.assertEqual([e[0] for e in expected_results],tokens['offset_mapping']				)
  
  
  
  
  
  
  
         @slow
         def 		snake_case_  ( self:   Any				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
                        __UpperCamelCase   = self.rust_tokenizer_class.from_pretrained(A_,**A_				)
                        __UpperCamelCase   = self.tokenizer_class.from_pretrained(A_,**A_				)
                        __UpperCamelCase   = [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]
  
                        __UpperCamelCase   = tokenizer_r.batch_encode_candidates(A_,max_length=10,return_tensors='np'				)
                        __UpperCamelCase   = tokenizer_p.batch_encode_candidates(A_,max_length=10,return_tensors='np'				)
  
                        __UpperCamelCase   = (2, 2, 10)
  
                        self.assertEqual(encoded_sentence_r['input_ids'].shape,A_				)
                        self.assertEqual(encoded_sentence_r['attention_mask'].shape,A_				)
                        self.assertEqual(encoded_sentence_r['token_type_ids'].shape,A_				)
  
                        self.assertEqual(encoded_sentence_p['input_ids'].shape,A_				)
                        self.assertEqual(encoded_sentence_p['attention_mask'].shape,A_				)
                        self.assertEqual(encoded_sentence_p['token_type_ids'].shape,A_				)
  
  
  
style_context: 
  
  
  from ..utils import DummyObject, requires_backends
  
  
  
  class   __lowerCamelCase (metaclass=_a	):
         _lowercase       							=    ["""torch""", """transformers""", """k_diffusion"""]
  
  
  
  
  
  
  
         def __init__( self:   Tuple,*A_:   Union[str, Any],**A_:   Tuple				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              requires_backends(self,['torch', 'transformers', 'k_diffusion']				)
  
  
  
  
  
  
  
         @classmethod
         def 		snake_case_  ( cls:   int,*A_:   Optional[Any],**A_:   Tuple				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              requires_backends(cls,['torch', 'transformers', 'k_diffusion']				)
  
  
  
  
  
  
  
         @classmethod
         def 		snake_case_  ( cls:   int,*A_:   Tuple,**A_:   Tuple				):
  
  
  
  
  
  
              '''simple docstring'''
  
  
  
  
              requires_backends(cls,['torch', 'transformers', 'k_diffusion']				)
  
  
  