code: 
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  
  
  @require_tokenizers
  class 		__magic_name__    (							lowerCamelCase__	, unittest.TestCase							):
  
     '''simple docstring'''
  
  
  
  
     lowerCamelCase__				:			Dict												=     RealmTokenizer
     lowerCamelCase__				:			Optional[Any]												=     RealmTokenizerFast
     lowerCamelCase__				:			Optional[int]												=     True
     lowerCamelCase__				:			List[str]												=     True
     lowerCamelCase__				:			Optional[int]												=     filter_non_english
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Optional[int]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       super().setUp()
  
       a__						=[
           '''[UNK]''',
           '''[CLS]''',
           '''[SEP]''',
           '''[PAD]''',
           '''[MASK]''',
           '''want''',
           '''##want''',
           '''##ed''',
           '''wa''',
           '''un''',
           '''runn''',
           '''##ing''',
           ''',''',
           '''low''',
           '''lowest''',
       ]
       a__						=os.path.join(self.tmpdirname,					VOCAB_FILES_NAMES['''vocab_file'''] )
       with open(self.vocab_file,					'''w''',					encoding='''utf-8''' ) as vocab_writer:
         vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )
  
  
  
     def 				_UpperCAmelCase				(       self,					lowercase_ )							->					Any:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						='''UNwant\u00E9d,running'''
       a__						='''unwanted, running'''
       return input_text, output_text
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Optional[int]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.tokenizer_class(self.vocab_file )
  
       a__						=tokenizer.tokenize('''UNwant\u00E9d,running''' )
       self.assertListEqual(lowercase_,					['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
       self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_ ),					[9, 6, 7, 12, 10, 11] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					int:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       if not self.test_rust_tokenizer:
         return
  
       a__						=self.get_tokenizer()
       a__						=self.get_rust_tokenizer()
  
       a__						='''UNwant\u00E9d,running'''
  
       a__						=tokenizer.tokenize(lowercase_ )
       a__						=rust_tokenizer.tokenize(lowercase_ )
       self.assertListEqual(lowercase_,					lowercase_ )
  
       a__						=tokenizer.encode(lowercase_,					add_special_tokens=lowercase_ )
       a__						=rust_tokenizer.encode(lowercase_,					add_special_tokens=lowercase_ )
       self.assertListEqual(lowercase_,					lowercase_ )
  
       a__						=self.get_rust_tokenizer()
       a__						=tokenizer.encode(lowercase_ )
       a__						=rust_tokenizer.encode(lowercase_ )
       self.assertListEqual(lowercase_,					lowercase_ )
  
       # With lower casing
       a__						=self.get_tokenizer(do_lower_case=lowercase_ )
       a__						=self.get_rust_tokenizer(do_lower_case=lowercase_ )
  
       a__						='''UNwant\u00E9d,running'''
  
       a__						=tokenizer.tokenize(lowercase_ )
       a__						=rust_tokenizer.tokenize(lowercase_ )
       self.assertListEqual(lowercase_,					lowercase_ )
  
       a__						=tokenizer.encode(lowercase_,					add_special_tokens=lowercase_ )
       a__						=rust_tokenizer.encode(lowercase_,					add_special_tokens=lowercase_ )
       self.assertListEqual(lowercase_,					lowercase_ )
  
       a__						=self.get_rust_tokenizer()
       a__						=tokenizer.encode(lowercase_ )
       a__						=rust_tokenizer.encode(lowercase_ )
       self.assertListEqual(lowercase_,					lowercase_ )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					str:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer()
  
       self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' ),					['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Optional[int]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_ )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' ),					['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),					['''hello'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					int:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_,					strip_accents=lowercase_ )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),					['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),					['''h\u00E9llo'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Any:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_,					strip_accents=lowercase_ )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),					['''hello'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Dict:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_ )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),					['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
       self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' ),					['''hello'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Any:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_ )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' ),					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Any:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_,					strip_accents=lowercase_ )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),					['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					List[Any]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_,					strip_accents=lowercase_ )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' ),					['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Tuple:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=BasicTokenizer(do_lower_case=lowercase_,					never_split=['''[UNK]'''] )
  
       self.assertListEqual(
           tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' ),					['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					List[str]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']
  
       a__						={}
       for i, token in enumerate(lowercase_ ):
         a__						=i
       a__						=WordpieceTokenizer(vocab=lowercase_,					unk_token='''[UNK]''' )
  
       self.assertListEqual(tokenizer.tokenize('''''' ),					[] )
  
       self.assertListEqual(tokenizer.tokenize('''unwanted running''' ),					['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )
  
       self.assertListEqual(tokenizer.tokenize('''unwantedX running''' ),					['''[UNK]''', '''runn''', '''##ing'''] )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					List[str]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       self.assertTrue(_is_whitespace(''' ''' ) )
       self.assertTrue(_is_whitespace('''\t''' ) )
       self.assertTrue(_is_whitespace('''\r''' ) )
       self.assertTrue(_is_whitespace('''\n''' ) )
       self.assertTrue(_is_whitespace('''\u00A0''' ) )
  
       self.assertFalse(_is_whitespace('''A''' ) )
       self.assertFalse(_is_whitespace('''-''' ) )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Any:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       self.assertTrue(_is_control('''\u0005''' ) )
  
       self.assertFalse(_is_control('''A''' ) )
       self.assertFalse(_is_control(''' ''' ) )
       self.assertFalse(_is_control('''\t''' ) )
       self.assertFalse(_is_control('''\r''' ) )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Dict:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       self.assertTrue(_is_punctuation('''-''' ) )
       self.assertTrue(_is_punctuation('''$''' ) )
       self.assertTrue(_is_punctuation('''`''' ) )
       self.assertTrue(_is_punctuation('''.''' ) )
  
       self.assertFalse(_is_punctuation('''A''' ) )
       self.assertFalse(_is_punctuation(''' ''' ) )
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Dict:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.get_tokenizer()
  
       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(lowercase_ ) for t in ['''Test''', '''\xad''', '''test''']],					[['''[UNK]'''], [], ['''[UNK]''']] )
  
       if self.test_rust_tokenizer:
         a__						=self.get_rust_tokenizer()
         self.assertListEqual(
             [rust_tokenizer.tokenize(lowercase_ ) for t in ['''Test''', '''\xad''', '''test''']],					[['''[UNK]'''], [], ['''[UNK]''']] )
  
  
  
     @slow
     def 				_UpperCAmelCase				(       self )							->					str:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       a__						=self.tokenizer_class.from_pretrained('''bert-base-uncased''' )
  
       a__						=tokenizer.encode('''sequence builders''',					add_special_tokens=lowercase_ )
       a__						=tokenizer.encode('''multi-sequence build''',					add_special_tokens=lowercase_ )
  
       a__						=tokenizer.build_inputs_with_special_tokens(lowercase_ )
       a__						=tokenizer.build_inputs_with_special_tokens(lowercase_,					lowercase_ )
  
       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]
  
  
  
     def 				_UpperCAmelCase				(       self )							->					Any:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
           a__						=self.rust_tokenizer_class.from_pretrained(lowercase_,					**lowercase_ )
  
           a__						=F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
           a__						=tokenizer_r.encode_plus(
               lowercase_,					return_attention_mask=lowercase_,					return_token_type_ids=lowercase_,					return_offsets_mapping=lowercase_,					add_special_tokens=lowercase_,					)
  
           a__						=tokenizer_r.do_lower_case if hasattr(lowercase_,					'''do_lower_case''' ) else False
           a__						=(
               [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''A'''),
                   ((1, 2), ''','''),
                   ((3, 5), '''na'''),
                   ((5, 6), '''##ï'''),
                   ((6, 8), '''##ve'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''Allen'''),
                   ((21, 23), '''##NL'''),
                   ((23, 24), '''##P'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
               if not do_lower_case
               else [
                   ((0, 0), tokenizer_r.cls_token),
                   ((0, 1), '''a'''),
                   ((1, 2), ''','''),
                   ((3, 8), '''naive'''),
                   ((9, 15), tokenizer_r.mask_token),
                   ((16, 21), '''allen'''),
                   ((21, 23), '''##nl'''),
                   ((23, 24), '''##p'''),
                   ((25, 33), '''sentence'''),
                   ((33, 34), '''.'''),
                   ((0, 0), tokenizer_r.sep_token),
               ]
           )
  
           self.assertEqual(
               [e[1] for e in expected_results],					tokenizer_r.convert_ids_to_tokens(tokens['''input_ids'''] ) )
           self.assertEqual([e[0] for e in expected_results],					tokens['''offset_mapping'''] )
  
  
  
     @slow
     def 				_UpperCAmelCase				(       self )							->					List[Any]:
  
  
  
  
  
  
       """simple docstring"""
  
  
  
  
  
  
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
           a__						=self.rust_tokenizer_class.from_pretrained(lowercase_,					**lowercase_ )
           a__						=self.tokenizer_class.from_pretrained(lowercase_,					**lowercase_ )
           a__						=[['''Hello world!''', '''Nice to meet you!'''], ['''The cute cat.''', '''The adorable dog.''']]
  
           a__						=tokenizer_r.batch_encode_candidates(lowercase_,					max_length=10,					return_tensors='''np''' )
           a__						=tokenizer_p.batch_encode_candidates(lowercase_,					max_length=10,					return_tensors='''np''' )
  
           a__						=(2, 2, 10)
  
           self.assertEqual(encoded_sentence_r['''input_ids'''].shape,					lowercase_ )
           self.assertEqual(encoded_sentence_r['''attention_mask'''].shape,					lowercase_ )
           self.assertEqual(encoded_sentence_r['''token_type_ids'''].shape,					lowercase_ )
  
           self.assertEqual(encoded_sentence_p['''input_ids'''].shape,					lowercase_ )
           self.assertEqual(encoded_sentence_p['''attention_mask'''].shape,					lowercase_ )
           self.assertEqual(encoded_sentence_p['''token_type_ids'''].shape,					lowercase_ )
  
style_context: 
  
  from dataclasses import dataclass
  from typing import List, Optional, Union
  
  import numpy as np
  import PIL
  from PIL import Image
  
  from ...utils import BaseOutput, is_torch_available, is_transformers_available
  
  
  
  
  
  
  
  @dataclass
  # Copied from diffusers.pipelines.stable_diffusion.__init__.StableDiffusionPipelineOutput with Stable->Alt
  class 		__magic_name__    (							lowerCamelCase__							):
  
     '''simple docstring'''
  
  
  
  
     lowerCamelCase__				:			Union[List[PIL.Image.Image], np.ndarray]
     lowerCamelCase__				:			Optional[List[bool]]
  
  
  if is_transformers_available() and is_torch_available():
         from .modeling_roberta_series import RobertaSeriesModelWithTransformation
         from .pipeline_alt_diffusion import AltDiffusionPipeline
         from .pipeline_alt_diffusion_imgaimg import AltDiffusionImgaImgPipeline
  