code: 
  
  
  import os
  import unittest
  
  from transformers import RealmTokenizerFast
  from transformers.models.bert.tokenization_bert import (
      VOCAB_FILES_NAMES,
      BasicTokenizer,
      WordpieceTokenizer,
      _is_control,
      _is_punctuation,
      _is_whitespace,
  )
  from transformers.models.realm.tokenization_realm import RealmTokenizer
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
  
  
  
  
  
  @require_tokenizers
  class    A					(     A_      ,       unittest.TestCase				):
  							UpperCamelCase_     :							Tuple =RealmTokenizer
  							UpperCamelCase_     :							Optional[int] =RealmTokenizerFast
  							UpperCamelCase_     :							int =True
  							UpperCamelCase_     :							Tuple =True
  							UpperCamelCase_     :							Optional[int] =filter_non_english
  
  
  							def     _A (self      ):
  													super().setUp()
  
  													__lowercase=    [
  													    '[UNK]',
  													    '[CLS]',
  													    '[SEP]',
  													    '[PAD]',
  													    '[MASK]',
  													    'want',
  													    '##want',
  													    '##ed',
  													    'wa',
  													    'un',
  													    'runn',
  													    '##ing',
  													    ',',
  													    'low',
  													    'lowest',
  													]
  													__lowercase=    os.path.join(self.tmpdirname   ,       VOCAB_FILES_NAMES['vocab_file']      )
  													with open(self.vocab_file   ,       'w'   ,       encoding='utf-8'      ) as vocab_writer:
  																			vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )
  
  
  							def     _A (self   ,       lowerCAmelCase      ):
  													__lowercase=    'UNwant\u00E9d,running'
  													__lowercase=    'unwanted, running'
  													return input_text, output_text
  
  
  							def     _A (self      ):
  													__lowercase=    self.tokenizer_class(self.vocab_file      )
  
  													__lowercase=    tokenizer.tokenize('UNwant\u00E9d,running'      )
  													self.assertListEqual(UpperCAmelCase_   ,       ['un', '##want', '##ed', ',', 'runn', '##ing']      )
  													self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )   ,       [9, 6, 7, 1_2, 1_0, 1_1]      )
  
  
  							def     _A (self      ):
  													if not self.test_rust_tokenizer:
  																			return
  
  													__lowercase=    self.get_tokenizer()
  													__lowercase=    self.get_rust_tokenizer()
  
  													__lowercase=    'UNwant\u00E9d,running'
  
  													__lowercase=    tokenizer.tokenize(UpperCAmelCase_      )
  													__lowercase=    rust_tokenizer.tokenize(UpperCAmelCase_      )
  													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )
  
  													__lowercase=    tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
  													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
  													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )
  
  													__lowercase=    self.get_rust_tokenizer()
  													__lowercase=    tokenizer.encode(UpperCAmelCase_      )
  													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_      )
  													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )
  
  													# With lower casing
  													__lowercase=    self.get_tokenizer(do_lower_case=UpperCAmelCase_      )
  													__lowercase=    self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_      )
  
  													__lowercase=    'UNwant\u00E9d,running'
  
  													__lowercase=    tokenizer.tokenize(UpperCAmelCase_      )
  													__lowercase=    rust_tokenizer.tokenize(UpperCAmelCase_      )
  													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )
  
  													__lowercase=    tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
  													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
  													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )
  
  													__lowercase=    self.get_rust_tokenizer()
  													__lowercase=    tokenizer.encode(UpperCAmelCase_      )
  													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_      )
  													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer()
  
  													self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'      )   ,       ['ah', '\u535A', '\u63A8', 'zz']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )   ,       ['hello', '!', 'how', 'are', 'you', '?']      )
  													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['hello']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['hällo', '!', 'how', 'are', 'you', '?']      )
  													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['h\u00E9llo']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['hallo', '!', 'how', 'are', 'you', '?']      )
  													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['hello']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['hallo', '!', 'how', 'are', 'you', '?']      )
  													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['hello']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )   ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['HäLLo', '!', 'how', 'Are', 'yoU', '?']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['HaLLo', '!', 'how', 'Are', 'yoU', '?']      )
  
  
  							def     _A (self      ):
  													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       never_split=['[UNK]']      )
  
  													self.assertListEqual(
  													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'      )   ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']      )
  
  
  							def     _A (self      ):
  													__lowercase=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']
  
  													__lowercase=    {}
  													for i, token in enumerate(UpperCAmelCase_      ):
  																			__lowercase=    i
  													__lowercase=    WordpieceTokenizer(vocab=UpperCAmelCase_   ,       unk_token='[UNK]'      )
  
  													self.assertListEqual(tokenizer.tokenize(''      )   ,       []      )
  
  													self.assertListEqual(tokenizer.tokenize('unwanted running'      )   ,       ['un', '##want', '##ed', 'runn', '##ing']      )
  
  													self.assertListEqual(tokenizer.tokenize('unwantedX running'      )   ,       ['[UNK]', 'runn', '##ing']      )
  
  
  							def     _A (self      ):
  													self.assertTrue(_is_whitespace(' '      )      )
  													self.assertTrue(_is_whitespace('\t'      )      )
  													self.assertTrue(_is_whitespace('\r'      )      )
  													self.assertTrue(_is_whitespace('\n'      )      )
  													self.assertTrue(_is_whitespace('\u00A0'      )      )
  
  													self.assertFalse(_is_whitespace('A'      )      )
  													self.assertFalse(_is_whitespace('-'      )      )
  
  
  							def     _A (self      ):
  													self.assertTrue(_is_control('\u0005'      )      )
  
  													self.assertFalse(_is_control('A'      )      )
  													self.assertFalse(_is_control(' '      )      )
  													self.assertFalse(_is_control('\t'      )      )
  													self.assertFalse(_is_control('\r'      )      )
  
  
  							def     _A (self      ):
  													self.assertTrue(_is_punctuation('-'      )      )
  													self.assertTrue(_is_punctuation('$'      )      )
  													self.assertTrue(_is_punctuation('`'      )      )
  													self.assertTrue(_is_punctuation('.'      )      )
  
  													self.assertFalse(_is_punctuation('A'      )      )
  													self.assertFalse(_is_punctuation(' '      )      )
  
  
  							def     _A (self      ):
  													__lowercase=    self.get_tokenizer()
  
  													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
  													self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_      ) for t in ['Test', '\xad', 'test']]   ,       [['[UNK]'], [], ['[UNK]']]      )
  
  													if self.test_rust_tokenizer:
  																			__lowercase=    self.get_rust_tokenizer()
  																			self.assertListEqual(
  																			    [rust_tokenizer.tokenize(UpperCAmelCase_      ) for t in ['Test', '\xad', 'test']]   ,       [['[UNK]'], [], ['[UNK]']]      )
  
  
  							@slow
  							def     _A (self      ):
  													__lowercase=    self.tokenizer_class.from_pretrained('bert-base-uncased'      )
  
  													__lowercase=    tokenizer.encode('sequence builders'   ,       add_special_tokens=UpperCAmelCase_      )
  													__lowercase=    tokenizer.encode('multi-sequence build'   ,       add_special_tokens=UpperCAmelCase_      )
  
  													__lowercase=    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_      )
  													__lowercase=    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_   ,       UpperCAmelCase_      )
  
  													assert encoded_sentence == [1_0_1] + text + [1_0_2]
  													assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]
  
  
  							def     _A (self      ):
  													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  																			with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
  																									__lowercase=    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_   ,       **UpperCAmelCase_      )
  
  																									__lowercase=    f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
  																									__lowercase=    tokenizer_r.encode_plus(
  																									    UpperCAmelCase_   ,       return_attention_mask=UpperCAmelCase_   ,       return_token_type_ids=UpperCAmelCase_   ,       return_offsets_mapping=UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_   ,       )
  
  																									__lowercase=    tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_   ,       'do_lower_case'      ) else False
  																									__lowercase=    (
  																									    [
  																									        ((0, 0), tokenizer_r.cls_token),
  																									        ((0, 1), 'A'),
  																									        ((1, 2), ','),
  																									        ((3, 5), 'na'),
  																									        ((5, 6), '##ï'),
  																									        ((6, 8), '##ve'),
  																									        ((9, 1_5), tokenizer_r.mask_token),
  																									        ((1_6, 2_1), 'Allen'),
  																									        ((2_1, 2_3), '##NL'),
  																									        ((2_3, 2_4), '##P'),
  																									        ((2_5, 3_3), 'sentence'),
  																									        ((3_3, 3_4), '.'),
  																									        ((0, 0), tokenizer_r.sep_token),
  																									    ]
  																									    if not do_lower_case
  																									    else [
  																									        ((0, 0), tokenizer_r.cls_token),
  																									        ((0, 1), 'a'),
  																									        ((1, 2), ','),
  																									        ((3, 8), 'naive'),
  																									        ((9, 1_5), tokenizer_r.mask_token),
  																									        ((1_6, 2_1), 'allen'),
  																									        ((2_1, 2_3), '##nl'),
  																									        ((2_3, 2_4), '##p'),
  																									        ((2_5, 3_3), 'sentence'),
  																									        ((3_3, 3_4), '.'),
  																									        ((0, 0), tokenizer_r.sep_token),
  																									    ]
  																									)
  
  																									self.assertEqual(
  																									    [e[1] for e in expected_results]   ,       tokenizer_r.convert_ids_to_tokens(tokens['input_ids']      )      )
  																									self.assertEqual([e[0] for e in expected_results]   ,       tokens['offset_mapping']      )
  
  
  
  
  
  
  
  							@slow
  							def     _A (self      ):
  													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
  																			with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
  																									__lowercase=    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_   ,       **UpperCAmelCase_      )
  																									__lowercase=    self.tokenizer_class.from_pretrained(UpperCAmelCase_   ,       **UpperCAmelCase_      )
  																									__lowercase=    [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]
  
  																									__lowercase=    tokenizer_r.batch_encode_candidates(UpperCAmelCase_   ,       max_length=1_0   ,       return_tensors='np'      )
  																									__lowercase=    tokenizer_p.batch_encode_candidates(UpperCAmelCase_   ,       max_length=1_0   ,       return_tensors='np'      )
  
  																									__lowercase=    (2, 2, 1_0)
  
  																									self.assertEqual(encoded_sentence_r['input_ids'].shape   ,       UpperCAmelCase_      )
  																									self.assertEqual(encoded_sentence_r['attention_mask'].shape   ,       UpperCAmelCase_      )
  																									self.assertEqual(encoded_sentence_r['token_type_ids'].shape   ,       UpperCAmelCase_      )
  
  																									self.assertEqual(encoded_sentence_p['input_ids'].shape   ,       UpperCAmelCase_      )
  																									self.assertEqual(encoded_sentence_p['attention_mask'].shape   ,       UpperCAmelCase_      )
  																									self.assertEqual(encoded_sentence_p['token_type_ids'].shape   ,       UpperCAmelCase_      )
  
  
  
  
  
  
  
style_context: 
  
  
  from dataclasses import dataclass
  from typing import List, Optional, Union
  
  import numpy as np
  import PIL
  from PIL import Image
  
  from ...utils import BaseOutput, is_torch_available, is_transformers_available
  
  
  
  
  
  @dataclass
  # Copied from diffusers.pipelines.stable_diffusion.__init__.StableDiffusionPipelineOutput with Stable->Alt
  class    A					(     A_				):
  							UpperCamelCase_     :							Union[List[PIL.Image.Image], np.ndarray]
  							UpperCamelCase_     :							Optional[List[bool]]
  
  
  if is_transformers_available() and is_torch_available():
  	from .modeling_roberta_series import RobertaSeriesModelWithTransformation
  	from .pipeline_alt_diffusion import AltDiffusionPipeline
  	from .pipeline_alt_diffusion_imgaimg import AltDiffusionImgaImgPipeline
  
  
  
  
  
  
  