


import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class    A					(     A_      ,       unittest.TestCase				):
							UpperCamelCase_     :							Tuple =RealmTokenizer
							UpperCamelCase_     :							Optional[int] =RealmTokenizerFast
							UpperCamelCase_     :							int =True
							UpperCamelCase_     :							Tuple =True
							UpperCamelCase_     :							Optional[int] =filter_non_english


							def     _A (self      ):
													super().setUp()

													__lowercase=    [
													    '[UNK]',
													    '[CLS]',
													    '[SEP]',
													    '[PAD]',
													    '[MASK]',
													    'want',
													    '##want',
													    '##ed',
													    'wa',
													    'un',
													    'runn',
													    '##ing',
													    ',',
													    'low',
													    'lowest',
													]
													__lowercase=    os.path.join(self.tmpdirname   ,       VOCAB_FILES_NAMES['vocab_file']      )
													with open(self.vocab_file   ,       'w'   ,       encoding='utf-8'      ) as vocab_writer:
																			vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )


							def     _A (self   ,       lowerCAmelCase      ):
													__lowercase=    'UNwant\u00E9d,running'
													__lowercase=    'unwanted, running'
													return input_text, output_text


							def     _A (self      ):
													__lowercase=    self.tokenizer_class(self.vocab_file      )

													__lowercase=    tokenizer.tokenize('UNwant\u00E9d,running'      )
													self.assertListEqual(UpperCAmelCase_   ,       ['un', '##want', '##ed', ',', 'runn', '##ing']      )
													self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )   ,       [9, 6, 7, 1_2, 1_0, 1_1]      )


							def     _A (self      ):
													if not self.test_rust_tokenizer:
																			return

													__lowercase=    self.get_tokenizer()
													__lowercase=    self.get_rust_tokenizer()

													__lowercase=    'UNwant\u00E9d,running'

													__lowercase=    tokenizer.tokenize(UpperCAmelCase_      )
													__lowercase=    rust_tokenizer.tokenize(UpperCAmelCase_      )
													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )

													__lowercase=    tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )

													__lowercase=    self.get_rust_tokenizer()
													__lowercase=    tokenizer.encode(UpperCAmelCase_      )
													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_      )
													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )

													# With lower casing
													__lowercase=    self.get_tokenizer(do_lower_case=UpperCAmelCase_      )
													__lowercase=    self.get_rust_tokenizer(do_lower_case=UpperCAmelCase_      )

													__lowercase=    'UNwant\u00E9d,running'

													__lowercase=    tokenizer.tokenize(UpperCAmelCase_      )
													__lowercase=    rust_tokenizer.tokenize(UpperCAmelCase_      )
													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )

													__lowercase=    tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_      )
													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )

													__lowercase=    self.get_rust_tokenizer()
													__lowercase=    tokenizer.encode(UpperCAmelCase_      )
													__lowercase=    rust_tokenizer.encode(UpperCAmelCase_      )
													self.assertListEqual(UpperCAmelCase_   ,       UpperCAmelCase_      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer()

													self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'      )   ,       ['ah', '\u535A', '\u63A8', 'zz']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )   ,       ['hello', '!', 'how', 'are', 'you', '?']      )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['hello']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['hällo', '!', 'how', 'are', 'you', '?']      )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['h\u00E9llo']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['hallo', '!', 'how', 'are', 'you', '?']      )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['hello']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['hallo', '!', 'how', 'are', 'you', '?']      )
													self.assertListEqual(tokenizer.tokenize('H\u00E9llo'      )   ,       ['hello']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '      )   ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['HäLLo', '!', 'how', 'Are', 'yoU', '?']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       strip_accents=UpperCAmelCase_      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '      )   ,       ['HaLLo', '!', 'how', 'Are', 'yoU', '?']      )


							def     _A (self      ):
													__lowercase=    BasicTokenizer(do_lower_case=UpperCAmelCase_   ,       never_split=['[UNK]']      )

													self.assertListEqual(
													    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'      )   ,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']      )


							def     _A (self      ):
													__lowercase=    ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

													__lowercase=    {}
													for i, token in enumerate(UpperCAmelCase_      ):
																			__lowercase=    i
													__lowercase=    WordpieceTokenizer(vocab=UpperCAmelCase_   ,       unk_token='[UNK]'      )

													self.assertListEqual(tokenizer.tokenize(''      )   ,       []      )

													self.assertListEqual(tokenizer.tokenize('unwanted running'      )   ,       ['un', '##want', '##ed', 'runn', '##ing']      )

													self.assertListEqual(tokenizer.tokenize('unwantedX running'      )   ,       ['[UNK]', 'runn', '##ing']      )


							def     _A (self      ):
													self.assertTrue(_is_whitespace(' '      )      )
													self.assertTrue(_is_whitespace('\t'      )      )
													self.assertTrue(_is_whitespace('\r'      )      )
													self.assertTrue(_is_whitespace('\n'      )      )
													self.assertTrue(_is_whitespace('\u00A0'      )      )

													self.assertFalse(_is_whitespace('A'      )      )
													self.assertFalse(_is_whitespace('-'      )      )


							def     _A (self      ):
													self.assertTrue(_is_control('\u0005'      )      )

													self.assertFalse(_is_control('A'      )      )
													self.assertFalse(_is_control(' '      )      )
													self.assertFalse(_is_control('\t'      )      )
													self.assertFalse(_is_control('\r'      )      )


							def     _A (self      ):
													self.assertTrue(_is_punctuation('-'      )      )
													self.assertTrue(_is_punctuation('$'      )      )
													self.assertTrue(_is_punctuation('`'      )      )
													self.assertTrue(_is_punctuation('.'      )      )

													self.assertFalse(_is_punctuation('A'      )      )
													self.assertFalse(_is_punctuation(' '      )      )


							def     _A (self      ):
													__lowercase=    self.get_tokenizer()

													# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
													self.assertListEqual([tokenizer.tokenize(UpperCAmelCase_      ) for t in ['Test', '\xad', 'test']]   ,       [['[UNK]'], [], ['[UNK]']]      )

													if self.test_rust_tokenizer:
																			__lowercase=    self.get_rust_tokenizer()
																			self.assertListEqual(
																			    [rust_tokenizer.tokenize(UpperCAmelCase_      ) for t in ['Test', '\xad', 'test']]   ,       [['[UNK]'], [], ['[UNK]']]      )


							@slow
							def     _A (self      ):
													__lowercase=    self.tokenizer_class.from_pretrained('bert-base-uncased'      )

													__lowercase=    tokenizer.encode('sequence builders'   ,       add_special_tokens=UpperCAmelCase_      )
													__lowercase=    tokenizer.encode('multi-sequence build'   ,       add_special_tokens=UpperCAmelCase_      )

													__lowercase=    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_      )
													__lowercase=    tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_   ,       UpperCAmelCase_      )

													assert encoded_sentence == [1_0_1] + text + [1_0_2]
													assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]


							def     _A (self      ):
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																									__lowercase=    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_   ,       **UpperCAmelCase_      )

																									__lowercase=    f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																									__lowercase=    tokenizer_r.encode_plus(
																									    UpperCAmelCase_   ,       return_attention_mask=UpperCAmelCase_   ,       return_token_type_ids=UpperCAmelCase_   ,       return_offsets_mapping=UpperCAmelCase_   ,       add_special_tokens=UpperCAmelCase_   ,       )

																									__lowercase=    tokenizer_r.do_lower_case if hasattr(UpperCAmelCase_   ,       'do_lower_case'      ) else False
																									__lowercase=    (
																									    [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), 'A'),
																									        ((1, 2), ','),
																									        ((3, 5), 'na'),
																									        ((5, 6), '##ï'),
																									        ((6, 8), '##ve'),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), 'Allen'),
																									        ((2_1, 2_3), '##NL'),
																									        ((2_3, 2_4), '##P'),
																									        ((2_5, 3_3), 'sentence'),
																									        ((3_3, 3_4), '.'),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									    if not do_lower_case
																									    else [
																									        ((0, 0), tokenizer_r.cls_token),
																									        ((0, 1), 'a'),
																									        ((1, 2), ','),
																									        ((3, 8), 'naive'),
																									        ((9, 1_5), tokenizer_r.mask_token),
																									        ((1_6, 2_1), 'allen'),
																									        ((2_1, 2_3), '##nl'),
																									        ((2_3, 2_4), '##p'),
																									        ((2_5, 3_3), 'sentence'),
																									        ((3_3, 3_4), '.'),
																									        ((0, 0), tokenizer_r.sep_token),
																									    ]
																									)

																									self.assertEqual(
																									    [e[1] for e in expected_results]   ,       tokenizer_r.convert_ids_to_tokens(tokens['input_ids']      )      )
																									self.assertEqual([e[0] for e in expected_results]   ,       tokens['offset_mapping']      )







							@slow
							def     _A (self      ):
													for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																			with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
																									__lowercase=    self.rust_tokenizer_class.from_pretrained(UpperCAmelCase_   ,       **UpperCAmelCase_      )
																									__lowercase=    self.tokenizer_class.from_pretrained(UpperCAmelCase_   ,       **UpperCAmelCase_      )
																									__lowercase=    [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																									__lowercase=    tokenizer_r.batch_encode_candidates(UpperCAmelCase_   ,       max_length=1_0   ,       return_tensors='np'      )
																									__lowercase=    tokenizer_p.batch_encode_candidates(UpperCAmelCase_   ,       max_length=1_0   ,       return_tensors='np'      )

																									__lowercase=    (2, 2, 1_0)

																									self.assertEqual(encoded_sentence_r['input_ids'].shape   ,       UpperCAmelCase_      )
																									self.assertEqual(encoded_sentence_r['attention_mask'].shape   ,       UpperCAmelCase_      )
																									self.assertEqual(encoded_sentence_r['token_type_ids'].shape   ,       UpperCAmelCase_      )

																									self.assertEqual(encoded_sentence_p['input_ids'].shape   ,       UpperCAmelCase_      )
																									self.assertEqual(encoded_sentence_p['attention_mask'].shape   ,       UpperCAmelCase_      )
																									self.assertEqual(encoded_sentence_p['token_type_ids'].shape   ,       UpperCAmelCase_      )










from dataclasses import dataclass
from typing import List, Optional, Union

import numpy as np
import PIL
from PIL import Image

from ...utils import BaseOutput, is_torch_available, is_transformers_available





@dataclass
# Copied from diffusers.pipelines.stable_diffusion.__init__.StableDiffusionPipelineOutput with Stable->Alt
class    A					(     A_				):
							UpperCamelCase_     :							Union[List[PIL.Image.Image], np.ndarray]
							UpperCamelCase_     :							Optional[List[bool]]


if is_transformers_available() and is_torch_available():
	from .modeling_roberta_series import RobertaSeriesModelWithTransformation
	from .pipeline_alt_diffusion import AltDiffusionPipeline
	from .pipeline_alt_diffusion_imgaimg import AltDiffusionImgaImgPipeline






