



import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class    _lowercase (a_   , unittest.TestCase   ):
				'''simple docstring'''







				lowercase__     =    RealmTokenizer
				lowercase__     =    RealmTokenizerFast
				lowercase__     =    True
				lowercase__     =    True
				lowercase__     =    filter_non_english

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									super().setUp()

									UpperCamelCase_			= [
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "[PAD]",
									    "[MASK]",
									    "want",
									    "##want",
									    "##ed",
									    "wa",
									    "un",
									    "runn",
									    "##ing",
									    ",",
									    "low",
									    "lowest",
									]
									UpperCamelCase_			= os.path.join(self.tmpdirname  ,			VOCAB_FILES_NAMES["vocab_file"]     )
									with open(self.vocab_file  ,			"w"  ,			encoding="utf-8"     ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]     )     )

				def    _lowerCamelCase       (			self  ,			snake_case__     ):

									'''simple docstring'''


									UpperCamelCase_			= "UNwant\u00E9d,running"
									UpperCamelCase_			= "unwanted, running"
									return input_text, output_text

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= self.tokenizer_class(self.vocab_file     )

									UpperCamelCase_			= tokenizer.tokenize("UNwant\u00E9d,running"     )
									self.assertListEqual(snake_case__  ,			["un", "##want", "##ed", ",", "runn", "##ing"]     )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__     )  ,			[9, 6, 7, 12, 10, 11]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									if not self.test_rust_tokenizer:
														return

									UpperCamelCase_			= self.get_tokenizer()
									UpperCamelCase_			= self.get_rust_tokenizer()

									UpperCamelCase_			= "UNwant\u00E9d,running"

									UpperCamelCase_			= tokenizer.tokenize(snake_case__     )
									UpperCamelCase_			= rust_tokenizer.tokenize(snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									UpperCamelCase_			= tokenizer.encode(snake_case__  ,			add_special_tokens=snake_case__     )
									UpperCamelCase_			= rust_tokenizer.encode(snake_case__  ,			add_special_tokens=snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									UpperCamelCase_			= self.get_rust_tokenizer()
									UpperCamelCase_			= tokenizer.encode(snake_case__     )
									UpperCamelCase_			= rust_tokenizer.encode(snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									# With lower casing
									UpperCamelCase_			= self.get_tokenizer(do_lower_case=snake_case__     )
									UpperCamelCase_			= self.get_rust_tokenizer(do_lower_case=snake_case__     )

									UpperCamelCase_			= "UNwant\u00E9d,running"

									UpperCamelCase_			= tokenizer.tokenize(snake_case__     )
									UpperCamelCase_			= rust_tokenizer.tokenize(snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									UpperCamelCase_			= tokenizer.encode(snake_case__  ,			add_special_tokens=snake_case__     )
									UpperCamelCase_			= rust_tokenizer.encode(snake_case__  ,			add_special_tokens=snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

									UpperCamelCase_			= self.get_rust_tokenizer()
									UpperCamelCase_			= tokenizer.encode(snake_case__     )
									UpperCamelCase_			= rust_tokenizer.encode(snake_case__     )
									self.assertListEqual(snake_case__  ,			snake_case__     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer()

									self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"     )  ,			["ah", "\u535A", "\u63A8", "zz"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "     )  ,			["hello", "!", "how", "are", "you", "?"]     )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     )  ,			["hello"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__  ,			strip_accents=snake_case__     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     )  ,			["hällo", "!", "how", "are", "you", "?"]     )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     )  ,			["h\u00E9llo"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__  ,			strip_accents=snake_case__     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     )  ,			["hallo", "!", "how", "are", "you", "?"]     )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     )  ,			["hello"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     )  ,			["hallo", "!", "how", "are", "you", "?"]     )
									self.assertListEqual(tokenizer.tokenize("H\u00E9llo"     )  ,			["hello"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "     )  ,			["HeLLo", "!", "how", "Are", "yoU", "?"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__  ,			strip_accents=snake_case__     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     )  ,			["HäLLo", "!", "how", "Are", "yoU", "?"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__  ,			strip_accents=snake_case__     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "     )  ,			["HaLLo", "!", "how", "Are", "yoU", "?"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= BasicTokenizer(do_lower_case=snake_case__  ,			never_split=["[UNK]"]     )

									self.assertListEqual(
									    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"     )  ,			["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

									UpperCamelCase_			= {}
									for i, token in enumerate(snake_case__     ):
														UpperCamelCase_			= i
									UpperCamelCase_			= WordpieceTokenizer(vocab=snake_case__  ,			unk_token="[UNK]"     )

									self.assertListEqual(tokenizer.tokenize(""     )  ,			[]     )

									self.assertListEqual(tokenizer.tokenize("unwanted running"     )  ,			["un", "##want", "##ed", "runn", "##ing"]     )

									self.assertListEqual(tokenizer.tokenize("unwantedX running"     )  ,			["[UNK]", "runn", "##ing"]     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									self.assertTrue(_is_whitespace(" "     )     )
									self.assertTrue(_is_whitespace("\t"     )     )
									self.assertTrue(_is_whitespace("\r"     )     )
									self.assertTrue(_is_whitespace("\n"     )     )
									self.assertTrue(_is_whitespace("\u00A0"     )     )

									self.assertFalse(_is_whitespace("A"     )     )
									self.assertFalse(_is_whitespace("-"     )     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									self.assertTrue(_is_control("\u0005"     )     )

									self.assertFalse(_is_control("A"     )     )
									self.assertFalse(_is_control(" "     )     )
									self.assertFalse(_is_control("\t"     )     )
									self.assertFalse(_is_control("\r"     )     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									self.assertTrue(_is_punctuation("-"     )     )
									self.assertTrue(_is_punctuation("$"     )     )
									self.assertTrue(_is_punctuation("`"     )     )
									self.assertTrue(_is_punctuation("."     )     )

									self.assertFalse(_is_punctuation("A"     )     )
									self.assertFalse(_is_punctuation(" "     )     )

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= self.get_tokenizer()

									# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
									self.assertListEqual([tokenizer.tokenize(snake_case__     ) for t in ["Test", "\xad", "test"]]  ,			[["[UNK]"], [], ["[UNK]"]]     )

									if self.test_rust_tokenizer:
														UpperCamelCase_			= self.get_rust_tokenizer()
														self.assertListEqual(
														    [rust_tokenizer.tokenize(snake_case__     ) for t in ["Test", "\xad", "test"]]  ,			[["[UNK]"], [], ["[UNK]"]]     )

				@slow
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									UpperCamelCase_			= self.tokenizer_class.from_pretrained("bert-base-uncased"     )

									UpperCamelCase_			= tokenizer.encode("sequence builders"  ,			add_special_tokens=snake_case__     )
									UpperCamelCase_			= tokenizer.encode("multi-sequence build"  ,			add_special_tokens=snake_case__     )

									UpperCamelCase_			= tokenizer.build_inputs_with_special_tokens(snake_case__     )
									UpperCamelCase_			= tokenizer.build_inputs_with_special_tokens(snake_case__  ,			snake_case__     )

									assert encoded_sentence == [101] + text + [102]
									assert encoded_pair == [101] + text + [102] + text_a + [102]

				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																			UpperCamelCase_			= self.rust_tokenizer_class.from_pretrained(snake_case__  ,			**snake_case__     )

																			UpperCamelCase_			= F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																			UpperCamelCase_			= tokenizer_r.encode_plus(
																			    snake_case__  ,			return_attention_mask=snake_case__  ,			return_token_type_ids=snake_case__  ,			return_offsets_mapping=snake_case__  ,			add_special_tokens=snake_case__  ,			)

																			UpperCamelCase_			= tokenizer_r.do_lower_case if hasattr(snake_case__  ,			"do_lower_case"     ) else False
																			UpperCamelCase_			= (
																			    [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "A"),
																			        ((1, 2), ","),
																			        ((3, 5), "na"),
																			        ((5, 6), "##ï"),
																			        ((6, 8), "##ve"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "Allen"),
																			        ((21, 23), "##NL"),
																			        ((23, 24), "##P"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			    if not do_lower_case
																			    else [
																			        ((0, 0), tokenizer_r.cls_token),
																			        ((0, 1), "a"),
																			        ((1, 2), ","),
																			        ((3, 8), "naive"),
																			        ((9, 15), tokenizer_r.mask_token),
																			        ((16, 21), "allen"),
																			        ((21, 23), "##nl"),
																			        ((23, 24), "##p"),
																			        ((25, 33), "sentence"),
																			        ((33, 34), "."),
																			        ((0, 0), tokenizer_r.sep_token),
																			    ]
																			)

																			self.assertEqual(
																			    [e[1] for e in expected_results]  ,			tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]     )     )
																			self.assertEqual([e[0] for e in expected_results]  ,			tokens["offset_mapping"]     )

				@slow
				def    _lowerCamelCase       (			self     ):

									'''simple docstring'''


									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""     ):
																			UpperCamelCase_			= self.rust_tokenizer_class.from_pretrained(snake_case__  ,			**snake_case__     )
																			UpperCamelCase_			= self.tokenizer_class.from_pretrained(snake_case__  ,			**snake_case__     )
																			UpperCamelCase_			= [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

																			UpperCamelCase_			= tokenizer_r.batch_encode_candidates(snake_case__  ,			max_length=10  ,			return_tensors="np"     )
																			UpperCamelCase_			= tokenizer_p.batch_encode_candidates(snake_case__  ,			max_length=10  ,			return_tensors="np"     )

																			UpperCamelCase_			= (2, 2, 10)

																			self.assertEqual(encoded_sentence_r["input_ids"].shape  ,			snake_case__     )
																			self.assertEqual(encoded_sentence_r["attention_mask"].shape  ,			snake_case__     )
																			self.assertEqual(encoded_sentence_r["token_type_ids"].shape  ,			snake_case__     )

																			self.assertEqual(encoded_sentence_p["input_ids"].shape  ,			snake_case__     )
																			self.assertEqual(encoded_sentence_p["attention_mask"].shape  ,			snake_case__     )
																			self.assertEqual(encoded_sentence_p["token_type_ids"].shape  ,			snake_case__     )





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_torch_available,
)


lowerCAmelCase					:				Optional[int]   	={
    'configuration_speech_to_text_2': ['SPEECH_TO_TEXT_2_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Speech2Text2Config'],
    'processing_speech_to_text_2': ['Speech2Text2Processor'],
    'tokenization_speech_to_text_2': ['Speech2Text2Tokenizer'],
}


try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCAmelCase					:				int   	=[
		    'SPEECH_TO_TEXT_2_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'Speech2Text2ForCausalLM',
		    'Speech2Text2PreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_speech_to_text_a import SPEECH_TO_TEXT_2_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextaConfig
		from .processing_speech_to_text_a import SpeechaTextaProcessor
		from .tokenization_speech_to_text_a import SpeechaTextaTokenizer

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_speech_to_text_a import (
				    SPEECH_TO_TEXT_2_PRETRAINED_MODEL_ARCHIVE_LIST,
				    SpeechaTextaForCausalLM,
				    SpeechaTextaPreTrainedModel,
				)

else:
		import sys

		lowerCAmelCase					:				Union[str, Any]   	=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

