




'''simple docstring'''





import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english

@require_tokenizers
class      _a       (		__lowerCAmelCase      ,      unittest.TestCase							):
					SCREAMING_SNAKE_CASE_							:					Tuple					       =		RealmTokenizer
					SCREAMING_SNAKE_CASE_							:					List[Any]					       =		RealmTokenizerFast
					SCREAMING_SNAKE_CASE_							:					Optional[int]					       =		True
					SCREAMING_SNAKE_CASE_							:					str					       =		True
					SCREAMING_SNAKE_CASE_							:					List[str]					       =		filter_non_english






					def        _lowercase			(				self					)							->    Any:
										super().setUp()

										_snake_case							=   [
										    "[UNK]",
										    "[CLS]",
										    "[SEP]",
										    "[PAD]",
										    "[MASK]",
										    "want",
										    "##want",
										    "##ed",
										    "wa",
										    "un",
										    "runn",
										    "##ing",
										    ",",
										    "low",
										    "lowest",
										]
										_snake_case							=   os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES["vocab_file"]					)
										with open(self.vocab_file      ,"w"      ,encoding="utf-8"					) as vocab_writer:
															vocab_writer.write("".join([x + "\n" for x in vocab_tokens]					)					)






					def        _lowercase			(				self      ,_SCREAMING_SNAKE_CASE					)							->    List[str]:
										_snake_case							=   "UNwant\u00E9d,running"
										_snake_case							=   "unwanted, running"
										return input_text, output_text






					def        _lowercase			(				self					)							->    Optional[int]:
										_snake_case							=   self.tokenizer_class(self.vocab_file					)

										_snake_case							=   tokenizer.tokenize("UNwant\u00E9d,running"					)
										self.assertListEqual(A_      ,["un", "##want", "##ed", ",", "runn", "##ing"]					)
										self.assertListEqual(tokenizer.convert_tokens_to_ids(A_					)      ,[9, 6, 7, 12, 10, 11]					)






					def        _lowercase			(				self					)							->    str:
										if not self.test_rust_tokenizer:
															return

										_snake_case							=   self.get_tokenizer()
										_snake_case							=   self.get_rust_tokenizer()

										_snake_case							=   "UNwant\u00E9d,running"

										_snake_case							=   tokenizer.tokenize(A_					)
										_snake_case							=   rust_tokenizer.tokenize(A_					)
										self.assertListEqual(A_      ,A_					)

										_snake_case							=   tokenizer.encode(A_      ,add_special_tokens=A_					)
										_snake_case							=   rust_tokenizer.encode(A_      ,add_special_tokens=A_					)
										self.assertListEqual(A_      ,A_					)

										_snake_case							=   self.get_rust_tokenizer()
										_snake_case							=   tokenizer.encode(A_					)
										_snake_case							=   rust_tokenizer.encode(A_					)
										self.assertListEqual(A_      ,A_					)

										# With lower casing
										_snake_case							=   self.get_tokenizer(do_lower_case=A_					)
										_snake_case							=   self.get_rust_tokenizer(do_lower_case=A_					)

										_snake_case							=   "UNwant\u00E9d,running"

										_snake_case							=   tokenizer.tokenize(A_					)
										_snake_case							=   rust_tokenizer.tokenize(A_					)
										self.assertListEqual(A_      ,A_					)

										_snake_case							=   tokenizer.encode(A_      ,add_special_tokens=A_					)
										_snake_case							=   rust_tokenizer.encode(A_      ,add_special_tokens=A_					)
										self.assertListEqual(A_      ,A_					)

										_snake_case							=   self.get_rust_tokenizer()
										_snake_case							=   tokenizer.encode(A_					)
										_snake_case							=   rust_tokenizer.encode(A_					)
										self.assertListEqual(A_      ,A_					)






					def        _lowercase			(				self					)							->    Tuple:
										_snake_case							=   BasicTokenizer()

										self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"					)      ,["ah", "\u535A", "\u63A8", "zz"]					)






					def        _lowercase			(				self					)							->    Union[str, Any]:
										_snake_case							=   BasicTokenizer(do_lower_case=A_					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)      ,["hello", "!", "how", "are", "you", "?"]					)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,["hello"]					)






					def        _lowercase			(				self					)							->    int:
										_snake_case							=   BasicTokenizer(do_lower_case=A_      ,strip_accents=A_					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,["hällo", "!", "how", "are", "you", "?"]					)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,["h\u00E9llo"]					)






					def        _lowercase			(				self					)							->    List[Any]:
										_snake_case							=   BasicTokenizer(do_lower_case=A_      ,strip_accents=A_					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,["hallo", "!", "how", "are", "you", "?"]					)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,["hello"]					)






					def        _lowercase			(				self					)							->    Tuple:
										_snake_case							=   BasicTokenizer(do_lower_case=A_					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,["hallo", "!", "how", "are", "you", "?"]					)
										self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)      ,["hello"]					)






					def        _lowercase			(				self					)							->    Optional[int]:
										_snake_case							=   BasicTokenizer(do_lower_case=A_					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)      ,["HeLLo", "!", "how", "Are", "yoU", "?"]					)






					def        _lowercase			(				self					)							->    Any:
										_snake_case							=   BasicTokenizer(do_lower_case=A_      ,strip_accents=A_					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,["HäLLo", "!", "how", "Are", "yoU", "?"]					)






					def        _lowercase			(				self					)							->    Optional[int]:
										_snake_case							=   BasicTokenizer(do_lower_case=A_      ,strip_accents=A_					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)      ,["HaLLo", "!", "how", "Are", "yoU", "?"]					)






					def        _lowercase			(				self					)							->    Any:
										_snake_case							=   BasicTokenizer(do_lower_case=A_      ,never_split=["[UNK]"]					)

										self.assertListEqual(
										    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"					)      ,["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]					)






					def        _lowercase			(				self					)							->    Tuple:
										_snake_case							=   ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

										_snake_case							=   {}
										for i, token in enumerate(A_					):
															_snake_case							=   i
										_snake_case							=   WordpieceTokenizer(vocab=A_      ,unk_token="[UNK]"					)

										self.assertListEqual(tokenizer.tokenize(""					)      ,[]					)

										self.assertListEqual(tokenizer.tokenize("unwanted running"					)      ,["un", "##want", "##ed", "runn", "##ing"]					)

										self.assertListEqual(tokenizer.tokenize("unwantedX running"					)      ,["[UNK]", "runn", "##ing"]					)






					def        _lowercase			(				self					)							->    int:
										self.assertTrue(_is_whitespace(" "					)					)
										self.assertTrue(_is_whitespace("\t"					)					)
										self.assertTrue(_is_whitespace("\r"					)					)
										self.assertTrue(_is_whitespace("\n"					)					)
										self.assertTrue(_is_whitespace("\u00A0"					)					)

										self.assertFalse(_is_whitespace("A"					)					)
										self.assertFalse(_is_whitespace("-"					)					)






					def        _lowercase			(				self					)							->    str:
										self.assertTrue(_is_control("\u0005"					)					)

										self.assertFalse(_is_control("A"					)					)
										self.assertFalse(_is_control(" "					)					)
										self.assertFalse(_is_control("\t"					)					)
										self.assertFalse(_is_control("\r"					)					)






					def        _lowercase			(				self					)							->    List[str]:
										self.assertTrue(_is_punctuation("-"					)					)
										self.assertTrue(_is_punctuation("$"					)					)
										self.assertTrue(_is_punctuation("`"					)					)
										self.assertTrue(_is_punctuation("."					)					)

										self.assertFalse(_is_punctuation("A"					)					)
										self.assertFalse(_is_punctuation(" "					)					)






					def        _lowercase			(				self					)							->    List[Any]:
										_snake_case							=   self.get_tokenizer()

										# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
										self.assertListEqual([tokenizer.tokenize(A_					) for t in ["Test", "\xad", "test"]]      ,[["[UNK]"], [], ["[UNK]"]]					)

										if self.test_rust_tokenizer:
															_snake_case							=   self.get_rust_tokenizer()
															self.assertListEqual(
															    [rust_tokenizer.tokenize(A_					) for t in ["Test", "\xad", "test"]]      ,[["[UNK]"], [], ["[UNK]"]]					)






					@slow
					def        _lowercase			(				self					)							->    List[Any]:
										_snake_case							=   self.tokenizer_class.from_pretrained("bert-base-uncased"					)

										_snake_case							=   tokenizer.encode("sequence builders"      ,add_special_tokens=A_					)
										_snake_case							=   tokenizer.encode("multi-sequence build"      ,add_special_tokens=A_					)

										_snake_case							=   tokenizer.build_inputs_with_special_tokens(A_					)
										_snake_case							=   tokenizer.build_inputs_with_special_tokens(A_      ,A_					)

										assert encoded_sentence == [101] + text + [102]
										assert encoded_pair == [101] + text + [102] + text_a + [102]






					def        _lowercase			(				self					)							->    int:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
																				_snake_case							=   self.rust_tokenizer_class.from_pretrained(A_      ,**A_					)

																				_snake_case							=   f"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																				_snake_case							=   tokenizer_r.encode_plus(
																				    A_      ,return_attention_mask=A_      ,return_token_type_ids=A_      ,return_offsets_mapping=A_      ,add_special_tokens=A_      ,)

																				_snake_case							=   tokenizer_r.do_lower_case if hasattr(A_      ,"do_lower_case"					) else False
																				_snake_case							=   (
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), "A"),
																				        ((1, 2), ","),
																				        ((3, 5), "na"),
																				        ((5, 6), "##ï"),
																				        ((6, 8), "##ve"),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), "Allen"),
																				        ((21, 23), "##NL"),
																				        ((23, 24), "##P"),
																				        ((25, 33), "sentence"),
																				        ((33, 34), "."),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), "a"),
																				        ((1, 2), ","),
																				        ((3, 8), "naive"),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), "allen"),
																				        ((21, 23), "##nl"),
																				        ((23, 24), "##p"),
																				        ((25, 33), "sentence"),
																				        ((33, 34), "."),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]      ,tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]					)					)
																				self.assertEqual([e[0] for e in expected_results]      ,tokens["offset_mapping"]					)






					@slow
					def        _lowercase			(				self					)							->    Dict:
										for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""					):
																				_snake_case							=   self.rust_tokenizer_class.from_pretrained(A_      ,**A_					)
																				_snake_case							=   self.tokenizer_class.from_pretrained(A_      ,**A_					)
																				_snake_case							=   [["Hello world!", "Nice to meet you!"], ["The cute cat.", "The adorable dog."]]

																				_snake_case							=   tokenizer_r.batch_encode_candidates(A_      ,max_length=10      ,return_tensors="np"					)
																				_snake_case							=   tokenizer_p.batch_encode_candidates(A_      ,max_length=10      ,return_tensors="np"					)

																				_snake_case							=   (2, 2, 10)

																				self.assertEqual(encoded_sentence_r["input_ids"].shape      ,A_					)
																				self.assertEqual(encoded_sentence_r["attention_mask"].shape      ,A_					)
																				self.assertEqual(encoded_sentence_r["token_type_ids"].shape      ,A_					)

																				self.assertEqual(encoded_sentence_p["input_ids"].shape      ,A_					)
																				self.assertEqual(encoded_sentence_p["attention_mask"].shape      ,A_					)
																				self.assertEqual(encoded_sentence_p["token_type_ids"].shape      ,A_					)








'''simple docstring'''





UpperCamelCase_		:    Optional[Any]									=		'''
# Diffusers installation
! pip install diffusers transformers datasets accelerate
# To install from source instead of the last release, comment the command above and uncomment the following one.
# ! pip install git+https://github.com/huggingface/diffusers.git
'''

UpperCamelCase_		:    List[Any]									=		[{'''type''': '''code''', '''content''': INSTALL_CONTENT}]


