





'''simple docstring'''







import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 				__SCREAMING_SNAKE_CASE  (snake_case_       ,							unittest.TestCase      ):






				"""simple docstring"""







				__a            =RoCBertTokenizer
				__a            =None
				__a            =False
				__a            =True
				__a            =filter_non_english




				def   UpperCamelCase__   (  self     :					Dict					):
								super().setUp()

								_a								= ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
								_a								= {}
								_a								= {}
								for i, value in enumerate(UpperCamelCase__					):
												_a								= i
												_a								= i
								_a								= os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES["vocab_file"]					)
								_a								= os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES["word_shape_file"]					)
								_a								= os.path.join(self.tmpdirname					,				VOCAB_FILES_NAMES["word_pronunciation_file"]					)
								with open(self.vocab_file					,				"w"					,				encoding="utf-8"					) as vocab_writer:
												vocab_writer.write("".join([x + "\n" for x in vocab_tokens]					)					)
								with open(self.word_shape_file					,				"w"					,				encoding="utf-8"					) as word_shape_writer:
												json.dump(UpperCamelCase__					,				UpperCamelCase__					,				ensure_ascii=UpperCamelCase__					)
								with open(self.word_pronunciation_file					,				"w"					,				encoding="utf-8"					) as word_pronunciation_writer:
												json.dump(UpperCamelCase__					,				UpperCamelCase__					,				ensure_ascii=UpperCamelCase__					)




				def   UpperCamelCase__   (  self     :					str					):
								_a								= self.tokenizer_class(self.vocab_file					,				self.word_shape_file					,				self.word_pronunciation_file					)

								_a								= tokenizer.tokenize("你好[SEP]你是谁"					)
								self.assertListEqual(UpperCamelCase__					,				["你", "好", "[SEP]", "你", "是", "谁"]					)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__					)					,				[5, 6, 2, 5, 7, 8]					)
								self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__					)					,				[5, 6, 2, 5, 7, 8]					)
								self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__					)					,				[5, 6, 2, 5, 7, 8]					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					):
								_a								= RoCBertBasicTokenizer()

								self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"					)					,				["ah", "\u535A", "\u63A8", "zz"]					)




				def   UpperCamelCase__   (  self     :					Any					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)					,				["hello", "!", "how", "are", "you", "?"]					)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)					,				["hello"]					)




				def   UpperCamelCase__   (  self     :					List[str]					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					,				strip_accents=UpperCamelCase__					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)					,				["hällo", "!", "how", "are", "you", "?"]					)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)					,				["h\u00E9llo"]					)




				def   UpperCamelCase__   (  self     :					Tuple					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					,				strip_accents=UpperCamelCase__					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)					,				["hallo", "!", "how", "are", "you", "?"]					)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)					,				["hello"]					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)					,				["hallo", "!", "how", "are", "you", "?"]					)
								self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)					,				["hello"]					)




				def   UpperCamelCase__   (  self     :					int					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)					,				["HeLLo", "!", "how", "Are", "yoU", "?"]					)




				def   UpperCamelCase__   (  self     :					Union[str, Any]					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					,				strip_accents=UpperCamelCase__					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)					,				["HäLLo", "!", "how", "Are", "yoU", "?"]					)




				def   UpperCamelCase__   (  self     :					Dict					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					,				strip_accents=UpperCamelCase__					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)					,				["HaLLo", "!", "how", "Are", "yoU", "?"]					)




				def   UpperCamelCase__   (  self     :					Optional[int]					):
								_a								= RoCBertBasicTokenizer(do_lower_case=UpperCamelCase__					,				never_split=["[UNK]"]					)

								self.assertListEqual(
								    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"					)					,				["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]					)




				def   UpperCamelCase__   (  self     :					List[str]					):
								_a								= ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

								_a								= {}
								for i, token in enumerate(UpperCamelCase__					):
												_a								= i
								_a								= RoCBertWordpieceTokenizer(vocab=UpperCamelCase__					,				unk_token="[UNK]"					)

								self.assertListEqual(tokenizer.tokenize(""					)					,				[]					)

								self.assertListEqual(tokenizer.tokenize("unwanted running"					)					,				["un", "##want", "##ed", "runn", "##ing"]					)

								self.assertListEqual(tokenizer.tokenize("unwantedX running"					)					,				["[UNK]", "runn", "##ing"]					)




				def   UpperCamelCase__   (  self     :					Any					):
								self.assertTrue(_is_whitespace(" "					)					)
								self.assertTrue(_is_whitespace("\t"					)					)
								self.assertTrue(_is_whitespace("\r"					)					)
								self.assertTrue(_is_whitespace("\n"					)					)
								self.assertTrue(_is_whitespace("\u00A0"					)					)

								self.assertFalse(_is_whitespace("A"					)					)
								self.assertFalse(_is_whitespace("-"					)					)




				def   UpperCamelCase__   (  self     :					Any					):
								self.assertTrue(_is_control("\u0005"					)					)

								self.assertFalse(_is_control("A"					)					)
								self.assertFalse(_is_control(" "					)					)
								self.assertFalse(_is_control("\t"					)					)
								self.assertFalse(_is_control("\r"					)					)




				def   UpperCamelCase__   (  self     :					Optional[Any]					):
								self.assertTrue(_is_punctuation("-"					)					)
								self.assertTrue(_is_punctuation("$"					)					)
								self.assertTrue(_is_punctuation("`"					)					)
								self.assertTrue(_is_punctuation("."					)					)

								self.assertFalse(_is_punctuation("A"					)					)
								self.assertFalse(_is_punctuation(" "					)					)




				def   UpperCamelCase__   (  self     :					Any					):
								_a								= self.get_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(UpperCamelCase__					) for t in ["Test", "\xad", "test"]]					,				[["[UNK]"], [], ["[UNK]"]]					)

								if self.test_rust_tokenizer:
												_a								= self.get_rust_tokenizer()
												self.assertListEqual(
												    [rust_tokenizer.tokenize(UpperCamelCase__					) for t in ["Test", "\xad", "test"]]					,				[["[UNK]"], [], ["[UNK]"]]					)




				def   UpperCamelCase__   (  self     :					List[str]					):




								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
																_a								= self.rust_tokenizer_class.from_pretrained(UpperCamelCase__					,				**UpperCamelCase__					)

																_a								= f'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																_a								= tokenizer_r.encode_plus(
																    UpperCamelCase__					,				return_attention_mask=UpperCamelCase__					,				return_token_type_ids=UpperCamelCase__					,				return_offsets_mapping=UpperCamelCase__					,				add_special_tokens=UpperCamelCase__					,				)

																_a								= tokenizer_r.do_lower_case if hasattr(UpperCamelCase__					,				"do_lower_case"					) else False
																_a								= (
																    [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "A"),
																        ((1, 2), ","),
																        ((3, 5), "na"),
																        ((5, 6), "##ï"),
																        ((6, 8), "##ve"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "Allen"),
																        ((21, 23), "##NL"),
																        ((23, 24), "##P"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																    if not do_lower_case
																    else [
																        ((0, 0), tokenizer_r.cls_token),
																        ((0, 1), "a"),
																        ((1, 2), ","),
																        ((3, 8), "naive"),
																        ((9, 15), tokenizer_r.mask_token),
																        ((16, 21), "allen"),
																        ((21, 23), "##nl"),
																        ((23, 24), "##p"),
																        ((25, 33), "sentence"),
																        ((33, 34), "."),
																        ((0, 0), tokenizer_r.sep_token),
																    ]
																)

																self.assertEqual(
																    [e[1] for e in expected_results]					,				tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]					)					)
																self.assertEqual([e[0] for e in expected_results]					,				tokens["offset_mapping"]					)




				def   UpperCamelCase__   (  self     :					List[str]					):
								_a								= ["的", "人", "有"]
								_a								= "".join(UpperCamelCase__					)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'					):
																_a								= True
																_a								= self.tokenizer_class.from_pretrained(UpperCamelCase__					,				**UpperCamelCase__					)
																_a								= self.rust_tokenizer_class.from_pretrained(UpperCamelCase__					,				**UpperCamelCase__					)

																_a								= tokenizer_p.encode(UpperCamelCase__					,				add_special_tokens=UpperCamelCase__					)
																_a								= tokenizer_r.encode(UpperCamelCase__					,				add_special_tokens=UpperCamelCase__					)

																_a								= tokenizer_r.convert_ids_to_tokens(UpperCamelCase__					)
																_a								= tokenizer_p.convert_ids_to_tokens(UpperCamelCase__					)

																# it is expected that each Chinese character is not preceded by "##"
																self.assertListEqual(UpperCamelCase__					,				UpperCamelCase__					)
																self.assertListEqual(UpperCamelCase__					,				UpperCamelCase__					)

																_a								= False
																_a								= self.rust_tokenizer_class.from_pretrained(UpperCamelCase__					,				**UpperCamelCase__					)
																_a								= self.tokenizer_class.from_pretrained(UpperCamelCase__					,				**UpperCamelCase__					)

																_a								= tokenizer_r.encode(UpperCamelCase__					,				add_special_tokens=UpperCamelCase__					)
																_a								= tokenizer_p.encode(UpperCamelCase__					,				add_special_tokens=UpperCamelCase__					)

																_a								= tokenizer_r.convert_ids_to_tokens(UpperCamelCase__					)
																_a								= tokenizer_p.convert_ids_to_tokens(UpperCamelCase__					)

																# it is expected that only the first Chinese character is not preceded by "##".
																_a								= [
																    f'##{token}' if idx != 0 else token for idx, token in enumerate(UpperCamelCase__					)
																]
																self.assertListEqual(UpperCamelCase__					,				UpperCamelCase__					)
																self.assertListEqual(UpperCamelCase__					,				UpperCamelCase__					)




				@slow
				def   UpperCamelCase__   (  self     :					List[str]					):
								_a								= self.tokenizer_class(self.vocab_file					,				self.word_shape_file					,				self.word_pronunciation_file					)

								_a								= tokenizer.encode("你好"					,				add_special_tokens=UpperCamelCase__					)
								_a								= tokenizer.encode("你是谁"					,				add_special_tokens=UpperCamelCase__					)

								_a								= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__					)
								_a								= tokenizer.build_inputs_with_special_tokens(UpperCamelCase__					,				UpperCamelCase__					)

								assert encoded_sentence == [1] + text + [2]
								assert encoded_pair == [1] + text + [2] + text_a + [2]




				def   UpperCamelCase__   (  self     :					str					):
								_a								= self.get_tokenizers(do_lower_case=UpperCamelCase__					)
								for tokenizer in tokenizers:
												with self.subTest(f'{tokenizer.__class__.__name__}'					):
																_a								= "你好，你是谁"
																_a								= tokenizer.tokenize(UpperCamelCase__					)
																_a								= tokenizer.convert_tokens_to_ids(UpperCamelCase__					)
																_a								= tokenizer.convert_tokens_to_shape_ids(UpperCamelCase__					)
																_a								= tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase__					)
																_a								= tokenizer.prepare_for_model(
																    UpperCamelCase__					,				UpperCamelCase__					,				UpperCamelCase__					,				add_special_tokens=UpperCamelCase__					)

																_a								= tokenizer.encode_plus(UpperCamelCase__					,				add_special_tokens=UpperCamelCase__					)

																self.assertEqual(UpperCamelCase__					,				UpperCamelCase__					)




"""simple docstring"""



def   __lowerCAmelCase						(						lowercase  :  int )	->						bool:


    """simple docstring"""





    if p < 2:
        raise ValueError("p should not be less than 2!" )
    elif p == 2:
        return True

    snake_case    :      Dict      					=    4
    snake_case    :      str      					=    (1 << p) - 1
    for _ in range(p - 2 ):
        snake_case    :      Any      					=    ((s * s) - 2) % m
    return s == 0


if __name__ == "__main__":
  print(lucas_lehmer_test(7))
  print(lucas_lehmer_test(11))





