




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 	_UpperCamelCase					( UpperCamelCase__		,				unittest.TestCase						):
      UpperCAmelCase_						=						BertTokenizer
      UpperCAmelCase_						=						BertTokenizerFast
      UpperCAmelCase_						=						True
      UpperCAmelCase_						=						True
      UpperCAmelCase_						=						filter_non_english
      def 		UpperCAmelCase_  (	self      :Tuple					)   ->	str:
       super().setUp()

       UpperCAmelCase__							       =  [
           "[UNK]",
           "[CLS]",
           "[SEP]",
           "[PAD]",
           "[MASK]",
           "want",
           "##want",
           "##ed",
           "wa",
           "un",
           "runn",
           "##ing",
           ",",
           "low",
           "lowest",
       ]
       UpperCAmelCase__							       =  os.path.join(self.tmpdirname       ,					VOCAB_FILES_NAMES["vocab_file"]					)
       with open(self.vocab_file       ,					"w"       ,					encoding="utf-8"					) as vocab_writer:
        vocab_writer.write("".join([x + "\n" for x in vocab_tokens]					)					)
      def 		UpperCAmelCase_  (	self      :Optional[Any]       ,					lowerCamelCase      :Optional[int]					)   ->	str:
       UpperCAmelCase__							       =  "UNwant\u00E9d,running"
       UpperCAmelCase__							       =  "unwanted, running"
       return input_text, output_text
      def 		UpperCAmelCase_  (	self      :Dict					)   ->	Tuple:
       UpperCAmelCase__							       =  self.tokenizer_class(self.vocab_file					)

       UpperCAmelCase__							       =  tokenizer.tokenize("UNwant\u00E9d,running"					)
       self.assertListEqual(__lowerCamelCase       ,					["un", "##want", "##ed", ",", "runn", "##ing"]					)
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase					)       ,					[9, 6, 7, 12, 10, 11]					)
      def 		UpperCAmelCase_  (	self      :Any					)   ->	Dict:
       if not self.test_rust_tokenizer:
        return

       UpperCAmelCase__							       =  self.get_tokenizer()
       UpperCAmelCase__							       =  self.get_rust_tokenizer()

       UpperCAmelCase__							       =  "UNwant\u00E9d,running"

       UpperCAmelCase__							       =  tokenizer.tokenize(__lowerCamelCase					)
       UpperCAmelCase__							       =  rust_tokenizer.tokenize(__lowerCamelCase					)
       self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)

       UpperCAmelCase__							       =  tokenizer.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)
       UpperCAmelCase__							       =  rust_tokenizer.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)
       self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)

       UpperCAmelCase__							       =  self.get_rust_tokenizer()
       UpperCAmelCase__							       =  tokenizer.encode(__lowerCamelCase					)
       UpperCAmelCase__							       =  rust_tokenizer.encode(__lowerCamelCase					)
       self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)

       # With lower casing
       UpperCAmelCase__							       =  self.get_tokenizer(do_lower_case=__lowerCamelCase					)
       UpperCAmelCase__							       =  self.get_rust_tokenizer(do_lower_case=__lowerCamelCase					)

       UpperCAmelCase__							       =  "UNwant\u00E9d,running"

       UpperCAmelCase__							       =  tokenizer.tokenize(__lowerCamelCase					)
       UpperCAmelCase__							       =  rust_tokenizer.tokenize(__lowerCamelCase					)
       self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)

       UpperCAmelCase__							       =  tokenizer.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)
       UpperCAmelCase__							       =  rust_tokenizer.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)
       self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)

       UpperCAmelCase__							       =  self.get_rust_tokenizer()
       UpperCAmelCase__							       =  tokenizer.encode(__lowerCamelCase					)
       UpperCAmelCase__							       =  rust_tokenizer.encode(__lowerCamelCase					)
       self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)
      def 		UpperCAmelCase_  (	self      :Optional[int]					)   ->	Union[str, Any]:
       UpperCAmelCase__							       =  BasicTokenizer()

       self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"					)       ,					["ah", "\u535A", "\u63A8", "zz"]					)
      def 		UpperCAmelCase_  (	self      :List[str]					)   ->	Dict:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)       ,					["hello", "!", "how", "are", "you", "?"]					)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)       ,					["hello"]					)
      def 		UpperCAmelCase_  (	self      :Union[str, Any]					)   ->	List[Any]:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase       ,					strip_accents=__lowerCamelCase					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)       ,					["hällo", "!", "how", "are", "you", "?"]					)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)       ,					["h\u00E9llo"]					)
      def 		UpperCAmelCase_  (	self      :Optional[int]					)   ->	int:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase       ,					strip_accents=__lowerCamelCase					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)       ,					["hallo", "!", "how", "are", "you", "?"]					)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)       ,					["hello"]					)
      def 		UpperCAmelCase_  (	self      :List[Any]					)   ->	List[Any]:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)       ,					["hallo", "!", "how", "are", "you", "?"]					)
       self.assertListEqual(tokenizer.tokenize("H\u00E9llo"					)       ,					["hello"]					)
      def 		UpperCAmelCase_  (	self      :Optional[int]					)   ->	Optional[Any]:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "					)       ,					["HeLLo", "!", "how", "Are", "yoU", "?"]					)
      def 		UpperCAmelCase_  (	self      :Optional[Any]					)   ->	List[str]:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase       ,					strip_accents=__lowerCamelCase					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)       ,					["HäLLo", "!", "how", "Are", "yoU", "?"]					)
      def 		UpperCAmelCase_  (	self      :List[str]					)   ->	List[Any]:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase       ,					strip_accents=__lowerCamelCase					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "					)       ,					["HaLLo", "!", "how", "Are", "yoU", "?"]					)
      def 		UpperCAmelCase_  (	self      :Optional[int]					)   ->	List[str]:
       UpperCAmelCase__							       =  BasicTokenizer(do_lower_case=__lowerCamelCase       ,					never_split=["[UNK]"]					)

       self.assertListEqual(
           tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"					)       ,					["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]					)
      def 		UpperCAmelCase_  (	self      :Optional[int]					)   ->	Optional[Any]:
       UpperCAmelCase__							       =  BasicTokenizer()
       UpperCAmelCase__							       =  "a\n\'ll !!to?\'d of, can\'t."
       UpperCAmelCase__							       =  ["a", "\'", "ll", "!", "!", "to", "?", "\'", "d", "of", ",", "can", "\'", "t", "."]
       self.assertListEqual(tokenizer.tokenize(__lowerCamelCase					)       ,					__lowerCamelCase					)
      def 		UpperCAmelCase_  (	self      :Optional[Any]					)   ->	Optional[Any]:
       UpperCAmelCase__							       =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

       UpperCAmelCase__							       =  {}
       for i, token in enumerate(__lowerCamelCase					):
        UpperCAmelCase__							       =  i
       UpperCAmelCase__							       =  WordpieceTokenizer(vocab=__lowerCamelCase       ,					unk_token="[UNK]"					)

       self.assertListEqual(tokenizer.tokenize(""					)       ,					[]					)

       self.assertListEqual(tokenizer.tokenize("unwanted running"					)       ,					["un", "##want", "##ed", "runn", "##ing"]					)

       self.assertListEqual(tokenizer.tokenize("unwantedX running"					)       ,					["[UNK]", "runn", "##ing"]					)
      def 		UpperCAmelCase_  (	self      :Dict					)   ->	str:
       self.assertTrue(_is_whitespace(" "					)					)
       self.assertTrue(_is_whitespace("\t"					)					)
       self.assertTrue(_is_whitespace("\r"					)					)
       self.assertTrue(_is_whitespace("\n"					)					)
       self.assertTrue(_is_whitespace("\u00A0"					)					)

       self.assertFalse(_is_whitespace("A"					)					)
       self.assertFalse(_is_whitespace("-"					)					)
      def 		UpperCAmelCase_  (	self      :List[Any]					)   ->	List[Any]:
       self.assertTrue(_is_control("\u0005"					)					)

       self.assertFalse(_is_control("A"					)					)
       self.assertFalse(_is_control(" "					)					)
       self.assertFalse(_is_control("\t"					)					)
       self.assertFalse(_is_control("\r"					)					)
      def 		UpperCAmelCase_  (	self      :int					)   ->	Dict:
       self.assertTrue(_is_punctuation("-"					)					)
       self.assertTrue(_is_punctuation("$"					)					)
       self.assertTrue(_is_punctuation("`"					)					)
       self.assertTrue(_is_punctuation("."					)					)

       self.assertFalse(_is_punctuation("A"					)					)
       self.assertFalse(_is_punctuation(" "					)					)
      def 		UpperCAmelCase_  (	self      :int					)   ->	Union[str, Any]:
       UpperCAmelCase__							       =  self.get_tokenizer()
       UpperCAmelCase__							       =  self.get_rust_tokenizer()

       # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
       self.assertListEqual([tokenizer.tokenize(__lowerCamelCase					) for t in ["Test", "\xad", "test"]]       ,					[["[UNK]"], [], ["[UNK]"]]					)

       self.assertListEqual(
           [rust_tokenizer.tokenize(__lowerCamelCase					) for t in ["Test", "\xad", "test"]]       ,					[["[UNK]"], [], ["[UNK]"]]					)
      @slow
      def 		UpperCAmelCase_  (	self      :Any					)   ->	Optional[Any]:
       UpperCAmelCase__							       =  self.tokenizer_class.from_pretrained("bert-base-uncased"					)

       UpperCAmelCase__							       =  tokenizer.encode("sequence builders"       ,					add_special_tokens=__lowerCamelCase					)
       UpperCAmelCase__							       =  tokenizer.encode("multi-sequence build"       ,					add_special_tokens=__lowerCamelCase					)

       UpperCAmelCase__							       =  tokenizer.build_inputs_with_special_tokens(__lowerCamelCase					)
       UpperCAmelCase__							       =  tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       ,					__lowerCamelCase					)

       assert encoded_sentence == [101] + text + [102]
       assert encoded_pair == [101] + text + [102] + text_a + [102]
      def 		UpperCAmelCase_  (	self      :List[str]					)   ->	Optional[Any]:
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
         UpperCAmelCase__							       =  self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,					**__lowerCamelCase					)

         UpperCAmelCase__							       =  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
         UpperCAmelCase__							       =  tokenizer_r.encode_plus(
             __lowerCamelCase       ,					return_attention_mask=__lowerCamelCase       ,					return_token_type_ids=__lowerCamelCase       ,					return_offsets_mapping=__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase       ,					)

         UpperCAmelCase__							       =  tokenizer_r.do_lower_case if hasattr(__lowerCamelCase       ,					"do_lower_case"					) else False
         UpperCAmelCase__							       =  (
             [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), "A"),
                 ((1, 2), ","),
                 ((3, 5), "na"),
                 ((5, 6), "##ï"),
                 ((6, 8), "##ve"),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), "Allen"),
                 ((21, 23), "##NL"),
                 ((23, 24), "##P"),
                 ((25, 33), "sentence"),
                 ((33, 34), "."),
                 ((0, 0), tokenizer_r.sep_token),
             ]
             if not do_lower_case
             else [
                 ((0, 0), tokenizer_r.cls_token),
                 ((0, 1), "a"),
                 ((1, 2), ","),
                 ((3, 8), "naive"),
                 ((9, 15), tokenizer_r.mask_token),
                 ((16, 21), "allen"),
                 ((21, 23), "##nl"),
                 ((23, 24), "##p"),
                 ((25, 33), "sentence"),
                 ((33, 34), "."),
                 ((0, 0), tokenizer_r.sep_token),
             ]
         )

         self.assertEqual(
             [e[1] for e in expected_results]       ,					tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]					)					)
         self.assertEqual([e[0] for e in expected_results]       ,					tokens["offset_mapping"]					)







      def 		UpperCAmelCase_  (	self      :int					)   ->	Union[str, Any]:
       UpperCAmelCase__							       =  ["的", "人", "有"]
       UpperCAmelCase__							       =  "".join(__lowerCamelCase					)
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''					):
         UpperCAmelCase__							       =  True
         UpperCAmelCase__							       =  self.tokenizer_class.from_pretrained(__lowerCamelCase       ,					**__lowerCamelCase					)
         UpperCAmelCase__							       =  self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,					**__lowerCamelCase					)

         UpperCAmelCase__							       =  tokenizer_p.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)
         UpperCAmelCase__							       =  tokenizer_r.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)

         UpperCAmelCase__							       =  tokenizer_r.convert_ids_to_tokens(__lowerCamelCase					)
         UpperCAmelCase__							       =  tokenizer_p.convert_ids_to_tokens(__lowerCamelCase					)

         # it is expected that each Chinese character is not preceded by "##"
         self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)
         self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)

         UpperCAmelCase__							       =  False
         UpperCAmelCase__							       =  self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,					**__lowerCamelCase					)
         UpperCAmelCase__							       =  self.tokenizer_class.from_pretrained(__lowerCamelCase       ,					**__lowerCamelCase					)

         UpperCAmelCase__							       =  tokenizer_r.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)
         UpperCAmelCase__							       =  tokenizer_p.encode(__lowerCamelCase       ,					add_special_tokens=__lowerCamelCase					)

         UpperCAmelCase__							       =  tokenizer_r.convert_ids_to_tokens(__lowerCamelCase					)
         UpperCAmelCase__							       =  tokenizer_p.convert_ids_to_tokens(__lowerCamelCase					)

         # it is expected that only the first Chinese character is not preceded by "##".
         UpperCAmelCase__							       =  [
             f'''##{token}''' if idx != 0 else token for idx, token in enumerate(__lowerCamelCase					)
         ]
         self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)
         self.assertListEqual(__lowerCamelCase       ,					__lowerCamelCase					)












from __future__ import annotations

import math


def 	UpperCamelCase__(       UpperCamelCase__			:       int  )->bool:

       if 1 < number < 4:
              # 2 and 3 are primes
              return True
       elif number < 2 or number % 2 == 0 or number % 3 == 0:
              # Negatives, 0, 1, all even numbers, all multiples of 3 are not primes
              return False

       # All primes number are in format of 6k +/- 1
       for i in range(5 ,     int(math.sqrt(UpperCamelCase__  ) + 1  ) ,     6  ):
              if number % i == 0 or number % (i + 2) == 0:
                     return False
       return True


def 	UpperCamelCase__(       UpperCamelCase__			:       int  )->list[int]:
       A__											=						str(UpperCamelCase__  )
       A__											=						[n]
       for i in range(1 ,     len(UpperCamelCase__  )  ):
              list_nums.append(int(str_num[i:]  )  )
              list_nums.append(int(str_num[:-i]  )  )
       return list_nums


def 	UpperCamelCase__(       UpperCamelCase__			:       int  )->bool:
       if len(str(UpperCamelCase__  )  ) > 3:
              if not is_prime(int(str(UpperCamelCase__  )[-3:]  )  ) or not is_prime(int(str(UpperCamelCase__  )[:3]  )  ):
                     return False
       return True


def 	UpperCamelCase__(       UpperCamelCase__			:       int = 11  )->list[int]:
       A__											=						[]
       A__											=						13
       while len(UpperCamelCase__  ) != count:
              if validate(UpperCamelCase__  ):
                     A__											=						list_truncated_nums(UpperCamelCase__  )
                     if all(is_prime(UpperCamelCase__  ) for i in list_nums  ):
                            list_truncated_primes.append(UpperCamelCase__  )
              num += 2
       return list_truncated_primes


def 	UpperCamelCase__(       )->int:
       return sum(compute_truncated_primes(11  )  )


if __name__ == "__main__":
       print(F"{sum(compute_truncated_primes(11)) = }")





