


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english







@require_tokenizers
class 			_lowercase (							lowerCAmelCase_							,  unittest.TestCase):


       """simple docstring"""




       A__											=			RoCBertTokenizer
       A__											=			None
       A__											=			False
       A__											=			True
       A__											=			filter_non_english





       def 						lowerCAmelCase     (   self							:			Union[str, Any]       ):


             '''simple docstring'''



             super().setUp()

             lowerCamelCase__		:				Union[str, Any]    =  ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
             lowerCamelCase__		:				Optional[Any]    =  {}
             lowerCamelCase__		:				int    =  {}
             for i, value in enumerate(__snake_case       ):
                   lowerCamelCase__		:				Dict    =  i
                   lowerCamelCase__		:				Optional[int]    =  i
             lowerCamelCase__		:				List[str]    =  os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES["vocab_file"]       )
             lowerCamelCase__		:				Optional[int]    =  os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES["word_shape_file"]       )
             lowerCamelCase__		:				Optional[int]    =  os.path.join(self.tmpdirname					,      VOCAB_FILES_NAMES["word_pronunciation_file"]       )
             with open(self.vocab_file					,      "w"					,      encoding="utf-8"       ) as vocab_writer:
                   vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )
             with open(self.word_shape_file					,      "w"					,      encoding="utf-8"       ) as word_shape_writer:
                   json.dump(__snake_case					,      __snake_case					,      ensure_ascii=__snake_case       )
             with open(self.word_pronunciation_file					,      "w"					,      encoding="utf-8"       ) as word_pronunciation_writer:
                   json.dump(__snake_case					,      __snake_case					,      ensure_ascii=__snake_case       )





       def 						lowerCAmelCase     (   self							:			Optional[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  self.tokenizer_class(self.vocab_file					,      self.word_shape_file					,      self.word_pronunciation_file       )

             lowerCamelCase__		:				Optional[Any]    =  tokenizer.tokenize("你好[SEP]你是谁"       )
             self.assertListEqual(__snake_case					,      ["你", "好", "[SEP]", "你", "是", "谁"]       )
             self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case       )					,      [5, 6, 2, 5, 7, 8]       )
             self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(__snake_case       )					,      [5, 6, 2, 5, 7, 8]       )
             self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(__snake_case       )					,      [5, 6, 2, 5, 7, 8]       )





       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             lowerCamelCase__		:				Union[str, Any]    =  RoCBertBasicTokenizer()

             self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )					,      ["ah", "\u535A", "\u63A8", "zz"]       )





       def 						lowerCAmelCase     (   self							:			Tuple       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  RoCBertBasicTokenizer(do_lower_case=__snake_case       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )					,      ["hello", "!", "how", "are", "you", "?"]       )
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,      ["hello"]       )





       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  RoCBertBasicTokenizer(do_lower_case=__snake_case					,      strip_accents=__snake_case       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,      ["hällo", "!", "how", "are", "you", "?"]       )
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,      ["h\u00E9llo"]       )





       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  RoCBertBasicTokenizer(do_lower_case=__snake_case					,      strip_accents=__snake_case       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,      ["hallo", "!", "how", "are", "you", "?"]       )
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,      ["hello"]       )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[int]    =  RoCBertBasicTokenizer(do_lower_case=__snake_case       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,      ["hallo", "!", "how", "are", "you", "?"]       )
             self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )					,      ["hello"]       )





       def 						lowerCAmelCase     (   self							:			Union[str, Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[Any]    =  RoCBertBasicTokenizer(do_lower_case=__snake_case       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )					,      ["HeLLo", "!", "how", "Are", "yoU", "?"]       )





       def 						lowerCAmelCase     (   self							:			Optional[int]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Tuple    =  RoCBertBasicTokenizer(do_lower_case=__snake_case					,      strip_accents=__snake_case       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,      ["HäLLo", "!", "how", "Are", "yoU", "?"]       )





       def 						lowerCAmelCase     (   self							:			Dict       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  RoCBertBasicTokenizer(do_lower_case=__snake_case					,      strip_accents=__snake_case       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )					,      ["HaLLo", "!", "how", "Are", "yoU", "?"]       )





       def 						lowerCAmelCase     (   self							:			Any       ):


             '''simple docstring'''



             lowerCamelCase__		:				Any    =  RoCBertBasicTokenizer(do_lower_case=__snake_case					,      never_split=["[UNK]"]       )

             self.assertListEqual(
                 tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )					,      ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )





       def 						lowerCAmelCase     (   self							:			Optional[int]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Any    =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

             lowerCamelCase__		:				List[str]    =  {}
             for i, token in enumerate(__snake_case       ):
                   lowerCamelCase__		:				Optional[Any]    =  i
             lowerCamelCase__		:				Optional[Any]    =  RoCBertWordpieceTokenizer(vocab=__snake_case					,      unk_token="[UNK]"       )

             self.assertListEqual(tokenizer.tokenize(""       )					,      []       )

             self.assertListEqual(tokenizer.tokenize("unwanted running"       )					,      ["un", "##want", "##ed", "runn", "##ing"]       )

             self.assertListEqual(tokenizer.tokenize("unwantedX running"       )					,      ["[UNK]", "runn", "##ing"]       )





       def 						lowerCAmelCase     (   self							:			List[str]       ):


             '''simple docstring'''



             self.assertTrue(_is_whitespace(" "       )       )
             self.assertTrue(_is_whitespace("\t"       )       )
             self.assertTrue(_is_whitespace("\r"       )       )
             self.assertTrue(_is_whitespace("\n"       )       )
             self.assertTrue(_is_whitespace("\u00A0"       )       )

             self.assertFalse(_is_whitespace("A"       )       )
             self.assertFalse(_is_whitespace("-"       )       )





       def 						lowerCAmelCase     (   self							:			List[str]       ):


             '''simple docstring'''



             self.assertTrue(_is_control("\u0005"       )       )

             self.assertFalse(_is_control("A"       )       )
             self.assertFalse(_is_control(" "       )       )
             self.assertFalse(_is_control("\t"       )       )
             self.assertFalse(_is_control("\r"       )       )





       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             self.assertTrue(_is_punctuation("-"       )       )
             self.assertTrue(_is_punctuation("$"       )       )
             self.assertTrue(_is_punctuation("`"       )       )
             self.assertTrue(_is_punctuation("."       )       )

             self.assertFalse(_is_punctuation("A"       )       )
             self.assertFalse(_is_punctuation(" "       )       )





       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Dict    =  self.get_tokenizer()

             # Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
             self.assertListEqual([tokenizer.tokenize(__snake_case       ) for t in ["Test", "\xad", "test"]]					,      [["[UNK]"], [], ["[UNK]"]]       )

             if self.test_rust_tokenizer:
                   lowerCamelCase__		:				str    =  self.get_rust_tokenizer()
                   self.assertListEqual(
                       [rust_tokenizer.tokenize(__snake_case       ) for t in ["Test", "\xad", "test"]]					,      [["[UNK]"], [], ["[UNK]"]]       )





       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
                         lowerCamelCase__		:				List[str]    =  self.rust_tokenizer_class.from_pretrained(__snake_case					,      **__snake_case       )

                         lowerCamelCase__		:				Tuple    =  f"A, naïve {tokenizer_r.mask_token} AllenNLP sentence."
                         lowerCamelCase__		:				List[str]    =  tokenizer_r.encode_plus(
                             __snake_case					,      return_attention_mask=__snake_case					,      return_token_type_ids=__snake_case					,      return_offsets_mapping=__snake_case					,      add_special_tokens=__snake_case					,      )

                         lowerCamelCase__		:				Union[str, Any]    =  tokenizer_r.do_lower_case if hasattr(__snake_case					,      "do_lower_case"       ) else False
                         lowerCamelCase__		:				Optional[int]    =  (
                             [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), "A"),
                                 ((1, 2), ","),
                                 ((3, 5), "na"),
                                 ((5, 6), "##ï"),
                                 ((6, 8), "##ve"),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), "Allen"),
                                 ((21, 23), "##NL"),
                                 ((23, 24), "##P"),
                                 ((25, 33), "sentence"),
                                 ((33, 34), "."),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                             if not do_lower_case
                             else [
                                 ((0, 0), tokenizer_r.cls_token),
                                 ((0, 1), "a"),
                                 ((1, 2), ","),
                                 ((3, 8), "naive"),
                                 ((9, 15), tokenizer_r.mask_token),
                                 ((16, 21), "allen"),
                                 ((21, 23), "##nl"),
                                 ((23, 24), "##p"),
                                 ((25, 33), "sentence"),
                                 ((33, 34), "."),
                                 ((0, 0), tokenizer_r.sep_token),
                             ]
                         )

                         self.assertEqual(
                             [e[1] for e in expected_results]					,      tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
                         self.assertEqual([e[0] for e in expected_results]					,      tokens["offset_mapping"]       )





       def 						lowerCAmelCase     (   self							:			str       ):


             '''simple docstring'''



             lowerCamelCase__		:				List[str]    =  ["的", "人", "有"]
             lowerCamelCase__		:				str    =  "".join(__snake_case       )
             for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                   with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"       ):
                         lowerCamelCase__		:				int    =  True
                         lowerCamelCase__		:				Union[str, Any]    =  self.tokenizer_class.from_pretrained(__snake_case					,      **__snake_case       )
                         lowerCamelCase__		:				int    =  self.rust_tokenizer_class.from_pretrained(__snake_case					,      **__snake_case       )

                         lowerCamelCase__		:				Tuple    =  tokenizer_p.encode(__snake_case					,      add_special_tokens=__snake_case       )
                         lowerCamelCase__		:				Optional[int]    =  tokenizer_r.encode(__snake_case					,      add_special_tokens=__snake_case       )

                         lowerCamelCase__		:				Optional[Any]    =  tokenizer_r.convert_ids_to_tokens(__snake_case       )
                         lowerCamelCase__		:				Tuple    =  tokenizer_p.convert_ids_to_tokens(__snake_case       )

                         # it is expected that each Chinese character is not preceded by "##"
                         self.assertListEqual(__snake_case					,      __snake_case       )
                         self.assertListEqual(__snake_case					,      __snake_case       )

                         lowerCamelCase__		:				Dict    =  False
                         lowerCamelCase__		:				List[str]    =  self.rust_tokenizer_class.from_pretrained(__snake_case					,      **__snake_case       )
                         lowerCamelCase__		:				List[str]    =  self.tokenizer_class.from_pretrained(__snake_case					,      **__snake_case       )

                         lowerCamelCase__		:				Tuple    =  tokenizer_r.encode(__snake_case					,      add_special_tokens=__snake_case       )
                         lowerCamelCase__		:				Optional[Any]    =  tokenizer_p.encode(__snake_case					,      add_special_tokens=__snake_case       )

                         lowerCamelCase__		:				Optional[int]    =  tokenizer_r.convert_ids_to_tokens(__snake_case       )
                         lowerCamelCase__		:				List[Any]    =  tokenizer_p.convert_ids_to_tokens(__snake_case       )

                         # it is expected that only the first Chinese character is not preceded by "##".
                         lowerCamelCase__		:				Union[str, Any]    =  [
                             f"##{token}" if idx != 0 else token for idx, token in enumerate(__snake_case       )
                         ]
                         self.assertListEqual(__snake_case					,      __snake_case       )
                         self.assertListEqual(__snake_case					,      __snake_case       )





       @slow
       def 						lowerCAmelCase     (   self							:			List[Any]       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  self.tokenizer_class(self.vocab_file					,      self.word_shape_file					,      self.word_pronunciation_file       )

             lowerCamelCase__		:				List[Any]    =  tokenizer.encode("你好"					,      add_special_tokens=__snake_case       )
             lowerCamelCase__		:				Union[str, Any]    =  tokenizer.encode("你是谁"					,      add_special_tokens=__snake_case       )

             lowerCamelCase__		:				Optional[Any]    =  tokenizer.build_inputs_with_special_tokens(__snake_case       )
             lowerCamelCase__		:				Optional[int]    =  tokenizer.build_inputs_with_special_tokens(__snake_case					,      __snake_case       )

             assert encoded_sentence == [1] + text + [2]
             assert encoded_pair == [1] + text + [2] + text_a + [2]





       def 						lowerCAmelCase     (   self							:			int       ):


             '''simple docstring'''



             lowerCamelCase__		:				Optional[Any]    =  self.get_tokenizers(do_lower_case=__snake_case       )
             for tokenizer in tokenizers:
                   with self.subTest(f"{tokenizer.__class__.__name__}"       ):
                         lowerCamelCase__		:				List[Any]    =  "你好，你是谁"
                         lowerCamelCase__		:				Tuple    =  tokenizer.tokenize(__snake_case       )
                         lowerCamelCase__		:				List[Any]    =  tokenizer.convert_tokens_to_ids(__snake_case       )
                         lowerCamelCase__		:				Optional[int]    =  tokenizer.convert_tokens_to_shape_ids(__snake_case       )
                         lowerCamelCase__		:				Dict    =  tokenizer.convert_tokens_to_pronunciation_ids(__snake_case       )
                         lowerCamelCase__		:				List[Any]    =  tokenizer.prepare_for_model(
                             __snake_case					,      __snake_case					,      __snake_case					,      add_special_tokens=__snake_case       )

                         lowerCamelCase__		:				Any    =  tokenizer.encode_plus(__snake_case					,      add_special_tokens=__snake_case       )

                         self.assertEqual(__snake_case					,      __snake_case       )





from manim import *






class 	__magic_name__    (			lowerCAmelCase_      ):





			def 		__magic_name__    (		self					)   -> Any:




						'''simple docstring'''
						__a        =Rectangle(height=0.5					,  width=0.5					)
						__a        =Rectangle(height=0.25					,  width=0.25					)
						__a        =Rectangle(height=0.46					,  width=0.46					).set_stroke(width=0					)

						__a        =[mem.copy() for i in range(6					)]
						__a        =[mem.copy() for i in range(6					)]
						__a        =VGroup(*__snake_case					).arrange(__snake_case					,  buff=0					)
						__a        =VGroup(*__snake_case					).arrange(__snake_case					,  buff=0					)
						__a        =VGroup(__snake_case					,  __snake_case					).arrange(__snake_case					,  buff=0					)
						__a        =Text('CPU'					,  font_size=24					)
						__a        =Group(__snake_case					,  __snake_case					).arrange(__snake_case					,  buff=0.5					,  aligned_edge=__snake_case					)
						cpu.move_to([-2.5, -0.5, 0]					)
						self.add(__snake_case					)

						__a        =[mem.copy() for i in range(4					)]
						__a        =VGroup(*__snake_case					).arrange(__snake_case					,  buff=0					)
						__a        =Text('GPU'					,  font_size=24					)
						__a        =Group(__snake_case					,  __snake_case					).arrange(__snake_case					,  buff=0.5					,  aligned_edge=__snake_case					)
						gpu.move_to([-1, -1, 0]					)
						self.add(__snake_case					)

						__a        =[mem.copy() for i in range(6					)]
						__a        =VGroup(*__snake_case					).arrange(__snake_case					,  buff=0					)

						__a        =Text('Model'					,  font_size=24					)
						__a        =Group(__snake_case					,  __snake_case					).arrange(__snake_case					,  buff=0.5					,  aligned_edge=__snake_case					)
						model.move_to([3, -1.0, 0]					)
						self.add(__snake_case					)

						__a        =[]
						__a        =[]
						__a        =[]

						for i, rect in enumerate(__snake_case					):
									rect.set_stroke(__snake_case					)

									__a        =Rectangle(height=0.46 / 4					,  width=0.46 / 3					).set_stroke(width=0.0					).set_fill(__snake_case					,  opacity=0.7					)

									if i == 0:
												cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT					)					,  buff=0.02					,  direction=__snake_case					)
												cpu_target.set_x(cpu_target.get_x() + 0.1					)
									elif i == 3:
												cpu_target.next_to(model_cpu_arr[0]					,  direction=__snake_case					,  buff=0.0					)
									else:
												cpu_target.next_to(model_cpu_arr[i - 1]					,  direction=__snake_case					,  buff=0.0					)
									self.add(__snake_case					)
									model_cpu_arr.append(__snake_case					)

						self.add(*__snake_case					,  *__snake_case					,  *__snake_case					)

						__a        =[mem.copy() for i in range(6					)]
						__a        =VGroup(*__snake_case					).arrange(__snake_case					,  buff=0					)

						__a        =Text('Loaded Checkpoint'					,  font_size=24					)
						__a        =Group(__snake_case					,  __snake_case					).arrange(__snake_case					,  buff=0.5					,  aligned_edge=__snake_case					)
						checkpoint.move_to([3, 0.5, 0]					)

						self.add(__snake_case					)

						__a        =[]
						__a        =[]

						for i, rect in enumerate(__snake_case					):
									__a        =fill.copy().set_fill(__snake_case					,  opacity=0.7					)
									target.move_to(__snake_case					)
									ckpt_arr.append(__snake_case					)

									__a        =target.copy()
									if i < 5:
												cpu_target.move_to(cpu_left_col_base[i + 1]					)
									else:
												cpu_target.move_to(cpu_right_col_base[i - 5]					)
									ckpt_cpu_arr.append(__snake_case					)
						self.add(*__snake_case					,  *__snake_case					)

						__a        =Square(side_length=2.2					)
						key.move_to([-5, 2, 0]					)

						__a        =MarkupText(
						    f'<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'					,  font_size=18					,  )

						key_text.move_to([-5, 2.4, 0]					)

						self.add(__snake_case					,  __snake_case					)

						__a        =MarkupText(
						    f'<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'					,  font_size=18					,  )

						blue_text.next_to(__snake_case					,  DOWN * 2.4					,  aligned_edge=key_text.get_left()					)
						self.add(__snake_case					)

						__a        =MarkupText(
						    f'Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'					,  font_size=24					,  )
						step_a.move_to([2, 2, 0]					)

						__a        =[meta_mem.copy() for i in range(6					)]
						__a        =[meta_mem.copy() for i in range(6					)]
						__a        =VGroup(*__snake_case					).arrange(__snake_case					,  buff=0					)
						__a        =VGroup(*__snake_case					).arrange(__snake_case					,  buff=0					)
						__a        =VGroup(__snake_case					,  __snake_case					).arrange(__snake_case					,  buff=0					)
						__a        =Text('Disk'					,  font_size=24					)
						__a        =Group(__snake_case					,  __snake_case					).arrange(__snake_case					,  buff=0.5					,  aligned_edge=__snake_case					)
						disk.move_to([-4.0, -1.25, 0]					)
						self.play(Write(__snake_case					,  run_time=3					)					,  Write(__snake_case					,  run_time=1					)					,  Create(__snake_case					,  run_time=1					)					)

						__a        =[]
						for i, rect in enumerate(__snake_case					):
									__a        =rect.copy()
									target.generate_target()
									target.target.move_to(disk_left_col_base[i]					).scale(0.5					)
									animations.append(MoveToTarget(__snake_case					,  run_time=1.5					)					)
						self.play(*__snake_case					)

						self.play(FadeOut(__snake_case					)					)

						__a        =MarkupText(f'Then, the checkpoint is removed from memory\nthrough garbage collection.'					,  font_size=24					)
						step_a.move_to([2, 2, 0]					)

						self.play(Write(__snake_case					,  run_time=3					)					)

						self.play(
						    FadeOut(__snake_case					,  __snake_case					,  *__snake_case					,  *__snake_case					)					,  )

						self.wait()






