


'''simple docstring'''


import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class 		lowerCAmelCase__	(	UpperCAmelCase__			,				unittest.TestCase		):
			lowerCAmelCase       :		Dict				      =   RoCBertTokenizer
			lowerCAmelCase       :		Dict				      =   None
			lowerCAmelCase       :		int				      =   False
			lowerCAmelCase       :		Dict				      =   True
			lowerCAmelCase       :		Optional[Any]				      =   filter_non_english




			def 							lowerCAmelCase__			(						self     :   List[str]       )      ->Optional[Any]:



					'''simple docstring'''







					super().setUp()

					_UpperCAmelCase						:				str	    =					["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "你", "好", "是", "谁", "a", "b", "c", "d"]
					_UpperCAmelCase						:				List[Any]	    =					{}
					_UpperCAmelCase						:				str	    =					{}
					for i, value in enumerate(lowerCamelCase__       ):
							_UpperCAmelCase						:				Optional[Any]	    =					i
							_UpperCAmelCase						:				str	    =					i
					_UpperCAmelCase						:				str	    =					os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["vocab_file"]       )
					_UpperCAmelCase						:				Dict	    =					os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["word_shape_file"]       )
					_UpperCAmelCase						:				Union[str, Any]	    =					os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["word_pronunciation_file"]       )
					with open(self.vocab_file		,     "w"		,     encoding="utf-8"       ) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]       )       )
					with open(self.word_shape_file		,     "w"		,     encoding="utf-8"       ) as word_shape_writer:
							json.dump(lowerCamelCase__		,     lowerCamelCase__		,     ensure_ascii=lowerCamelCase__       )
					with open(self.word_pronunciation_file		,     "w"		,     encoding="utf-8"       ) as word_pronunciation_writer:
							json.dump(lowerCamelCase__		,     lowerCamelCase__		,     ensure_ascii=lowerCamelCase__       )




			def 							lowerCAmelCase__			(						self     :   Union[str, Any]       )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[int]	    =					self.tokenizer_class(self.vocab_file		,     self.word_shape_file		,     self.word_pronunciation_file       )

					_UpperCAmelCase						:				Union[str, Any]	    =					tokenizer.tokenize("你好[SEP]你是谁"       )
					self.assertListEqual(lowerCamelCase__		,     ["你", "好", "[SEP]", "你", "是", "谁"]       )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__       )		,     [5, 6, 2, 5, 7, 8]       )
					self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(lowerCamelCase__       )		,     [5, 6, 2, 5, 7, 8]       )
					self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(lowerCamelCase__       )		,     [5, 6, 2, 5, 7, 8]       )




			def 							lowerCAmelCase__			(						self     :   Tuple       )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[int]	    =					RoCBertBasicTokenizer()

					self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"       )		,     ["ah", "\u535A", "\u63A8", "zz"]       )




			def 							lowerCAmelCase__			(						self     :   List[Any]       )      ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )		,     ["hello", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,     ["hello"]       )




			def 							lowerCAmelCase__			(						self     :   Tuple       )      ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[Any]	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__		,     strip_accents=lowerCamelCase__       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,     ["hällo", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,     ["h\u00E9llo"]       )




			def 							lowerCAmelCase__			(						self     :   Tuple       )      ->str:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[Any]	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__		,     strip_accents=lowerCamelCase__       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,     ["hallo", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,     ["hello"]       )




			def 							lowerCAmelCase__			(						self     :   Dict       )      ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,     ["hallo", "!", "how", "are", "you", "?"]       )
					self.assertListEqual(tokenizer.tokenize("H\u00E9llo"       )		,     ["hello"]       )




			def 							lowerCAmelCase__			(						self     :   int       )      ->int:



					'''simple docstring'''







					_UpperCAmelCase						:				List[Any]	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "       )		,     ["HeLLo", "!", "how", "Are", "yoU", "?"]       )




			def 							lowerCAmelCase__			(						self     :   List[Any]       )      ->Dict:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[int]	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__		,     strip_accents=lowerCamelCase__       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,     ["HäLLo", "!", "how", "Are", "yoU", "?"]       )




			def 							lowerCAmelCase__			(						self     :   Tuple       )      ->str:



					'''simple docstring'''







					_UpperCAmelCase						:				List[str]	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__		,     strip_accents=lowerCamelCase__       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "       )		,     ["HaLLo", "!", "how", "Are", "yoU", "?"]       )




			def 							lowerCAmelCase__			(						self     :   Tuple       )      ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				str	    =					RoCBertBasicTokenizer(do_lower_case=lowerCamelCase__		,     never_split=["[UNK]"]       )

					self.assertListEqual(
					    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"       )		,     ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]       )




			def 							lowerCAmelCase__			(						self     :   Optional[Any]       )      ->int:



					'''simple docstring'''







					_UpperCAmelCase						:				int	    =					["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

					_UpperCAmelCase						:				Optional[int]	    =					{}
					for i, token in enumerate(lowerCamelCase__       ):
							_UpperCAmelCase						:				Optional[int]	    =					i
					_UpperCAmelCase						:				int	    =					RoCBertWordpieceTokenizer(vocab=lowerCamelCase__		,     unk_token="[UNK]"       )

					self.assertListEqual(tokenizer.tokenize(""       )		,     []       )

					self.assertListEqual(tokenizer.tokenize("unwanted running"       )		,     ["un", "##want", "##ed", "runn", "##ing"]       )

					self.assertListEqual(tokenizer.tokenize("unwantedX running"       )		,     ["[UNK]", "runn", "##ing"]       )




			def 							lowerCAmelCase__			(						self     :   int       )      ->Optional[int]:



					'''simple docstring'''







					self.assertTrue(_is_whitespace(" "       )       )
					self.assertTrue(_is_whitespace("\t"       )       )
					self.assertTrue(_is_whitespace("\r"       )       )
					self.assertTrue(_is_whitespace("\n"       )       )
					self.assertTrue(_is_whitespace("\u00A0"       )       )

					self.assertFalse(_is_whitespace("A"       )       )
					self.assertFalse(_is_whitespace("-"       )       )




			def 							lowerCAmelCase__			(						self     :   Union[str, Any]       )      ->Union[str, Any]:



					'''simple docstring'''







					self.assertTrue(_is_control("\u0005"       )       )

					self.assertFalse(_is_control("A"       )       )
					self.assertFalse(_is_control(" "       )       )
					self.assertFalse(_is_control("\t"       )       )
					self.assertFalse(_is_control("\r"       )       )




			def 							lowerCAmelCase__			(						self     :   Any       )      ->int:



					'''simple docstring'''







					self.assertTrue(_is_punctuation("-"       )       )
					self.assertTrue(_is_punctuation("$"       )       )
					self.assertTrue(_is_punctuation("`"       )       )
					self.assertTrue(_is_punctuation("."       )       )

					self.assertFalse(_is_punctuation("A"       )       )
					self.assertFalse(_is_punctuation(" "       )       )




			def 							lowerCAmelCase__			(						self     :   Any       )      ->Optional[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[Any]	    =					self.get_tokenizer()

					# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
					self.assertListEqual([tokenizer.tokenize(lowerCamelCase__       ) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]       )

					if self.test_rust_tokenizer:
							_UpperCAmelCase						:				str	    =					self.get_rust_tokenizer()
							self.assertListEqual(
							    [rust_tokenizer.tokenize(lowerCamelCase__       ) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]       )




			def 							lowerCAmelCase__			(						self     :   Dict       )      ->Dict:



					'''simple docstring'''







					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
									_UpperCAmelCase						:				Any	    =					self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,     **lowerCamelCase__       )

									_UpperCAmelCase						:				List[str]	    =					F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
									_UpperCAmelCase						:				List[Any]	    =					tokenizer_r.encode_plus(
									    lowerCamelCase__		,     return_attention_mask=lowerCamelCase__		,     return_token_type_ids=lowerCamelCase__		,     return_offsets_mapping=lowerCamelCase__		,     add_special_tokens=lowerCamelCase__		,     )

									_UpperCAmelCase						:				Tuple	    =					tokenizer_r.do_lower_case if hasattr(lowerCamelCase__		,     "do_lower_case"       ) else False
									_UpperCAmelCase						:				List[Any]	    =					(
									    [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), "A"),
									        ((1, 2), ","),
									        ((3, 5), "na"),
									        ((5, 6), "##ï"),
									        ((6, 8), "##ve"),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), "Allen"),
									        ((21, 23), "##NL"),
									        ((23, 24), "##P"),
									        ((25, 33), "sentence"),
									        ((33, 34), "."),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									    if not do_lower_case
									    else [
									        ((0, 0), tokenizer_r.cls_token),
									        ((0, 1), "a"),
									        ((1, 2), ","),
									        ((3, 8), "naive"),
									        ((9, 15), tokenizer_r.mask_token),
									        ((16, 21), "allen"),
									        ((21, 23), "##nl"),
									        ((23, 24), "##p"),
									        ((25, 33), "sentence"),
									        ((33, 34), "."),
									        ((0, 0), tokenizer_r.sep_token),
									    ]
									)

									self.assertEqual(
									    [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]       )       )
									self.assertEqual([e[0] for e in expected_results]		,     tokens["offset_mapping"]       )




			def 							lowerCAmelCase__			(						self     :   int       )      ->Union[str, Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					["的", "人", "有"]
					_UpperCAmelCase						:				Optional[int]	    =					"".join(lowerCamelCase__       )
					for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
							with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
									_UpperCAmelCase						:				Any	    =					True
									_UpperCAmelCase						:				Dict	    =					self.tokenizer_class.from_pretrained(lowerCamelCase__		,     **lowerCamelCase__       )
									_UpperCAmelCase						:				List[Any]	    =					self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,     **lowerCamelCase__       )

									_UpperCAmelCase						:				Dict	    =					tokenizer_p.encode(lowerCamelCase__		,     add_special_tokens=lowerCamelCase__       )
									_UpperCAmelCase						:				List[str]	    =					tokenizer_r.encode(lowerCamelCase__		,     add_special_tokens=lowerCamelCase__       )

									_UpperCAmelCase						:				Optional[Any]	    =					tokenizer_r.convert_ids_to_tokens(lowerCamelCase__       )
									_UpperCAmelCase						:				Tuple	    =					tokenizer_p.convert_ids_to_tokens(lowerCamelCase__       )

									# it is expected that each Chinese character is not preceded by "##"
									self.assertListEqual(lowerCamelCase__		,     lowerCamelCase__       )
									self.assertListEqual(lowerCamelCase__		,     lowerCamelCase__       )

									_UpperCAmelCase						:				List[str]	    =					False
									_UpperCAmelCase						:				int	    =					self.rust_tokenizer_class.from_pretrained(lowerCamelCase__		,     **lowerCamelCase__       )
									_UpperCAmelCase						:				List[Any]	    =					self.tokenizer_class.from_pretrained(lowerCamelCase__		,     **lowerCamelCase__       )

									_UpperCAmelCase						:				Optional[int]	    =					tokenizer_r.encode(lowerCamelCase__		,     add_special_tokens=lowerCamelCase__       )
									_UpperCAmelCase						:				Tuple	    =					tokenizer_p.encode(lowerCamelCase__		,     add_special_tokens=lowerCamelCase__       )

									_UpperCAmelCase						:				Union[str, Any]	    =					tokenizer_r.convert_ids_to_tokens(lowerCamelCase__       )
									_UpperCAmelCase						:				Union[str, Any]	    =					tokenizer_p.convert_ids_to_tokens(lowerCamelCase__       )

									# it is expected that only the first Chinese character is not preceded by "##".
									_UpperCAmelCase						:				List[Any]	    =					[
									    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(lowerCamelCase__       )
									]
									self.assertListEqual(lowerCamelCase__		,     lowerCamelCase__       )
									self.assertListEqual(lowerCamelCase__		,     lowerCamelCase__       )




			@slow
			def 							lowerCAmelCase__			(						self     :   Optional[Any]       )      ->int:



					'''simple docstring'''







					_UpperCAmelCase						:				Tuple	    =					self.tokenizer_class(self.vocab_file		,     self.word_shape_file		,     self.word_pronunciation_file       )

					_UpperCAmelCase						:				Any	    =					tokenizer.encode("你好"		,     add_special_tokens=lowerCamelCase__       )
					_UpperCAmelCase						:				List[str]	    =					tokenizer.encode("你是谁"		,     add_special_tokens=lowerCamelCase__       )

					_UpperCAmelCase						:				Dict	    =					tokenizer.build_inputs_with_special_tokens(lowerCamelCase__       )
					_UpperCAmelCase						:				List[Any]	    =					tokenizer.build_inputs_with_special_tokens(lowerCamelCase__		,     lowerCamelCase__       )

					assert encoded_sentence == [1] + text + [2]
					assert encoded_pair == [1] + text + [2] + text_a + [2]






			def 							lowerCAmelCase__			(						self     :   List[str]       )      ->List[str]:



					'''simple docstring'''







					_UpperCAmelCase						:				str	    =					self.get_tokenizers(do_lower_case=lowerCamelCase__       )
					for tokenizer in tokenizers:
							with self.subTest(F"""{tokenizer.__class__.__name__}"""       ):
									_UpperCAmelCase						:				Optional[Any]	    =					"你好，你是谁"
									_UpperCAmelCase						:				int	    =					tokenizer.tokenize(lowerCamelCase__       )
									_UpperCAmelCase						:				int	    =					tokenizer.convert_tokens_to_ids(lowerCamelCase__       )
									_UpperCAmelCase						:				Any	    =					tokenizer.convert_tokens_to_shape_ids(lowerCamelCase__       )
									_UpperCAmelCase						:				int	    =					tokenizer.convert_tokens_to_pronunciation_ids(lowerCamelCase__       )
									_UpperCAmelCase						:				int	    =					tokenizer.prepare_for_model(
									    lowerCamelCase__		,     lowerCamelCase__		,     lowerCamelCase__		,     add_special_tokens=lowerCamelCase__       )

									_UpperCAmelCase						:				Optional[Any]	    =					tokenizer.encode_plus(lowerCamelCase__		,     add_special_tokens=lowerCamelCase__       )

									self.assertEqual(lowerCamelCase__		,     lowerCamelCase__       )







'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_fnet import FNetTokenizer
else:
						lowerCamelCase__					=   None

lowerCamelCase__					=   logging.get_logger(__name__)
lowerCamelCase__					=   {'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

lowerCamelCase__					=   {
    'vocab_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/spiece.model',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'google/fnet-base': 'https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json',
        'google/fnet-large': 'https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json',
    },
}

lowerCamelCase__					=   {
    'google/fnet-base': 512,
    'google/fnet-large': 512,
}

lowerCamelCase__					=   '▁'






class 		lowerCAmelCase__	(	UpperCAmelCase__		):
			lowerCAmelCase       :		Dict				      =   VOCAB_FILES_NAMES
			lowerCAmelCase       :		Optional[Any]				      =   PRETRAINED_VOCAB_FILES_MAP
			lowerCAmelCase       :		List[str]				      =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCAmelCase       :		Optional[int]				      =   ["input_ids", "token_type_ids"]
			lowerCAmelCase       :		Optional[Any]				      =   FNetTokenizer




			def __init__(						self     :   Dict		,     lowerCamelCase__     :   int=None		,     lowerCamelCase__     :   List[Any]=None		,     lowerCamelCase__     :   Any=False		,     lowerCamelCase__     :   List[str]=True		,     lowerCamelCase__     :   Dict=True		,     lowerCamelCase__     :   str="<unk>"		,     lowerCamelCase__     :   List[str]="[SEP]"		,     lowerCamelCase__     :   Union[str, Any]="<pad>"		,     lowerCamelCase__     :   Optional[Any]="[CLS]"		,     lowerCamelCase__     :   Any="[MASK]"		,     **lowerCamelCase__     :   Any		,     )      ->List[Any]:



					'''simple docstring'''







					_UpperCAmelCase						:				Dict	    =					(
					    AddedToken(lowerCamelCase__		,     lstrip=lowerCamelCase__		,     rstrip=lowerCamelCase__		,     normalized=lowerCamelCase__       )
					    if isinstance(lowerCamelCase__		,     lowerCamelCase__       )
					    else mask_token
					)

					super().__init__(
					    lowerCamelCase__		,     tokenizer_file=lowerCamelCase__		,     do_lower_case=lowerCamelCase__		,     remove_space=lowerCamelCase__		,     keep_accents=lowerCamelCase__		,     unk_token=lowerCamelCase__		,     sep_token=lowerCamelCase__		,     pad_token=lowerCamelCase__		,     cls_token=lowerCamelCase__		,     mask_token=lowerCamelCase__		,     **lowerCamelCase__		,     )

					_UpperCAmelCase						:				Optional[Any]	    =					do_lower_case
					_UpperCAmelCase						:				Tuple	    =					remove_space
					_UpperCAmelCase						:				List[Any]	    =					keep_accents
					_UpperCAmelCase						:				Tuple	    =					vocab_file
					_UpperCAmelCase						:				str	    =					False if not self.vocab_file else True




			def 							lowerCAmelCase__			(						self     :   str		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[Any]	    =					[self.sep_token_id]
					_UpperCAmelCase						:				Tuple	    =					[self.cls_token_id]
					if token_ids_a is None:
							return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep




			def 							lowerCAmelCase__			(						self     :   int		,     lowerCamelCase__     :   List[int]		,     lowerCamelCase__     :   Optional[List[int]] = None       )      ->List[int]:



					'''simple docstring'''







					_UpperCAmelCase						:				Optional[int]	    =					[self.sep_token_id]
					_UpperCAmelCase						:				Optional[Any]	    =					[self.cls_token_id]

					if token_ids_a is None:
							return len(cls + token_ids_a + sep       ) * [0]
					return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]






			def 							lowerCAmelCase__			(						self     :   Dict		,     lowerCamelCase__     :   str		,     lowerCamelCase__     :   Optional[str] = None       )      ->Tuple[str]:



					'''simple docstring'''







					if not os.path.isdir(lowerCamelCase__       ):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""       )
							return
					_UpperCAmelCase						:				Union[str, Any]	    =					os.path.join(
					    lowerCamelCase__		,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

					if os.path.abspath(self.vocab_file       ) != os.path.abspath(lowerCamelCase__       ):
							copyfile(self.vocab_file		,     lowerCamelCase__       )

					return (out_vocab_file,)



