



from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A__  :  List[str]								=					logging.get_logger(__name__)


A__  :  Tuple								=					{
    '''nielsr/canine-s''': 2048,
}

# Unicode defines 1,114,112 total “codepoints”
A__  :  Dict								=					111_4112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
A__  :  List[Any]								=					0

A__  :  Optional[int]								=					0XE000
A__  :  Union[str, Any]								=					0XE001
A__  :  int								=					0XE002
A__  :  Optional[Any]								=					0XE003
A__  :  Dict								=					0XE004

# Maps special codepoints to human-readable names.
A__  :  Dict[int, str]								=					{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
A__  :  Dict[str, int]								=					{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}



class 				__snake_case						(     UpperCamelCase_						):
							_a			      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




							def __init__(       self     :       Dict				,				A_     :       List[str]=chr(A_)				,				A_     :       Union[str, Any]=chr(A_)				,				A_     :       List[Any]=chr(A_)				,				A_     :       Union[str, Any]=chr(A_)				,				A_     :       Tuple=chr(A_)				,				A_     :       Optional[int]=chr(A_)				,				A_     :       List[Any]=False				,				A_     :       Optional[Any]=2_0_4_8				,				**A_     :       Dict				,				):
									lowerCAmelCase_			:     List[str]									= AddedToken(A_				,				lstrip=A_				,				rstrip=A_) if isinstance(A_				,				A_) else bos_token
									lowerCAmelCase_			:     str									= AddedToken(A_				,				lstrip=A_				,				rstrip=A_) if isinstance(A_				,				A_) else eos_token
									lowerCAmelCase_			:     List[Any]									= AddedToken(A_				,				lstrip=A_				,				rstrip=A_) if isinstance(A_				,				A_) else sep_token
									lowerCAmelCase_			:     int									= AddedToken(A_				,				lstrip=A_				,				rstrip=A_) if isinstance(A_				,				A_) else cls_token
									lowerCAmelCase_			:     Tuple									= AddedToken(A_				,				lstrip=A_				,				rstrip=A_) if isinstance(A_				,				A_) else pad_token

									# Mask token behave like a normal word, i.e. include the space before it
									lowerCAmelCase_			:     Tuple									= AddedToken(A_				,				lstrip=A_				,				rstrip=A_) if isinstance(A_				,				A_) else mask_token

									super().__init__(
									    bos_token=A_				,				eos_token=A_				,				sep_token=A_				,				cls_token=A_				,				pad_token=A_				,				mask_token=A_				,				add_prefix_space=A_				,				model_max_length=A_				,				**A_				,				)

									# Creates a mapping for looking up the IDs of special symbols.
									lowerCAmelCase_			:     Dict[str, int]									= {}
									for codepoint, name in SPECIAL_CODEPOINTS.items():
											lowerCAmelCase_			:     List[str]									= codepoint

									# Creates a mapping for looking up the string forms of special symbol IDs.
									lowerCAmelCase_			:     Dict[int, str]									= {
									    codepoint: name for name, codepoint in self._special_codepoints.items()
									}

									lowerCAmelCase_			:     Tuple									= UNICODE_VOCAB_SIZE
									lowerCAmelCase_			:     List[str]									= len(self._special_codepoints)




							@property
							def    UpperCAmelCase__			(       self     :       str):
									return self._unicode_vocab_size




							def    UpperCAmelCase__			(       self     :       List[str]				,				A_     :       str):
									return list(A_)




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       str):
									try:
											return ord(A_)
									except TypeError:
											raise ValueError(F"""invalid token: '{token}'""")




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       int):
									try:
											if index in SPECIAL_CODEPOINTS:
													return SPECIAL_CODEPOINTS[index]
											return chr(A_)
									except TypeError:
											raise ValueError(F"""invalid id: {index}""")




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       List[str]):
									return "".join(A_)




							def    UpperCAmelCase__			(       self     :       str				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None):
									lowerCAmelCase_			:     Optional[int]									= [self.sep_token_id]
									lowerCAmelCase_			:     Optional[int]									= [self.cls_token_id]

									lowerCAmelCase_			:     Dict									= cls + token_ids_a + sep
									if token_ids_a is not None:
											result += token_ids_a + sep
									return result




							def    UpperCAmelCase__			(       self     :       Dict				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None				,				A_     :       bool = False):
									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=A_				,				token_ids_a=A_				,				already_has_special_tokens=A_)

									lowerCAmelCase_			:     Dict									= [1] + ([0] * len(A_)) + [1]
									if token_ids_a is not None:
											result += ([0] * len(A_)) + [1]
									return result




							def    UpperCAmelCase__			(       self     :       int				,				A_     :       List[int]				,				A_     :       Optional[List[int]] = None):
									lowerCAmelCase_			:     Any									= [self.sep_token_id]
									lowerCAmelCase_			:     int									= [self.cls_token_id]

									lowerCAmelCase_			:     Optional[int]									= len(cls + token_ids_a + sep) * [0]
									if token_ids_a is not None:
											result += len(token_ids_a + sep) * [1]
									return result







							def    UpperCAmelCase__			(       self     :       Optional[Any]				,				A_     :       str				,				A_     :       Optional[str] = None):
									return ()










import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english


@require_tokenizers
class       a							(			__lowerCamelCase  ,							unittest.TestCase				):
		__lowerCAmelCase	:    int											=					BertTokenizer
		__lowerCAmelCase	:    int											=					BertTokenizerFast
		__lowerCAmelCase	:    int											=					True
		__lowerCAmelCase	:    List[Any]											=					True
		__lowerCAmelCase	:    Any											=					filter_non_english



		def 	__lowerCamelCase			(     self     :str  ):
						super().setUp()

						snake_case__	:      str     		=	[
						    '''[UNK]''',
						    '''[CLS]''',
						    '''[SEP]''',
						    '''[PAD]''',
						    '''[MASK]''',
						    '''want''',
						    '''##want''',
						    '''##ed''',
						    '''wa''',
						    '''un''',
						    '''runn''',
						    '''##ing''',
						    ''',''',
						    '''low''',
						    '''lowest''',
						]
						snake_case__	:      str     		=	os.path.join(self.tmpdirname   ,VOCAB_FILES_NAMES['''vocab_file''']  )
						with open(self.vocab_file   ,'''w'''   ,encoding='''utf-8'''  ) as vocab_writer:
										vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )



		def 	__lowerCamelCase			(     self     :List[str]   ,__lowercase     :str  ):
						snake_case__	:      List[Any]     		=	'''UNwant\u00E9d,running'''
						snake_case__	:      str     		=	'''unwanted, running'''
						return input_text, output_text



		def 	__lowerCamelCase			(     self     :Optional[int]  ):
						snake_case__	:      str     		=	self.tokenizer_class(self.vocab_file  )

						snake_case__	:      str     		=	tokenizer.tokenize('''UNwant\u00E9d,running'''  )
						self.assertListEqual(__lowercase   ,['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase  )   ,[9, 6, 7, 1_2, 1_0, 1_1]  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						if not self.test_rust_tokenizer:
										return

						snake_case__	:      str     		=	self.get_tokenizer()
						snake_case__	:      Dict     		=	self.get_rust_tokenizer()

						snake_case__	:      List[Any]     		=	'''UNwant\u00E9d,running'''

						snake_case__	:      Dict     		=	tokenizer.tokenize(__lowercase  )
						snake_case__	:      int     		=	rust_tokenizer.tokenize(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      List[str]     		=	tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						snake_case__	:      List[str]     		=	rust_tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      Optional[Any]     		=	self.get_rust_tokenizer()
						snake_case__	:      List[Any]     		=	tokenizer.encode(__lowercase  )
						snake_case__	:      Dict     		=	rust_tokenizer.encode(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						# With lower casing
						snake_case__	:      List[Any]     		=	self.get_tokenizer(do_lower_case=__lowercase  )
						snake_case__	:      Optional[int]     		=	self.get_rust_tokenizer(do_lower_case=__lowercase  )

						snake_case__	:      int     		=	'''UNwant\u00E9d,running'''

						snake_case__	:      List[str]     		=	tokenizer.tokenize(__lowercase  )
						snake_case__	:      Union[str, Any]     		=	rust_tokenizer.tokenize(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      int     		=	tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						snake_case__	:      List[str]     		=	rust_tokenizer.encode(__lowercase   ,add_special_tokens=__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )

						snake_case__	:      Tuple     		=	self.get_rust_tokenizer()
						snake_case__	:      int     		=	tokenizer.encode(__lowercase  )
						snake_case__	:      Dict     		=	rust_tokenizer.encode(__lowercase  )
						self.assertListEqual(__lowercase   ,__lowercase  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      Optional[Any]     		=	BasicTokenizer()

						self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''  )   ,['''ah''', '''\u535A''', '''\u63A8''', '''zz''']  )



		def 	__lowerCamelCase			(     self     :Any  ):
						snake_case__	:      Dict     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )   ,['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      Optional[int]     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''h\u00E9llo''']  )



		def 	__lowerCamelCase			(     self     :str  ):
						snake_case__	:      Union[str, Any]     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      Union[str, Any]     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']  )
						self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''  )   ,['''hello''']  )



		def 	__lowerCamelCase			(     self     :Optional[Any]  ):
						snake_case__	:      str     		=	BasicTokenizer(do_lower_case=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''  )   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      Any     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :int  ):
						snake_case__	:      str     		=	BasicTokenizer(do_lower_case=__lowercase   ,strip_accents=__lowercase  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''  )   ,['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']  )



		def 	__lowerCamelCase			(     self     :Tuple  ):
						snake_case__	:      List[Any]     		=	BasicTokenizer(do_lower_case=__lowercase   ,never_split=['''[UNK]''']  )

						self.assertListEqual(
						    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''  )   ,['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']  )



		def 	__lowerCamelCase			(     self     :Dict  ):
						snake_case__	:      List[str]     		=	BasicTokenizer()
						snake_case__	:      Any     		=	'''a\n\'ll !!to?\'d of, can\'t.'''
						snake_case__	:      Optional[int]     		=	['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
						self.assertListEqual(tokenizer.tokenize(__lowercase  )   ,__lowercase  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      Any     		=	['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

						snake_case__	:      Optional[Any]     		=	{}
						for i, token in enumerate(__lowercase  ):
										snake_case__	:      Dict     		=	i
						snake_case__	:      int     		=	WordpieceTokenizer(vocab=__lowercase   ,unk_token='''[UNK]'''  )

						self.assertListEqual(tokenizer.tokenize(''''''  )   ,[]  )

						self.assertListEqual(tokenizer.tokenize('''unwanted running'''  )   ,['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']  )

						self.assertListEqual(tokenizer.tokenize('''unwantedX running'''  )   ,['''[UNK]''', '''runn''', '''##ing''']  )



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						self.assertTrue(_is_whitespace(''' '''  )  )
						self.assertTrue(_is_whitespace('''\t'''  )  )
						self.assertTrue(_is_whitespace('''\r'''  )  )
						self.assertTrue(_is_whitespace('''\n'''  )  )
						self.assertTrue(_is_whitespace('''\u00A0'''  )  )

						self.assertFalse(_is_whitespace('''A'''  )  )
						self.assertFalse(_is_whitespace('''-'''  )  )



		def 	__lowerCamelCase			(     self     :Union[str, Any]  ):
						self.assertTrue(_is_control('''\u0005'''  )  )

						self.assertFalse(_is_control('''A'''  )  )
						self.assertFalse(_is_control(''' '''  )  )
						self.assertFalse(_is_control('''\t'''  )  )
						self.assertFalse(_is_control('''\r'''  )  )



		def 	__lowerCamelCase			(     self     :str  ):
						self.assertTrue(_is_punctuation('''-'''  )  )
						self.assertTrue(_is_punctuation('''$'''  )  )
						self.assertTrue(_is_punctuation('''`'''  )  )
						self.assertTrue(_is_punctuation('''.'''  )  )

						self.assertFalse(_is_punctuation('''A'''  )  )
						self.assertFalse(_is_punctuation(''' '''  )  )



		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      str     		=	self.get_tokenizer()
						snake_case__	:      List[str]     		=	self.get_rust_tokenizer()

						# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
						self.assertListEqual([tokenizer.tokenize(__lowercase  ) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]  )

						self.assertListEqual(
						    [rust_tokenizer.tokenize(__lowercase  ) for t in ['''Test''', '''\xad''', '''test''']]   ,[['''[UNK]'''], [], ['''[UNK]''']]  )



		@slow
		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      List[str]     		=	self.tokenizer_class.from_pretrained('''bert-base-uncased'''  )

						snake_case__	:      Union[str, Any]     		=	tokenizer.encode('''sequence builders'''   ,add_special_tokens=__lowercase  )
						snake_case__	:      str     		=	tokenizer.encode('''multi-sequence build'''   ,add_special_tokens=__lowercase  )

						snake_case__	:      Tuple     		=	tokenizer.build_inputs_with_special_tokens(__lowercase  )
						snake_case__	:      int     		=	tokenizer.build_inputs_with_special_tokens(__lowercase   ,__lowercase  )

						assert encoded_sentence == [1_0_1] + text + [1_0_2]
						assert encoded_pair == [1_0_1] + text + [1_0_2] + text_a + [1_0_2]



		def 	__lowerCamelCase			(     self     :List[Any]  ):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
														snake_case__	:      str     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      List[str]     		=	F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
														snake_case__	:      List[Any]     		=	tokenizer_r.encode_plus(
														    __lowercase   ,return_attention_mask=__lowercase   ,return_token_type_ids=__lowercase   ,return_offsets_mapping=__lowercase   ,add_special_tokens=__lowercase   ,)

														snake_case__	:      List[str]     		=	tokenizer_r.do_lower_case if hasattr(__lowercase   ,'''do_lower_case'''  ) else False
														snake_case__	:      Optional[int]     		=	(
														    [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''A'''),
														        ((1, 2), ''','''),
														        ((3, 5), '''na'''),
														        ((5, 6), '''##ï'''),
														        ((6, 8), '''##ve'''),
														        ((9, 1_5), tokenizer_r.mask_token),
														        ((1_6, 2_1), '''Allen'''),
														        ((2_1, 2_3), '''##NL'''),
														        ((2_3, 2_4), '''##P'''),
														        ((2_5, 3_3), '''sentence'''),
														        ((3_3, 3_4), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														    if not do_lower_case
														    else [
														        ((0, 0), tokenizer_r.cls_token),
														        ((0, 1), '''a'''),
														        ((1, 2), ''','''),
														        ((3, 8), '''naive'''),
														        ((9, 1_5), tokenizer_r.mask_token),
														        ((1_6, 2_1), '''allen'''),
														        ((2_1, 2_3), '''##nl'''),
														        ((2_3, 2_4), '''##p'''),
														        ((2_5, 3_3), '''sentence'''),
														        ((3_3, 3_4), '''.'''),
														        ((0, 0), tokenizer_r.sep_token),
														    ]
														)

														self.assertEqual(
														    [e[1] for e in expected_results]   ,tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']  )  )
														self.assertEqual([e[0] for e in expected_results]   ,tokens['''offset_mapping''']  )






		def 	__lowerCamelCase			(     self     :List[str]  ):
						snake_case__	:      str     		=	['''的''', '''人''', '''有''']
						snake_case__	:      Optional[int]     		=	''''''.join(__lowercase  )
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
														snake_case__	:      Optional[Any]     		=	True
														snake_case__	:      int     		=	self.tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )
														snake_case__	:      Tuple     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      Union[str, Any]     		=	tokenizer_p.encode(__lowercase   ,add_special_tokens=__lowercase  )
														snake_case__	:      List[str]     		=	tokenizer_r.encode(__lowercase   ,add_special_tokens=__lowercase  )

														snake_case__	:      List[str]     		=	tokenizer_r.convert_ids_to_tokens(__lowercase  )
														snake_case__	:      Optional[Any]     		=	tokenizer_p.convert_ids_to_tokens(__lowercase  )

														# it is expected that each Chinese character is not preceded by "##"
														self.assertListEqual(__lowercase   ,__lowercase  )
														self.assertListEqual(__lowercase   ,__lowercase  )

														snake_case__	:      str     		=	False
														snake_case__	:      Dict     		=	self.rust_tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )
														snake_case__	:      int     		=	self.tokenizer_class.from_pretrained(__lowercase   ,**__lowercase  )

														snake_case__	:      int     		=	tokenizer_r.encode(__lowercase   ,add_special_tokens=__lowercase  )
														snake_case__	:      Tuple     		=	tokenizer_p.encode(__lowercase   ,add_special_tokens=__lowercase  )

														snake_case__	:      List[Any]     		=	tokenizer_r.convert_ids_to_tokens(__lowercase  )
														snake_case__	:      Union[str, Any]     		=	tokenizer_p.convert_ids_to_tokens(__lowercase  )

														# it is expected that only the first Chinese character is not preceded by "##".
														snake_case__	:      Optional[int]     		=	[
														    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__lowercase  )
														]
														self.assertListEqual(__lowercase   ,__lowercase  )
														self.assertListEqual(__lowercase   ,__lowercase  )

