

import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english






@require_tokenizers
class     lowerCamelCase_			(  _lowercase	,			unittest.TestCase  ):
	_lowercase		:      Tuple    				=							ElectraTokenizer
	_lowercase		:      List[Any]    				=							ElectraTokenizerFast
	_lowercase		:      int    				=							True
	_lowercase		:      Dict    				=							True
	_lowercase		:      List[Any]    				=							filter_non_english




	def 					lowerCAmelCase_					(    self	:    Dict       ):
		super().setUp()

		__A   : Dict								= [
		    """[UNK]""",
		    """[CLS]""",
		    """[SEP]""",
		    """[PAD]""",
		    """[MASK]""",
		    """want""",
		    """##want""",
		    """##ed""",
		    """wa""",
		    """un""",
		    """runn""",
		    """##ing""",
		    """,""",
		    """low""",
		    """lowest""",
		]
		__A   : Optional[int]								= os.path.join(self.tmpdirname				,			VOCAB_FILES_NAMES["""vocab_file"""]       )
		with open(self.vocab_file				,			"""w"""				,			encoding="""utf-8"""       ) as vocab_writer:
			vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )




	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    Tuple       ):
		__A   : Dict								= """UNwant\u00E9d,running"""
		__A   : Dict								= """unwanted, running"""
		return input_text, output_text




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		__A   : Dict								= self.tokenizer_class(self.vocab_file       )

		__A   : List[str]								= tokenizer.tokenize("""UNwant\u00E9d,running"""       )
		self.assertListEqual(__A				,			["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]       )
		self.assertListEqual(tokenizer.convert_tokens_to_ids(__A       )				,			[9, 6, 7, 12, 10, 11]       )




	def 					lowerCAmelCase_					(    self	:    Optional[Any]       ):
		if not self.test_rust_tokenizer:
			return

		__A   : Any								= self.get_tokenizer()
		__A   : List[Any]								= self.get_rust_tokenizer()

		__A   : List[Any]								= """UNwant\u00E9d,running"""

		__A   : Any								= tokenizer.tokenize(__A       )
		__A   : Tuple								= rust_tokenizer.tokenize(__A       )
		self.assertListEqual(__A				,			__A       )

		__A   : Dict								= tokenizer.encode(__A				,			add_special_tokens=__A       )
		__A   : Optional[int]								= rust_tokenizer.encode(__A				,			add_special_tokens=__A       )
		self.assertListEqual(__A				,			__A       )

		__A   : int								= self.get_rust_tokenizer()
		__A   : Tuple								= tokenizer.encode(__A       )
		__A   : str								= rust_tokenizer.encode(__A       )
		self.assertListEqual(__A				,			__A       )

		# With lower casing
		__A   : Tuple								= self.get_tokenizer(do_lower_case=__A       )
		__A   : Dict								= self.get_rust_tokenizer(do_lower_case=__A       )

		__A   : Union[str, Any]								= """UNwant\u00E9d,running"""

		__A   : str								= tokenizer.tokenize(__A       )
		__A   : Optional[Any]								= rust_tokenizer.tokenize(__A       )
		self.assertListEqual(__A				,			__A       )

		__A   : Tuple								= tokenizer.encode(__A				,			add_special_tokens=__A       )
		__A   : Dict								= rust_tokenizer.encode(__A				,			add_special_tokens=__A       )
		self.assertListEqual(__A				,			__A       )

		__A   : str								= self.get_rust_tokenizer()
		__A   : List[str]								= tokenizer.encode(__A       )
		__A   : int								= rust_tokenizer.encode(__A       )
		self.assertListEqual(__A				,			__A       )




	def 					lowerCAmelCase_					(    self	:    Optional[Any]       ):
		__A   : List[str]								= BasicTokenizer()

		self.assertListEqual(tokenizer.tokenize("""ah\u535A\u63A8zz"""       )				,			["""ah""", """\u535A""", """\u63A8""", """zz"""]       )




	def 					lowerCAmelCase_					(    self	:    List[str]       ):
		__A   : Dict								= BasicTokenizer(do_lower_case=__A       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )				,			["""hello""", """!""", """how""", """are""", """you""", """?"""]       )
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )				,			["""hello"""]       )




	def 					lowerCAmelCase_					(    self	:    int       ):
		__A   : Any								= BasicTokenizer(do_lower_case=__A				,			strip_accents=__A       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )				,			["""hällo""", """!""", """how""", """are""", """you""", """?"""]       )
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )				,			["""h\u00E9llo"""]       )




	def 					lowerCAmelCase_					(    self	:    Dict       ):
		__A   : str								= BasicTokenizer(do_lower_case=__A				,			strip_accents=__A       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )				,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )				,			["""hello"""]       )




	def 					lowerCAmelCase_					(    self	:    Any       ):
		__A   : Optional[int]								= BasicTokenizer(do_lower_case=__A       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )				,			["""hallo""", """!""", """how""", """are""", """you""", """?"""]       )
		self.assertListEqual(tokenizer.tokenize("""H\u00E9llo"""       )				,			["""hello"""]       )




	def 					lowerCAmelCase_					(    self	:    Dict       ):
		__A   : str								= BasicTokenizer(do_lower_case=__A       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU?  """       )				,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		__A   : List[Any]								= BasicTokenizer(do_lower_case=__A				,			strip_accents=__A       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )				,			["""HäLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )




	def 					lowerCAmelCase_					(    self	:    List[str]       ):
		__A   : List[Any]								= BasicTokenizer(do_lower_case=__A				,			strip_accents=__A       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHäLLo!how  \n Are yoU?  """       )				,			["""HaLLo""", """!""", """how""", """Are""", """yoU""", """?"""]       )




	def 					lowerCAmelCase_					(    self	:    int       ):
		__A   : Union[str, Any]								= BasicTokenizer(do_lower_case=__A				,			never_split=["""[UNK]"""]       )

		self.assertListEqual(
		    tokenizer.tokenize(""" \tHeLLo!how  \n Are yoU? [UNK]"""       )				,			["""HeLLo""", """!""", """how""", """Are""", """yoU""", """?""", """[UNK]"""]       )




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		__A   : Dict								= ["""[UNK]""", """[CLS]""", """[SEP]""", """want""", """##want""", """##ed""", """wa""", """un""", """runn""", """##ing"""]

		__A   : Tuple								= {}
		for i, token in enumerate(__A       ):
			__A   : Union[str, Any]								= i
		__A   : Optional[int]								= WordpieceTokenizer(vocab=__A				,			unk_token="""[UNK]"""       )

		self.assertListEqual(tokenizer.tokenize(""""""       )				,			[]       )

		self.assertListEqual(tokenizer.tokenize("""unwanted running"""       )				,			["""un""", """##want""", """##ed""", """runn""", """##ing"""]       )

		self.assertListEqual(tokenizer.tokenize("""unwantedX running"""       )				,			["""[UNK]""", """runn""", """##ing"""]       )




	def 					lowerCAmelCase_					(    self	:    Optional[Any]       ):
		self.assertTrue(_is_whitespace(""" """       )       )
		self.assertTrue(_is_whitespace("""\t"""       )       )
		self.assertTrue(_is_whitespace("""\r"""       )       )
		self.assertTrue(_is_whitespace("""\n"""       )       )
		self.assertTrue(_is_whitespace("""\u00A0"""       )       )

		self.assertFalse(_is_whitespace("""A"""       )       )
		self.assertFalse(_is_whitespace("""-"""       )       )




	def 					lowerCAmelCase_					(    self	:    Dict       ):
		self.assertTrue(_is_control("""\u0005"""       )       )

		self.assertFalse(_is_control("""A"""       )       )
		self.assertFalse(_is_control(""" """       )       )
		self.assertFalse(_is_control("""\t"""       )       )
		self.assertFalse(_is_control("""\r"""       )       )




	def 					lowerCAmelCase_					(    self	:    Union[str, Any]       ):
		self.assertTrue(_is_punctuation("""-"""       )       )
		self.assertTrue(_is_punctuation("""$"""       )       )
		self.assertTrue(_is_punctuation("""`"""       )       )
		self.assertTrue(_is_punctuation("""."""       )       )

		self.assertFalse(_is_punctuation("""A"""       )       )
		self.assertFalse(_is_punctuation(""" """       )       )




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		__A   : Union[str, Any]								= self.get_tokenizer()
		__A   : Any								= self.get_rust_tokenizer()

		# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
		self.assertListEqual([tokenizer.tokenize(__A       ) for t in ["""Test""", """\xad""", """test"""]]				,			[["""[UNK]"""], [], ["""[UNK]"""]]       )

		self.assertListEqual(
		    [rust_tokenizer.tokenize(__A       ) for t in ["""Test""", """\xad""", """test"""]]				,			[["""[UNK]"""], [], ["""[UNK]"""]]       )




	@slow
	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
		__A   : Dict								= self.tokenizer_class.from_pretrained("""google/electra-base-discriminator"""       )

		__A   : str								= tokenizer.encode("""sequence builders"""				,			add_special_tokens=__A       )
		__A   : List[str]								= tokenizer.encode("""multi-sequence build"""				,			add_special_tokens=__A       )

		__A   : List[str]								= tokenizer.build_inputs_with_special_tokens(__A       )
		__A   : Union[str, Any]								= tokenizer.build_inputs_with_special_tokens(__A				,			__A       )

		assert encoded_sentence == [101] + text + [102]
		assert encoded_pair == [101] + text + [102] + text_a + [102]




	def 					lowerCAmelCase_					(    self	:    Union[str, Any]       ):
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
				__A   : Optional[Any]								= self.rust_tokenizer_class.from_pretrained(__A				,			**__A       )

				__A   : int								= F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
				__A   : int								= tokenizer_r.encode_plus(
				    __A				,			return_attention_mask=__A				,			return_token_type_ids=__A				,			return_offsets_mapping=__A				,			add_special_tokens=__A				,			)

				__A   : str								= tokenizer_r.do_lower_case if hasattr(__A				,			"""do_lower_case"""       ) else False
				__A   : Tuple								= (
				    [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), """A"""),
				        ((1, 2), ""","""),
				        ((3, 5), """na"""),
				        ((5, 6), """##ï"""),
				        ((6, 8), """##ve"""),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), """Allen"""),
				        ((21, 23), """##NL"""),
				        ((23, 24), """##P"""),
				        ((25, 33), """sentence"""),
				        ((33, 34), """."""),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				    if not do_lower_case
				    else [
				        ((0, 0), tokenizer_r.cls_token),
				        ((0, 1), """a"""),
				        ((1, 2), ""","""),
				        ((3, 8), """naive"""),
				        ((9, 15), tokenizer_r.mask_token),
				        ((16, 21), """allen"""),
				        ((21, 23), """##nl"""),
				        ((23, 24), """##p"""),
				        ((25, 33), """sentence"""),
				        ((33, 34), """."""),
				        ((0, 0), tokenizer_r.sep_token),
				    ]
				)

				self.assertEqual(
				    [e[1] for e in expected_results]				,			tokenizer_r.convert_ids_to_tokens(tokens["""input_ids"""]       )       )
				self.assertEqual([e[0] for e in expected_results]				,			tokens["""offset_mapping"""]       )




	def 					lowerCAmelCase_					(    self	:    List[Any]       ):
		__A   : str								= ["""的""", """人""", """有"""]
		__A   : Optional[int]								= """""".join(__A       )
		for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
			with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""       ):
				__A   : Optional[Any]								= True
				__A   : List[Any]								= self.tokenizer_class.from_pretrained(__A				,			**__A       )
				__A   : List[Any]								= self.rust_tokenizer_class.from_pretrained(__A				,			**__A       )

				__A   : str								= tokenizer_p.encode(__A				,			add_special_tokens=__A       )
				__A   : Union[str, Any]								= tokenizer_r.encode(__A				,			add_special_tokens=__A       )

				__A   : List[Any]								= tokenizer_r.convert_ids_to_tokens(__A       )
				__A   : Dict								= tokenizer_p.convert_ids_to_tokens(__A       )

				# it is expected that each Chinese character is not preceded by "##"
				self.assertListEqual(__A				,			__A       )
				self.assertListEqual(__A				,			__A       )

				__A   : int								= False
				__A   : Dict								= self.rust_tokenizer_class.from_pretrained(__A				,			**__A       )
				__A   : Tuple								= self.tokenizer_class.from_pretrained(__A				,			**__A       )

				__A   : Union[str, Any]								= tokenizer_r.encode(__A				,			add_special_tokens=__A       )
				__A   : Optional[Any]								= tokenizer_p.encode(__A				,			add_special_tokens=__A       )

				__A   : Dict								= tokenizer_r.convert_ids_to_tokens(__A       )
				__A   : Optional[Any]								= tokenizer_p.convert_ids_to_tokens(__A       )

				# it is expected that only the first Chinese character is not preceded by "##".
				__A   : Tuple								= [
				    F"""##{token}""" if idx != 0 else token for idx, token in enumerate(__A       )
				]
				self.assertListEqual(__A				,			__A       )
				self.assertListEqual(__A				,			__A       )








import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer






class     lowerCamelCase_			(  tf.keras.layers.Layer  ):




	def __init__(    self	:    int				,			__A	:    List				,			__A	:    bool				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    int = None				,			__A	:    str = "longest"				,			__A	:    bool = True				,			__A	:    int = 512				,			__A	:    int = None				,			__A	:    bool = True				,			__A	:    bool = True				,			__A	:    bool = True				,			**__A	:    Any				,			):
		super().__init__()
		if use_fast_bert_tokenizer:
			__A   : Optional[int]								= FastBertTokenizer(
			    __A				,			token_out_type=tf.intaa				,			lower_case_nfd_strip_accents=__A				,			**__A       )
		else:
			__A   : Optional[Any]								= tf.lookup.StaticVocabularyTable(
			    tf.lookup.KeyValueTensorInitializer(
			        keys=__A				,			key_dtype=tf.string				,			values=tf.range(tf.size(__A				,			out_type=tf.intaa       )				,			dtype=tf.intaa       )				,			value_dtype=tf.intaa				,			)				,			num_oov_buckets=1				,			)
			__A   : Any								= BertTokenizerLayer(
			    __A				,			token_out_type=tf.intaa				,			lower_case=__A				,			**__A       )

		__A   : Union[str, Any]								= vocab_list
		__A   : List[str]								= do_lower_case
		__A   : List[Any]								= cls_token_id or vocab_list.index("""[CLS]"""       )
		__A   : Tuple								= sep_token_id or vocab_list.index("""[SEP]"""       )
		__A   : str								= pad_token_id or vocab_list.index("""[PAD]"""       )
		__A   : Optional[int]								= ShrinkLongestTrimmer(max_length - 3				,			axis=1       )  # Allow room for special tokens
		__A   : Union[str, Any]								= max_length
		__A   : List[Any]								= padding
		__A   : Optional[int]								= truncation
		__A   : Tuple								= pad_to_multiple_of
		__A   : Optional[Any]								= return_token_type_ids
		__A   : int								= return_attention_mask




	@classmethod
	def 					lowerCAmelCase_					(    cls	:    List[Any]				,			__A	:    "PreTrainedTokenizerBase"				,			**__A	:    List[str]       ):  # noqa: F821
		__A   : Any								= kwargs.pop("""do_lower_case"""				,			__A       )
		__A   : Dict								= tokenizer.do_lower_case if do_lower_case is None else do_lower_case
		__A   : List[str]								= kwargs.pop("""cls_token_id"""				,			__A       )
		__A   : str								= tokenizer.cls_token_id if cls_token_id is None else cls_token_id
		__A   : str								= kwargs.pop("""sep_token_id"""				,			__A       )
		__A   : Dict								= tokenizer.sep_token_id if sep_token_id is None else sep_token_id
		__A   : Tuple								= kwargs.pop("""pad_token_id"""				,			__A       )
		__A   : List[Any]								= tokenizer.pad_token_id if pad_token_id is None else pad_token_id

		__A   : Union[str, Any]								= tokenizer.get_vocab()
		__A   : Any								= sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]				,			key=lambda __A       : x[1]       )
		__A   : int								= [entry[0] for entry in vocab]
		return cls(
		    vocab_list=__A				,			do_lower_case=__A				,			cls_token_id=__A				,			sep_token_id=__A				,			pad_token_id=__A				,			**__A				,			)




	@classmethod
	def 					lowerCAmelCase_					(    cls	:    List[str]				,			__A	:    Union[str, os.PathLike]				,			*__A	:    Tuple				,			**__A	:    Optional[int]       ):
		try:
			__A   : int								= BertTokenizer.from_pretrained(__A				,			*__A				,			**__A       )
		except:  # noqa: E722
			from .tokenization_bert_fast import BertTokenizerFast

			__A   : Optional[int]								= BertTokenizerFast.from_pretrained(__A				,			*__A				,			**__A       )
		return cls.from_tokenizer(__A				,			**__A       )




	def 					lowerCAmelCase_					(    self	:    Optional[int]				,			__A	:    Optional[Any]       ):
		if self.do_lower_case:
			__A   : Tuple								= case_fold_utfa(__A       )
		__A   : Optional[int]								= self.tf_tokenizer.tokenize(__A       )
		return tokens.merge_dims(1				,			-1       )




	def 					lowerCAmelCase_					(    self	:    Any				,			__A	:    Optional[Any]				,			__A	:    Dict=None				,			__A	:    List[Any]=None				,			__A	:    str=None				,			__A	:    Union[str, Any]=None				,			__A	:    Union[str, Any]=None				,			__A	:    int=None				,			__A	:    Any=None				,			):
		if padding is None:
			__A   : List[Any]								= self.padding
		if padding not in ("longest", "max_length"):
			raise ValueError("""Padding must be either 'longest' or 'max_length'!"""       )
		if max_length is not None and text_pair is not None:
			# Because we have to instantiate a Trimmer to do it properly
			raise ValueError("""max_length cannot be overridden at call time when truncating paired texts!"""       )
		if max_length is None:
			__A   : Any								= self.max_length
		if truncation is None:
			__A   : int								= self.truncation
		if pad_to_multiple_of is None:
			__A   : List[str]								= self.pad_to_multiple_of
		if return_token_type_ids is None:
			__A   : int								= self.return_token_type_ids
		if return_attention_mask is None:
			__A   : Any								= self.return_attention_mask
		if not isinstance(__A				,			tf.Tensor       ):
			__A   : str								= tf.convert_to_tensor(__A       )
		if text_pair is not None and not isinstance(__A				,			tf.Tensor       ):
			__A   : Optional[Any]								= tf.convert_to_tensor(__A       )
		if text_pair is not None:
			if text.shape.rank > 1:
				raise ValueError("""text argument should not be multidimensional when a text pair is supplied!"""       )
			if text_pair.shape.rank > 1:
				raise ValueError("""text_pair should not be multidimensional!"""       )
		if text.shape.rank == 2:
			__A						,							__A   : Optional[int]								= text[:, 0], text[:, 1]
		__A   : List[Any]								= self.unpaired_tokenize(__A       )
		if text_pair is None:  # Unpaired text
			if truncation:
				__A   : Union[str, Any]								= text[:, : max_length - 2]  # Allow room for special tokens
			__A						,							__A   : List[Any]								= combine_segments(
			    (text,)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
		else:  # Paired text
			__A   : Tuple								= self.unpaired_tokenize(__A       )
			if truncation:
				__A						,							__A   : Optional[int]								= self.paired_trimmer.trim([text, text_pair]       )
			__A						,							__A   : int								= combine_segments(
			    (text, text_pair)				,			start_of_sequence_id=self.cls_token_id				,			end_of_segment_id=self.sep_token_id       )
		if padding == "longest":
			__A   : str								= input_ids.bounding_shape(axis=1       )
			if pad_to_multiple_of is not None:
				# No ceiling division in tensorflow, so we negate floordiv instead
				__A   : Optional[Any]								= pad_to_multiple_of * (-tf.math.floordiv(-pad_length				,			__A       ))
		else:
			__A   : Dict								= max_length

		__A						,							__A   : str								= pad_model_inputs(__A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
		__A   : List[str]								= {"""input_ids""": input_ids}
		if return_attention_mask:
			__A   : Any								= attention_mask
		if return_token_type_ids:
			__A						,							__A   : Tuple								= pad_model_inputs(
			    __A				,			max_seq_length=__A				,			pad_value=self.pad_token_id       )
			__A   : List[str]								= token_type_ids
		return output




	def 					lowerCAmelCase_					(    self	:    Optional[int]       ):
		return {
		    "vocab_list": self.vocab_list,
		    "do_lower_case": self.do_lower_case,
		    "cls_token_id": self.cls_token_id,
		    "sep_token_id": self.sep_token_id,
		    "pad_token_id": self.pad_token_id,
		}





