



'''simple docstring'''
import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_ :	List[str]         =  get_tests_dir('fixtures/spiece.model')






@require_sentencepiece
@require_tokenizers
class 							lowercase__	(							lowercase						,					unittest.TestCase   ):
	lowercase__  				=			DebertaVaTokenizer
	lowercase__  				=			DebertaVaTokenizerFast
	lowercase__  				=			True
	lowercase__  				=			True





	def 	UpperCamelCase_  ( self					:				Dict ):







							'''simple docstring'''
							super().setUp()

							# We have a SentencePiece fixture for testing
							_UpperCamelCase					:		Dict 				=   DebertaVaTokenizer(A_ ,unk_token='<unk>' )
							tokenizer.save_pretrained(self.tmpdirname )





	def 	UpperCamelCase_  ( self					:				List[Any] ,lowerCamelCase__					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		Optional[Any] 				=   'this is a test'
							_UpperCamelCase					:		List[Any] 				=   'this is a test'
							return input_text, output_text





	def 	UpperCamelCase_  ( self					:				Dict ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   '<pad>'
							_UpperCamelCase					:		Dict 				=   0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(A_ ) ,A_ )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(A_ ) ,A_ )





	def 	UpperCamelCase_  ( self					:				Optional[int] ):







							'''simple docstring'''
							_UpperCamelCase					:		List[Any] 				=   list(self.get_tokenizer().get_vocab().keys() )
							self.assertEqual(vocab_keys[0] ,'<pad>' )
							self.assertEqual(vocab_keys[1] ,'<unk>' )
							self.assertEqual(vocab_keys[-1] ,'[PAD]' )
							self.assertEqual(len(A_ ) ,30001 )





	def 	UpperCamelCase_  ( self					:				Dict ):







							'''simple docstring'''
							self.assertEqual(self.get_tokenizer().vocab_size ,30000 )





	def 	UpperCamelCase_  ( self					:				Tuple ):







							'''simple docstring'''
							_UpperCamelCase					:		Tuple 				=   ' \tHeLLo!how  \n Are yoU?  '
							_UpperCamelCase					:		Optional[int] 				=   ['▁hello', '!', 'how', '▁are', '▁you', '?']
							# fmt: on

							_UpperCamelCase					:		List[Any] 				=   DebertaVaTokenizer(A_ ,do_lower_case=A_ )
							_UpperCamelCase					:		Dict 				=   tokenizer.convert_ids_to_tokens(tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		str 				=   DebertaVaTokenizerFast(A_ ,do_lower_case=A_ )
							_UpperCamelCase					:		List[str] 				=   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )





	@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.' )
	def 	UpperCamelCase_  ( self					:				Optional[Any] ):







							'''simple docstring'''
							pass





	@unittest.skip('There is an inconsistency between slow and fast tokenizer due to a bug in the fast one.' )
	def 	UpperCamelCase_  ( self					:				Union[str, Any] ):







							'''simple docstring'''
							pass





	def 	UpperCamelCase_  ( self					:				Union[str, Any] ):







							'''simple docstring'''
							_UpperCamelCase					:		Dict 				=   'I was born in 92000, and this is falsé.'
							_UpperCamelCase					:		Tuple 				=   ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
							# fmt: on

							_UpperCamelCase					:		Any 				=   DebertaVaTokenizer(A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		int 				=   tokenizer.convert_ids_to_tokens(tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		Tuple 				=   DebertaVaTokenizerFast(A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Union[str, Any] 				=   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )





	def 	UpperCamelCase_  ( self					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   'I was born in 92000, and this is falsé.'
							_UpperCamelCase					:		Tuple 				=   ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
							# fmt: on

							_UpperCamelCase					:		Any 				=   DebertaVaTokenizer(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Tuple 				=   tokenizer.convert_ids_to_tokens(tokenizer.encode(A_ ,add_special_tokens=A_ ) )
							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		Dict 				=   DebertaVaTokenizerFast(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Union[str, Any] 				=   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_ ,add_special_tokens=A_ ) )
							self.assertListEqual(A_ ,A_ )





	def 	UpperCamelCase_  ( self					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		Optional[int] 				=   'I was born in 92000, and this is falsé.'
							_UpperCamelCase					:		Optional[Any] 				=   ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
							# fmt: on

							_UpperCamelCase					:		Union[str, Any] 				=   DebertaVaTokenizer(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Union[str, Any] 				=   tokenizer.convert_ids_to_tokens(tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		Union[str, Any] 				=   DebertaVaTokenizerFast(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		str 				=   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )





	def 	UpperCamelCase_  ( self					:				int ):







							'''simple docstring'''
							_UpperCamelCase					:		Union[str, Any] 				=   'I was born in 92000, and this is falsé.'
							_UpperCamelCase					:		Optional[int] 				=   ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', '▁', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '▁', '.', ]
							# fmt: on

							_UpperCamelCase					:		List[str] 				=   DebertaVaTokenizer(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Union[str, Any] 				=   tokenizer.convert_ids_to_tokens(tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		Optional[Any] 				=   DebertaVaTokenizerFast(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Any 				=   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )





	def 	UpperCamelCase_  ( self					:				str ):







							'''simple docstring'''
							_UpperCamelCase					:		Optional[int] 				=   ' \tHeLLo!how  \n Are yoU?  '
							_UpperCamelCase					:		List[Any] 				=   ['▁', '<unk>', 'e', '<unk>', 'o', '!', 'how', '▁', '<unk>', 're', '▁yo', '<unk>', '?']
							# fmt: on

							_UpperCamelCase					:		Dict 				=   DebertaVaTokenizer(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Tuple 				=   tokenizer.convert_ids_to_tokens(tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		str 				=   DebertaVaTokenizerFast(A_ ,do_lower_case=A_ ,split_by_punct=A_ )
							_UpperCamelCase					:		Optional[int] 				=   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_ ,add_special_tokens=A_ ) )

							self.assertListEqual(A_ ,A_ )





	def 	UpperCamelCase_  ( self					:				Optional[int] ):







							'''simple docstring'''
							_UpperCamelCase					:		List[str] 				=   self.get_tokenizer()
							_UpperCamelCase					:		Dict 				=   self.get_rust_tokenizer()

							_UpperCamelCase					:		Any 				=   'I was born in 92000, and this is falsé.'

							_UpperCamelCase					:		Tuple 				=   tokenizer.convert_ids_to_tokens(tokenizer.encode(A_ ,add_special_tokens=A_ ) )
							_UpperCamelCase					:		Dict 				=   rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(A_ ,add_special_tokens=A_ ) )
							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		int 				=   tokenizer.encode(A_ ,add_special_tokens=A_ )
							_UpperCamelCase					:		Any 				=   rust_tokenizer.encode(A_ ,add_special_tokens=A_ )
							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		Optional[Any] 				=   self.get_rust_tokenizer()
							_UpperCamelCase					:		List[str] 				=   tokenizer.encode(A_ )
							_UpperCamelCase					:		str 				=   rust_tokenizer.encode(A_ )
							self.assertListEqual(A_ ,A_ )





	def 	UpperCamelCase_  ( self					:				Optional[Any] ):







							'''simple docstring'''
							_UpperCamelCase					:		Optional[Any] 				=   'This is a test'
							_UpperCamelCase					:		List[Any] 				=   [13, 1, 4398, 25, 21, 1289]
							_UpperCamelCase					:		str 				=   ['▁', 'T', 'his', '▁is', '▁a', '▁test']
							_UpperCamelCase					:		Any 				=   ['▁', '<unk>', 'his', '▁is', '▁a', '▁test']

							_UpperCamelCase					:		int 				=   DebertaVaTokenizer(A_ ,keep_accents=A_ )
							_UpperCamelCase					:		Optional[int] 				=   DebertaVaTokenizerFast(A_ ,keep_accents=A_ )

							_UpperCamelCase					:		Any 				=   tokenizer.encode(A_ ,add_special_tokens=A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		str 				=   tokenizer.tokenize(A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		Any 				=   tokenizer.convert_ids_to_tokens(A_ )
							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		Optional[Any] 				=   rust_tokenizer.encode(A_ ,add_special_tokens=A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		str 				=   rust_tokenizer.tokenize(A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		Dict 				=   rust_tokenizer.convert_ids_to_tokens(A_ )
							self.assertListEqual(A_ ,A_ )

							# fmt: off
							_UpperCamelCase					:		Optional[int] 				=   'I was born in 92000, and this is falsé.'
							_UpperCamelCase					:		Optional[int] 				=   [13, 1, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]
							_UpperCamelCase					:		str 				=   ['▁', 'I', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.', ]
							_UpperCamelCase					:		List[Any] 				=   ['▁', '<unk>', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.', ]
							# fmt: on

							_UpperCamelCase					:		List[Any] 				=   tokenizer.encode(A_ ,add_special_tokens=A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		Union[str, Any] 				=   tokenizer.tokenize(A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		str 				=   tokenizer.convert_ids_to_tokens(A_ )
							self.assertListEqual(A_ ,A_ )

							_UpperCamelCase					:		str 				=   rust_tokenizer.encode(A_ ,add_special_tokens=A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		str 				=   rust_tokenizer.tokenize(A_ )
							self.assertListEqual(A_ ,A_ )
							_UpperCamelCase					:		int 				=   rust_tokenizer.convert_ids_to_tokens(A_ )
							self.assertListEqual(A_ ,A_ )





	def 	UpperCamelCase_  ( self					:				Any ):







							'''simple docstring'''
							_UpperCamelCase					:		List[Any] 				=   DebertaVaTokenizer(A_ )

							_UpperCamelCase					:		Optional[int] 				=   tokenizer.encode('sequence builders' )
							_UpperCamelCase					:		str 				=   tokenizer.encode('multi-sequence build' )

							_UpperCamelCase					:		List[str] 				=   tokenizer.build_inputs_with_special_tokens(A_ )
							_UpperCamelCase					:		List[Any] 				=   tokenizer.build_inputs_with_special_tokens(A_ ,A_ )

							self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] ,A_ )
							self.assertEqual(
							    [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id] ,A_ ,)





	@slow
	def 	UpperCamelCase_  ( self					:				Any ):







							'''simple docstring'''
							_UpperCamelCase					:		Union[str, Any] 				=   {'input_ids': [[1, 39867, 36, 19390, 486, 27, 35052, 81436, 18, 60685, 1225, 7, 35052, 81436, 18, 9367, 16899, 18, 15937, 53, 594, 773, 18, 16287, 30465, 36, 15937, 6, 41139, 38, 36979, 60763, 191, 6, 34132, 99, 6, 50538, 390, 43230, 6, 34132, 2779, 20850, 14, 699, 1072, 1194, 36, 382, 10901, 53, 7, 699, 1072, 2084, 36, 20422, 630, 53, 19, 105, 3049, 1896, 1053, 16899, 1506, 11, 37978, 4243, 7, 1237, 31869, 200, 16566, 654, 6, 35052, 81436, 7, 55630, 13593, 4, 2], [1, 26, 15011, 13, 667, 8, 1053, 18, 23611, 1237, 72356, 12820, 34, 104134, 1209, 35, 13313, 6627, 21, 202, 347, 7, 164, 2399, 11, 46, 4485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1232, 2864, 15785, 14951, 105, 5, 8581, 1250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=A_ ,model_name='microsoft/deberta-v2-xlarge' ,revision='ad6e42c1532ddf3a15c39246b63f5559d558b670' ,)








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowerCamelCase				:		str     =   {
    "configuration_nezha": ["NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP", "NezhaConfig"],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 lowerCamelCase				:		List[str]     =   [
     "NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST",
     "NezhaForNextSentencePrediction",
     "NezhaForMaskedLM",
     "NezhaForPreTraining",
     "NezhaForMultipleChoice",
     "NezhaForQuestionAnswering",
     "NezhaForSequenceClassification",
     "NezhaForTokenClassification",
     "NezhaModel",
     "NezhaPreTrainedModel",
 ]


if TYPE_CHECKING:
 from .configuration_nezha import NEZHA_PRETRAINED_CONFIG_ARCHIVE_MAP, NezhaConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_nezha import (
      NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST,
      NezhaForMaskedLM,
      NezhaForMultipleChoice,
      NezhaForNextSentencePrediction,
      NezhaForPreTraining,
      NezhaForQuestionAnswering,
      NezhaForSequenceClassification,
      NezhaForTokenClassification,
      NezhaModel,
      NezhaPreTrainedModel,
  )


else:
 import sys

 lowerCamelCase				:		str     =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



