

"""simple docstring"""





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


__UpperCamelCase  :      List[str]							      =		logging.get_logger(__name__)


__UpperCamelCase  :      Dict							      =		{
    '''Salesforce/codegen-350M-nl''': '''https://huggingface.co/Salesforce/codegen-350M-nl/resolve/main/config.json''',
    '''Salesforce/codegen-350M-multi''': '''https://huggingface.co/Salesforce/codegen-350M-multi/resolve/main/config.json''',
    '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/config.json''',
    '''Salesforce/codegen-2B-nl''': '''https://huggingface.co/Salesforce/codegen-2B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-2B-multi''': '''https://huggingface.co/Salesforce/codegen-2B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-2B-mono''': '''https://huggingface.co/Salesforce/codegen-2B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-6B-nl''': '''https://huggingface.co/Salesforce/codegen-6B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-6B-multi''': '''https://huggingface.co/Salesforce/codegen-6B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-6B-mono''': '''https://huggingface.co/Salesforce/codegen-6B-mono/resolve/main/config.json''',
    '''Salesforce/codegen-16B-nl''': '''https://huggingface.co/Salesforce/codegen-16B-nl/resolve/main/config.json''',
    '''Salesforce/codegen-16B-multi''': '''https://huggingface.co/Salesforce/codegen-16B-multi/resolve/main/config.json''',
    '''Salesforce/codegen-16B-mono''': '''https://huggingface.co/Salesforce/codegen-16B-mono/resolve/main/config.json''',
}







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					lowercase__				 =       "codegen"
					lowercase__				 =       {
					    "max_position_embeddings": "n_positions",
					    "hidden_size": "n_embd",
					    "num_attention_heads": "n_head",
					    "num_hidden_layers": "n_layer",
					}







					def __init__(       self			:				Tuple   ,lowercase_			:				Tuple=5_0_4_0_0   ,lowercase_			:				Any=2_0_4_8   ,lowercase_			:				Union[str, Any]=2_0_4_8   ,lowercase_			:				List[str]=4_0_9_6   ,lowercase_			:				Dict=2_8   ,lowercase_			:				List[Any]=1_6   ,lowercase_			:				Tuple=6_4   ,lowercase_			:				str=None   ,lowercase_			:				List[str]="gelu_new"   ,lowercase_			:				int=0.0   ,lowercase_			:				Optional[Any]=0.0   ,lowercase_			:				Union[str, Any]=0.0   ,lowercase_			:				Tuple=1E-5   ,lowercase_			:				Union[str, Any]=0.02   ,lowercase_			:				str=True   ,lowercase_			:				Any=5_0_2_5_6   ,lowercase_			:				List[str]=5_0_2_5_6   ,lowercase_			:				List[str]=False   ,**lowercase_			:				Tuple   ,):
						lowerCAmelCase__	:						str  				=	vocab_size
						lowerCAmelCase__	:						Union[str, Any]  				=	n_ctx
						lowerCAmelCase__	:						Optional[int]  				=	n_positions
						lowerCAmelCase__	:						Dict  				=	n_embd
						lowerCAmelCase__	:						Any  				=	n_layer
						lowerCAmelCase__	:						Union[str, Any]  				=	n_head
						lowerCAmelCase__	:						List[str]  				=	n_inner
						lowerCAmelCase__	:						Optional[Any]  				=	rotary_dim
						lowerCAmelCase__	:						Union[str, Any]  				=	activation_function
						lowerCAmelCase__	:						Optional[int]  				=	resid_pdrop
						lowerCAmelCase__	:						Optional[Any]  				=	embd_pdrop
						lowerCAmelCase__	:						List[str]  				=	attn_pdrop
						lowerCAmelCase__	:						Union[str, Any]  				=	layer_norm_epsilon
						lowerCAmelCase__	:						Optional[int]  				=	initializer_range
						lowerCAmelCase__	:						Tuple  				=	use_cache

						lowerCAmelCase__	:						List[str]  				=	bos_token_id
						lowerCAmelCase__	:						Optional[Any]  				=	eos_token_id

						super().__init__(
						    bos_token_id=lowercase_   ,eos_token_id=lowercase_   ,tie_word_embeddings=lowercase_   ,**lowercase_		)







class 				SCREAMING_SNAKE_CASE					(		a_       ):






					"""simple docstring"""



					def __init__(       self			:				Optional[int]   ,lowercase_			:				PretrainedConfig   ,lowercase_			:				str = "default"   ,lowercase_			:				List[PatchingSpec] = None   ,lowercase_			:				bool = False   ,):
						super().__init__(lowercase_   ,task=lowercase_   ,patching_specs=lowercase_   ,use_past=lowercase_		)
						if not getattr(self._config   ,'''pad_token_id'''   ,lowercase_		):
							# TODO: how to do that better?
							lowerCAmelCase__	:						List[Any]  				=	0







					@property
					def        __lowerCAmelCase (       self			:				Tuple		):
						lowerCAmelCase__	:						Union[str, Any]  				=	OrderedDict({'''input_ids''': {0: '''batch''', 1: '''sequence'''}}		)
						if self.use_past:
							self.fill_with_past_key_values_(lowercase_   ,direction='''inputs'''		)
							lowerCAmelCase__	:						Union[str, Any]  				=	{0: '''batch''', 1: '''past_sequence + sequence'''}
						else:
							lowerCAmelCase__	:						List[Any]  				=	{0: '''batch''', 1: '''sequence'''}

						return common_inputs







					@property
					def        __lowerCAmelCase (       self			:				int		):
						return self._config.n_layer







					@property
					def        __lowerCAmelCase (       self			:				List[Any]		):
						return self._config.n_head







					def        __lowerCAmelCase (       self			:				Optional[Any]   ,lowercase_			:				PreTrainedTokenizer   ,lowercase_			:				int = -1   ,lowercase_			:				int = -1   ,lowercase_			:				bool = False   ,lowercase_			:				Optional[TensorType] = None   ,):
						lowerCAmelCase__	:						str  				=	super(lowercase_   ,self		).generate_dummy_inputs(
						    lowercase_   ,batch_size=lowercase_   ,seq_length=lowercase_   ,is_pair=lowercase_   ,framework=lowercase_		)

						# We need to order the input in the way they appears in the forward()
						lowerCAmelCase__	:						Any  				=	OrderedDict({'''input_ids''': common_inputs['''input_ids''']}		)

						# Need to add the past_keys
						if self.use_past:
							if not is_torch_available():
								raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''		)
							else:
								import torch

								lowerCAmelCase__				,lowerCAmelCase__	:						Union[str, Any]  				=	common_inputs['''input_ids'''].shape
								# Not using the same length for past_key_values
								lowerCAmelCase__	:						Optional[Any]  				=	seqlen + 2
								lowerCAmelCase__	:						Union[str, Any]  				=	(
								    batch,
								    self.num_attention_heads,
								    past_key_values_length,
								    self._config.hidden_size // self.num_attention_heads,
								)
								lowerCAmelCase__	:						Dict  				=	[
								    (torch.zeros(lowercase_		), torch.zeros(lowercase_		)) for _ in range(self.num_layers		)
								]

						lowerCAmelCase__	:						Any  				=	common_inputs['''attention_mask''']
						if self.use_past:
							lowerCAmelCase__	:						List[str]  				=	ordered_inputs['''attention_mask'''].dtype
							lowerCAmelCase__	:						Dict  				=	torch.cat(
							    [ordered_inputs['''attention_mask'''], torch.ones(lowercase_   ,lowercase_   ,dtype=lowercase_		)]   ,dim=1		)

						return ordered_inputs







					@property
					def        __lowerCAmelCase (       self			:				Any		):
						return 1_3






"""simple docstring"""




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_A			:       int   		=       get_tests_dir("""fixtures/spiece.model""")

@require_sentencepiece
@require_tokenizers
class 		a__						(      a_,			unittest.TestCase						):
   __lowerCAmelCase            =				DebertaVaTokenizer
   __lowerCAmelCase            =				DebertaVaTokenizerFast
   __lowerCAmelCase            =				True
   __lowerCAmelCase            =				True
   def 			__magic_name__			(			self				):
          super().setUp()

          # We have a SentencePiece fixture for testing
          lowercase    :				Any       		=							DebertaVaTokenizer(_a	,    unk_token="<unk>"				)
          tokenizer.save_pretrained(self.tmpdirname				)
   def 			__magic_name__			(			self	,    _a				):
          lowercase    :				int       		=							"this is a test"
          lowercase    :				Tuple       		=							"this is a test"
          return input_text, output_text
   def 			__magic_name__			(			self				):
          lowercase    :				List[Any]       		=							"<pad>"
          lowercase    :				List[str]       		=							0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a				)	,    _a				)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a				)	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				int       		=							list(self.get_tokenizer().get_vocab().keys()				)
          self.assertEqual(vocab_keys[0]	,    "<pad>"				)
          self.assertEqual(vocab_keys[1]	,    "<unk>"				)
          self.assertEqual(vocab_keys[-1]	,    "[PAD]"				)
          self.assertEqual(len(_a				)	,    30_001				)
   def 			__magic_name__			(			self				):
          self.assertEqual(self.get_tokenizer().vocab_size	,    30_000				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				List[str]       		=							" \tHeLLo!how  \n Are yoU?  "
          lowercase    :				str       		=							["▁hello", "!", "how", "▁are", "▁you", "?"]
          # fmt: on

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a				)
          lowercase    :				Any       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				str       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a				)
          lowercase    :				Optional[Any]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."				)
   def 			__magic_name__			(			self				):
          pass
   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."				)
   def 			__magic_name__			(			self				):
          pass
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Optional[Any]       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Tuple       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				List[Any]       		=							DebertaVaTokenizer(_a	,    split_by_punct=_a				)
          lowercase    :				Tuple       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    split_by_punct=_a				)
          lowercase    :				Any       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Tuple       		=							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				List[str]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[Any]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[int]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				List[Any]       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Optional[int]       		=							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          lowercase    :				List[Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[int]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Dict       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Any       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Dict       		=							" \tHeLLo!how  \n Are yoU?  "
          lowercase    :				str       		=							["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
          # fmt: on

          lowercase    :				Optional[int]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				int       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				List[str]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				str       		=							self.get_tokenizer()
          lowercase    :				Dict       		=							self.get_rust_tokenizer()

          lowercase    :				str       		=							"I was born in 92000, and this is falsé."

          lowercase    :				Optional[Any]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				str       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							self.get_rust_tokenizer()
          lowercase    :				Tuple       		=							tokenizer.encode(_a				)
          lowercase    :				List[str]       		=							rust_tokenizer.encode(_a				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				str       		=							"This is a test"
          lowercase    :				Tuple       		=							[13, 1, 4_398, 25, 21, 1_289]
          lowercase    :				Optional[int]       		=							["▁", "T", "his", "▁is", "▁a", "▁test"]
          lowercase    :				Optional[Any]       		=							["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

          lowercase    :				Any       		=							DebertaVaTokenizer(_a	,    keep_accents=_a				)
          lowercase    :				Dict       		=							DebertaVaTokenizerFast(_a	,    keep_accents=_a				)

          lowercase    :				str       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				int       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Any       		=							rust_tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Union[str, Any]       		=							rust_tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Any       		=							[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
          lowercase    :				List[str]       		=							["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
          lowercase    :				str       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          lowercase    :				Tuple       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				Optional[int]       		=							DebertaVaTokenizer(_a				)

          lowercase    :				List[Any]       		=							tokenizer.encode("sequence builders"				)
          lowercase    :				Dict       		=							tokenizer.encode("multi-sequence build"				)

          lowercase    :				List[Any]       		=							tokenizer.build_inputs_with_special_tokens(_a				)
          lowercase    :				Dict       		=							tokenizer.build_inputs_with_special_tokens(_a	,    _a				)

          self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]	,    _a				)
          self.assertEqual(
              [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]	,    _a	,    )

   @slow
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Dict       		=							{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_a	,    model_name="microsoft/deberta-v2-xlarge"	,    revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"	,    )


