


from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers.models.xglm.modeling_tf_xglm import (
       TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
       TFXGLMForCausalLM,
       TFXGLMModel,
   )






@require_tf
class     UpperCAmelCase__       :



    """simple docstring"""






    UpperCAmelCase__						:		List[Any]      =  XGLMConfig
    UpperCAmelCase__						:		List[str]      =  {}
    UpperCAmelCase__						:		Any      =  "gelu"






    def __init__(			self					,   A_					,   A_=14					,   A_=7					,   A_=True					,   A_=True					,   A_=True					,   A_=99					,   A_=32					,   A_=2					,   A_=4					,   A_=37					,   A_="gelu"					,   A_=0.1					,   A_=0.1					,   A_=512					,   A_=0.02					,   )  ->       Optional[Any]:
         __UpperCamelCase    							=parent
         __UpperCamelCase    							=batch_size
         __UpperCamelCase    							=seq_length
         __UpperCamelCase    							=is_training
         __UpperCamelCase    							=use_input_mask
         __UpperCamelCase    							=use_labels
         __UpperCamelCase    							=vocab_size
         __UpperCamelCase    							=d_model
         __UpperCamelCase    							=num_hidden_layers
         __UpperCamelCase    							=num_attention_heads
         __UpperCamelCase    							=ffn_dim
         __UpperCamelCase    							=activation_function
         __UpperCamelCase    							=activation_dropout
         __UpperCamelCase    							=attention_dropout
         __UpperCamelCase    							=max_position_embeddings
         __UpperCamelCase    							=initializer_range
         __UpperCamelCase    							=None
         __UpperCamelCase    							=0
         __UpperCamelCase    							=2
         __UpperCamelCase    							=1






    def _a					(			self    )  ->       Optional[Any]:
         return XGLMConfig.from_pretrained('facebook/xglm-564M'    )






    def _a					(			self    )  ->       Dict:
         __UpperCamelCase    							=tf.clip_by_value(
             ids_tensor([self.batch_size, self.seq_length]					,   self.vocab_size    )					,   clip_value_min=0					,   clip_value_max=3    )

         __UpperCamelCase    							=None
         if self.use_input_mask:
              __UpperCamelCase    							=random_attention_mask([self.batch_size, self.seq_length]    )

         __UpperCamelCase    							=self.get_config()

         __UpperCamelCase    							=floats_tensor([self.num_hidden_layers, self.num_attention_heads]					,   2    )

         return (
             config,
             input_ids,
             input_mask,
             head_mask,
         )






    def _a					(			self    )  ->       Optional[int]:
         return XGLMConfig(
             vocab_size=self.vocab_size					,   d_model=self.hidden_size					,   num_layers=self.num_hidden_layers					,   attention_heads=self.num_attention_heads					,   ffn_dim=self.ffn_dim					,   activation_function=self.activation_function					,   activation_dropout=self.activation_dropout					,   attention_dropout=self.attention_dropout					,   max_position_embeddings=self.max_position_embeddings					,   initializer_range=self.initializer_range					,   use_cache=A_					,   bos_token_id=self.bos_token_id					,   eos_token_id=self.eos_token_id					,   pad_token_id=self.pad_token_id					,   return_dict=A_					,   )






    def _a					(			self    )  ->       Any:
         __UpperCamelCase    							=self.prepare_config_and_inputs()

         (
             (
             __UpperCamelCase
         )							,      (
             __UpperCamelCase
         )							,      (
             __UpperCamelCase
         )							,      (
             __UpperCamelCase
         )							,      
         )    							=config_and_inputs

         __UpperCamelCase    							={
             'input_ids': input_ids,
             'head_mask': head_mask,
         }

         return config, inputs_dict






@require_tf
class     UpperCAmelCase__       (					A_   ,  A_   ,  unittest.TestCase		):



    """simple docstring"""






    UpperCAmelCase__						:		int      =  (TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
    UpperCAmelCase__						:		Optional[Any]      =  (TFXGLMForCausalLM,) if is_tf_available() else ()
    UpperCAmelCase__						:		Dict      =  (
        {"feature-extraction": TFXGLMModel, "text-generation": TFXGLMForCausalLM} if is_tf_available() else {}
    )
    UpperCAmelCase__						:		Tuple      =  False
    UpperCAmelCase__						:		Optional[int]      =  False
    UpperCAmelCase__						:		Optional[Any]      =  False






    def _a					(			self    )  ->       Tuple:
         __UpperCamelCase    							=TFXGLMModelTester(self    )
         __UpperCamelCase    							=ConfigTester(self					,   config_class=A_					,   n_embd=37    )






    def _a					(			self    )  ->       Any:
         self.config_tester.run_common_tests()






    @slow
    def _a					(			self    )  ->       Optional[int]:
         for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
              __UpperCamelCase    							=TFXGLMModel.from_pretrained(A_    )
              self.assertIsNotNone(A_    )






    @unittest.skip(reason='Currently, model embeddings are going to undergo a major refactor.'    )
    def _a					(			self    )  ->       str:
         super().test_resize_token_embeddings()






@require_tf
class     UpperCAmelCase__       (					unittest.TestCase		):



    """simple docstring"""






    @slow
    def _a					(			self					,   A_=True    )  ->       int:
         __UpperCamelCase    							=TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'    )
         __UpperCamelCase    							=tf.convert_to_tensor([[2, 268, 9865]]					,   dtype=tf.intaa    )  # The dog
         # </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
         # fmt: off
         __UpperCamelCase    							=[2, 268, 9865, 67, 11, 1988, 57252, 9865, 5, 984, 67, 1988, 213838, 1658, 53, 70446, 33, 6657, 278, 1581]
         # fmt: on
         __UpperCamelCase    							=model.generate(A_					,   do_sample=A_					,   num_beams=1    )
         if verify_outputs:
              self.assertListEqual(output_ids[0].numpy().tolist()					,   A_    )






    @slow
    def _a					(			self    )  ->       Any:
         __UpperCamelCase    							=XGLMTokenizer.from_pretrained('facebook/xglm-564M'    )
         __UpperCamelCase    							=TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'    )

         tf.random.set_seed(0    )
         __UpperCamelCase    							=tokenizer('Today is a nice day and'					,   return_tensors='tf'    )
         __UpperCamelCase    							=tokenized.input_ids
         # forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
         with tf.device(':/CPU:0'    ):
              __UpperCamelCase    							=model.generate(A_					,   do_sample=A_					,   seed=[7, 0]    )
         __UpperCamelCase    							=tokenizer.decode(output_ids[0]					,   skip_special_tokens=A_    )

         __UpperCamelCase    							=(
             'Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due'
         )
         self.assertEqual(A_					,   A_    )






    @slow
    def _a					(			self    )  ->       Optional[Any]:
         __UpperCamelCase    							=TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'    )
         __UpperCamelCase    							=XGLMTokenizer.from_pretrained('facebook/xglm-564M'    )

         __UpperCamelCase    							='left'

         # use different length sentences to test batching
         __UpperCamelCase    							=[
             'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
             'left-padding, such as in batched generation. The output for the sequence below should be the same '
             'regardless of whether left padding is applied or not. When',
             'Hello, my dog is a little',
         ]

         __UpperCamelCase    							=tokenizer(A_					,   return_tensors='tf'					,   padding=A_    )
         __UpperCamelCase    							=inputs['input_ids']

         __UpperCamelCase    							=model.generate(input_ids=A_					,   attention_mask=inputs['attention_mask']					,   max_new_tokens=12    )

         __UpperCamelCase    							=tokenizer(sentences[0]					,   return_tensors='tf'    ).input_ids
         __UpperCamelCase    							=model.generate(input_ids=A_					,   max_new_tokens=12    )

         __UpperCamelCase    							=tokenizer(sentences[1]					,   return_tensors='tf'    ).input_ids
         __UpperCamelCase    							=model.generate(input_ids=A_					,   max_new_tokens=12    )

         __UpperCamelCase    							=tokenizer.batch_decode(A_					,   skip_special_tokens=A_    )
         __UpperCamelCase    							=tokenizer.decode(output_non_padded[0]					,   skip_special_tokens=A_    )
         __UpperCamelCase    							=tokenizer.decode(output_padded[0]					,   skip_special_tokens=A_    )

         __UpperCamelCase    							=[
             'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
             'left-padding, such as in batched generation. The output for the sequence below should be the same '
             'regardless of whether left padding is applied or not. When left padding is applied, the sequence will be '
             'a single',
             'Hello, my dog is a little bit of a shy one, but he is very friendly',
         ]
         self.assertListEqual(A_					,   A_    )
         self.assertListEqual(A_					,   [non_padded_sentence, padded_sentence]    )







"""simple docstring"""




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_A			:       int   		=       get_tests_dir("""fixtures/spiece.model""")

@require_sentencepiece
@require_tokenizers
class 		a__						(      a_,			unittest.TestCase						):
   __lowerCAmelCase            =				DebertaVaTokenizer
   __lowerCAmelCase            =				DebertaVaTokenizerFast
   __lowerCAmelCase            =				True
   __lowerCAmelCase            =				True
   def 			__magic_name__			(			self				):
          super().setUp()

          # We have a SentencePiece fixture for testing
          lowercase    :				Any       		=							DebertaVaTokenizer(_a	,    unk_token="<unk>"				)
          tokenizer.save_pretrained(self.tmpdirname				)
   def 			__magic_name__			(			self	,    _a				):
          lowercase    :				int       		=							"this is a test"
          lowercase    :				Tuple       		=							"this is a test"
          return input_text, output_text
   def 			__magic_name__			(			self				):
          lowercase    :				List[Any]       		=							"<pad>"
          lowercase    :				List[str]       		=							0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a				)	,    _a				)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a				)	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				int       		=							list(self.get_tokenizer().get_vocab().keys()				)
          self.assertEqual(vocab_keys[0]	,    "<pad>"				)
          self.assertEqual(vocab_keys[1]	,    "<unk>"				)
          self.assertEqual(vocab_keys[-1]	,    "[PAD]"				)
          self.assertEqual(len(_a				)	,    30_001				)
   def 			__magic_name__			(			self				):
          self.assertEqual(self.get_tokenizer().vocab_size	,    30_000				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				List[str]       		=							" \tHeLLo!how  \n Are yoU?  "
          lowercase    :				str       		=							["▁hello", "!", "how", "▁are", "▁you", "?"]
          # fmt: on

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a				)
          lowercase    :				Any       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				str       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a				)
          lowercase    :				Optional[Any]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."				)
   def 			__magic_name__			(			self				):
          pass
   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."				)
   def 			__magic_name__			(			self				):
          pass
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Optional[Any]       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Tuple       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				List[Any]       		=							DebertaVaTokenizer(_a	,    split_by_punct=_a				)
          lowercase    :				Tuple       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    split_by_punct=_a				)
          lowercase    :				Any       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Tuple       		=							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				List[str]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[Any]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[int]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				List[Any]       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Optional[int]       		=							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          lowercase    :				List[Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[int]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Dict       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Any       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Dict       		=							" \tHeLLo!how  \n Are yoU?  "
          lowercase    :				str       		=							["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
          # fmt: on

          lowercase    :				Optional[int]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				int       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				List[str]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				str       		=							self.get_tokenizer()
          lowercase    :				Dict       		=							self.get_rust_tokenizer()

          lowercase    :				str       		=							"I was born in 92000, and this is falsé."

          lowercase    :				Optional[Any]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				str       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							self.get_rust_tokenizer()
          lowercase    :				Tuple       		=							tokenizer.encode(_a				)
          lowercase    :				List[str]       		=							rust_tokenizer.encode(_a				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				str       		=							"This is a test"
          lowercase    :				Tuple       		=							[13, 1, 4_398, 25, 21, 1_289]
          lowercase    :				Optional[int]       		=							["▁", "T", "his", "▁is", "▁a", "▁test"]
          lowercase    :				Optional[Any]       		=							["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

          lowercase    :				Any       		=							DebertaVaTokenizer(_a	,    keep_accents=_a				)
          lowercase    :				Dict       		=							DebertaVaTokenizerFast(_a	,    keep_accents=_a				)

          lowercase    :				str       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				int       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Any       		=							rust_tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Union[str, Any]       		=							rust_tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Any       		=							[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
          lowercase    :				List[str]       		=							["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
          lowercase    :				str       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          lowercase    :				Tuple       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				Optional[int]       		=							DebertaVaTokenizer(_a				)

          lowercase    :				List[Any]       		=							tokenizer.encode("sequence builders"				)
          lowercase    :				Dict       		=							tokenizer.encode("multi-sequence build"				)

          lowercase    :				List[Any]       		=							tokenizer.build_inputs_with_special_tokens(_a				)
          lowercase    :				Dict       		=							tokenizer.build_inputs_with_special_tokens(_a	,    _a				)

          self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]	,    _a				)
          self.assertEqual(
              [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]	,    _a	,    )

   @slow
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Dict       		=							{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_a	,    model_name="microsoft/deberta-v2-xlarge"	,    revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"	,    )


