





'''simple docstring'''

import argparse
import os
from pathlib import Path
from typing import Dict

import tensorflow as tf
import torch
from tqdm import tqdm

from transformers import PegasusConfig, PegasusForConditionalGeneration, PegasusTokenizer
from transformers.models.pegasus.configuration_pegasus import DEFAULTS, task_specific_params


__lowerCAmelCase							:	Any			=[
    # replace left string with right string to get the relevant state_dict key (identical state dict to bart)
    ["""memory_attention""", """encoder_attn"""],
    ["""attention""", """attn"""],
    ["""/""", """."""],
    [""".LayerNorm.gamma""", """_layer_norm.weight"""],
    [""".LayerNorm.beta""", """_layer_norm.bias"""],
    ["""r.layer_""", """r.layers."""],
    ["""output_proj""", """out_proj"""],
    ["""ffn.dense_1.""", """fc2."""],
    ["""ffn.dense.""", """fc1."""],
    ["""ffn_layer_norm""", """final_layer_norm"""],
    ["""kernel""", """weight"""],
    ["""encoder_layer_norm.""", """encoder.layer_norm."""],
    ["""decoder_layer_norm.""", """decoder.layer_norm."""],
    ["""embeddings.weights""", """shared.weight"""],
]





def 						UpperCamelCase				(							_lowerCamelCase					:					Union[str, Any]       ):
    for pegasus_name, hf_name in PATTERNS:
        A__				=			k.replace(__snake_case						,       __snake_case       )
    return k





def 						UpperCamelCase				(							_lowerCamelCase					:					dict						,       _lowerCamelCase					:					dict       ):
    A__				=			DEFAULTS.copy()
    cfg_kwargs.update(__snake_case       )
    A__				=			PegasusConfig(**__snake_case       )
    A__				=			PegasusForConditionalGeneration(__snake_case       )
    A__				=			torch_model.model.state_dict()
    A__				=			{}
    for k, v in tf_weights.items():
        A__				=			rename_state_dict_key(__snake_case       )
        if new_k not in sd:
            raise ValueError(F"could not find new key {new_k} in state dict. (converted from {k})"       )

        if "dense" in k or "proj" in new_k:
            A__				=			v.T
        A__				=			torch.tensor(__snake_case						,       dtype=sd[new_k].dtype       )
        assert v.shape == sd[new_k].shape, F"{new_k}, {k}, {v.shape}, {sd[new_k].shape}"
    # make sure embedding.padding_idx is respected
    A__				=			torch.zeros_like(mapping["shared.weight"][cfg.pad_token_id + 1]       )
    A__				=			mapping["shared.weight"]
    A__				=			mapping["shared.weight"]
    A__				=			{k: torch.zeros_like(__snake_case       ) for k, v in sd.items() if k.endswith("bias"       ) and k not in mapping}
    mapping.update(**__snake_case       )
    A__				=			torch_model.model.load_state_dict(__snake_case						,       strict=__snake_case       )
    A__				=			[
        k for k in missing if k not in ["encoder.embed_positions.weight", "decoder.embed_positions.weight"]
    ]
    assert unexpected_missing == [], F"no matches found for the following torch keys {unexpected_missing}"
    assert extra == [], F"no matches found for the following tf keys {extra}"
    return torch_model





def 						UpperCamelCase				(							_lowerCamelCase					:					List[str]="./ckpt/aeslc/model.ckpt-32000"       ):
    A__				=			tf.train.list_variables(__snake_case       )
    A__				=			{}
    A__				=			["Adafactor", "global_step"]
    for name, shape in tqdm(__snake_case						,       desc="converting tf checkpoint to dict"       ):
        A__				=			any(pat in name for pat in ignore_name       )
        if skip_key:
            continue
        A__				=			tf.train.load_variable(__snake_case						,       __snake_case       )
        A__				=			array
    return tf_weights





def 						UpperCamelCase				(							_lowerCamelCase					:					str						,       _lowerCamelCase					:					str       ):
    # save tokenizer first
    A__				=			Path(__snake_case       ).parent.name
    A__				=			task_specific_params[F"summarization_{dataset}"]["max_position_embeddings"]
    A__				=			PegasusTokenizer.from_pretrained("sshleifer/pegasus"						,       model_max_length=__snake_case       )
    assert tok.model_max_length == desired_max_model_length
    tok.save_pretrained(__snake_case       )

    # convert model
    A__				=			get_tf_weights_as_numpy(__snake_case       )
    A__				=			task_specific_params[F"summarization_{dataset}"]
    if dataset == "large":
        A__				=			task_specific_params
    A__				=			convert_pegasus(__snake_case						,       __snake_case       )
    torch_model.save_pretrained(__snake_case       )
    A__				=			torch_model.state_dict()
    sd.pop("model.decoder.embed_positions.weight"       )
    sd.pop("model.encoder.embed_positions.weight"       )
    torch.save(__snake_case						,       Path(__snake_case       ) / "pytorch_model.bin"       )


if __name__ == "__main__":
  __lowerCAmelCase							:	Union[str, Any]			=argparse.ArgumentParser()
  # Required parameters
  parser.add_argument("tf_ckpt_path", type=str, help="passed to tf.train.list_variables")
  parser.add_argument("save_dir", default=None, type=str, help="Path to the output PyTorch model.")
  __lowerCAmelCase							:	Union[str, Any]			=parser.parse_args()
  if args.save_dir is None:
    __lowerCAmelCase							:	Any			=Path(args.tf_ckpt_path).parent.name
    __lowerCAmelCase							:	Dict			=os.path.join("pegasus", dataset)
  convert_pegasus_ckpt_to_pytorch(args.tf_ckpt_path, args.save_dir)





"""simple docstring"""




import unittest

from transformers import DebertaVaTokenizer, DebertaVaTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_A			:       int   		=       get_tests_dir("""fixtures/spiece.model""")

@require_sentencepiece
@require_tokenizers
class 		a__						(      a_,			unittest.TestCase						):
   __lowerCAmelCase            =				DebertaVaTokenizer
   __lowerCAmelCase            =				DebertaVaTokenizerFast
   __lowerCAmelCase            =				True
   __lowerCAmelCase            =				True
   def 			__magic_name__			(			self				):
          super().setUp()

          # We have a SentencePiece fixture for testing
          lowercase    :				Any       		=							DebertaVaTokenizer(_a	,    unk_token="<unk>"				)
          tokenizer.save_pretrained(self.tmpdirname				)
   def 			__magic_name__			(			self	,    _a				):
          lowercase    :				int       		=							"this is a test"
          lowercase    :				Tuple       		=							"this is a test"
          return input_text, output_text
   def 			__magic_name__			(			self				):
          lowercase    :				List[Any]       		=							"<pad>"
          lowercase    :				List[str]       		=							0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a				)	,    _a				)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a				)	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				int       		=							list(self.get_tokenizer().get_vocab().keys()				)
          self.assertEqual(vocab_keys[0]	,    "<pad>"				)
          self.assertEqual(vocab_keys[1]	,    "<unk>"				)
          self.assertEqual(vocab_keys[-1]	,    "[PAD]"				)
          self.assertEqual(len(_a				)	,    30_001				)
   def 			__magic_name__			(			self				):
          self.assertEqual(self.get_tokenizer().vocab_size	,    30_000				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				List[str]       		=							" \tHeLLo!how  \n Are yoU?  "
          lowercase    :				str       		=							["▁hello", "!", "how", "▁are", "▁you", "?"]
          # fmt: on

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a				)
          lowercase    :				Any       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				str       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a				)
          lowercase    :				Optional[Any]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."				)
   def 			__magic_name__			(			self				):
          pass
   @unittest.skip("There is an inconsistency between slow and fast tokenizer due to a bug in the fast one."				)
   def 			__magic_name__			(			self				):
          pass
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Optional[Any]       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Tuple       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				List[Any]       		=							DebertaVaTokenizer(_a	,    split_by_punct=_a				)
          lowercase    :				Tuple       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    split_by_punct=_a				)
          lowercase    :				Any       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Tuple       		=							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				List[str]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[Any]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[int]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				List[Any]       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Optional[int]       		=							["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          lowercase    :				List[Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Optional[int]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Dict       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", "▁", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "▁", ".", ]
          # fmt: on

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Any       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				Union[str, Any]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Dict       		=							" \tHeLLo!how  \n Are yoU?  "
          lowercase    :				str       		=							["▁", "<unk>", "e", "<unk>", "o", "!", "how", "▁", "<unk>", "re", "▁yo", "<unk>", "?"]
          # fmt: on

          lowercase    :				Optional[int]       		=							DebertaVaTokenizer(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				int       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)

          lowercase    :				List[str]       		=							DebertaVaTokenizerFast(_a	,    do_lower_case=_a	,    split_by_punct=_a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)

          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				str       		=							self.get_tokenizer()
          lowercase    :				Dict       		=							self.get_rust_tokenizer()

          lowercase    :				str       		=							"I was born in 92000, and this is falsé."

          lowercase    :				Optional[Any]       		=							tokenizer.convert_ids_to_tokens(tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          lowercase    :				Dict       		=							rust_tokenizer.convert_ids_to_tokens(rust_tokenizer.encode(_a	,    add_special_tokens=_a				)				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				str       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          lowercase    :				Dict       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							self.get_rust_tokenizer()
          lowercase    :				Tuple       		=							tokenizer.encode(_a				)
          lowercase    :				List[str]       		=							rust_tokenizer.encode(_a				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				str       		=							"This is a test"
          lowercase    :				Tuple       		=							[13, 1, 4_398, 25, 21, 1_289]
          lowercase    :				Optional[int]       		=							["▁", "T", "his", "▁is", "▁a", "▁test"]
          lowercase    :				Optional[Any]       		=							["▁", "<unk>", "his", "▁is", "▁a", "▁test"]

          lowercase    :				Any       		=							DebertaVaTokenizer(_a	,    keep_accents=_a				)
          lowercase    :				Dict       		=							DebertaVaTokenizerFast(_a	,    keep_accents=_a				)

          lowercase    :				str       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				int       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Any       		=							rust_tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				Union[str, Any]       		=							rust_tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          # fmt: off
          lowercase    :				int       		=							"I was born in 92000, and this is falsé."
          lowercase    :				Any       		=							[13, 1, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]
          lowercase    :				List[str]       		=							["▁", "I", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", ".", ]
          lowercase    :				str       		=							["▁", "<unk>", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", ".", ]
          # fmt: on

          lowercase    :				Tuple       		=							tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				str       		=							tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)

          lowercase    :				Optional[int]       		=							rust_tokenizer.encode(_a	,    add_special_tokens=_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.tokenize(_a				)
          self.assertListEqual(_a	,    _a				)
          lowercase    :				List[Any]       		=							rust_tokenizer.convert_ids_to_tokens(_a				)
          self.assertListEqual(_a	,    _a				)
   def 			__magic_name__			(			self				):
          lowercase    :				Optional[int]       		=							DebertaVaTokenizer(_a				)

          lowercase    :				List[Any]       		=							tokenizer.encode("sequence builders"				)
          lowercase    :				Dict       		=							tokenizer.encode("multi-sequence build"				)

          lowercase    :				List[Any]       		=							tokenizer.build_inputs_with_special_tokens(_a				)
          lowercase    :				Dict       		=							tokenizer.build_inputs_with_special_tokens(_a	,    _a				)

          self.assertEqual([tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]	,    _a				)
          self.assertEqual(
              [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [tokenizer.sep_token_id]	,    _a	,    )

   @slow
   def 			__magic_name__			(			self				):
          # fmt: off
          lowercase    :				Dict       		=							{"input_ids": [[1, 39_867, 36, 19_390, 486, 27, 35_052, 81_436, 18, 60_685, 1_225, 7, 35_052, 81_436, 18, 9_367, 16_899, 18, 15_937, 53, 594, 773, 18, 16_287, 30_465, 36, 15_937, 6, 41_139, 38, 36_979, 60_763, 191, 6, 34_132, 99, 6, 50_538, 390, 43_230, 6, 34_132, 2_779, 20_850, 14, 699, 1_072, 1_194, 36, 382, 10_901, 53, 7, 699, 1_072, 2_084, 36, 20_422, 630, 53, 19, 105, 3_049, 1_896, 1_053, 16_899, 1_506, 11, 37_978, 4_243, 7, 1_237, 31_869, 200, 16_566, 654, 6, 35_052, 81_436, 7, 55_630, 13_593, 4, 2], [1, 26, 15_011, 13, 667, 8, 1_053, 18, 23_611, 1_237, 72_356, 12_820, 34, 104_134, 1_209, 35, 13_313, 6_627, 21, 202, 347, 7, 164, 2_399, 11, 46, 4_485, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 5, 1_232, 2_864, 15_785, 14_951, 105, 5, 8_581, 1_250, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=_a	,    model_name="microsoft/deberta-v2-xlarge"	,    revision="ad6e42c1532ddf3a15c39246b63f5559d558b670"	,    )


