


"""simple docstring"""



import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaTokenizer, GPTJConfig, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, tooslow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
  import jax
  import jax.numpy as jnp

  from transformers.modeling_flax_pytorch_utils import (
      convert_pytorch_state_dict_to_flax,
      load_flax_weights_in_pytorch_model,
  )
  from transformers.models.gptj.modeling_flax_gptj import FlaxGPTJForCausalLM, FlaxGPTJModel

if is_torch_available():
  import torch







class _lowerCAmelCase				:







   def __init__(	self						,	UpperCamelCase__						,	UpperCamelCase__=14						,	UpperCamelCase__=7						,	UpperCamelCase__=True						,	UpperCamelCase__=True						,	UpperCamelCase__=False						,	UpperCamelCase__=True						,	UpperCamelCase__=99						,	UpperCamelCase__=32						,	UpperCamelCase__=4						,	UpperCamelCase__=4						,	UpperCamelCase__=4						,	UpperCamelCase__=37						,	UpperCamelCase__="gelu"						,	UpperCamelCase__=0.1						,	UpperCamelCase__=0.1						,	UpperCamelCase__=512						,	UpperCamelCase__=0.02						,	)	->		str:

       '''simple docstring'''




       snake_case    :      Tuple      					=    parent
       snake_case    :      Dict      					=    batch_size
       snake_case    :      Optional[int]      					=    seq_length
       snake_case    :      Tuple      					=    is_training
       snake_case    :      Tuple      					=    use_input_mask
       snake_case    :      List[Any]      					=    use_token_type_ids
       snake_case    :      str      					=    use_labels
       snake_case    :      Optional[int]      					=    vocab_size
       snake_case    :      Any      					=    hidden_size
       snake_case    :      Dict      					=    rotary_dim
       snake_case    :      List[Any]      					=    num_hidden_layers
       snake_case    :      Union[str, Any]      					=    num_attention_heads
       snake_case    :      Optional[int]      					=    intermediate_size
       snake_case    :      Tuple      					=    hidden_act
       snake_case    :      Dict      					=    hidden_dropout_prob
       snake_case    :      List[Any]      					=    attention_probs_dropout_prob
       snake_case    :      int      					=    max_position_embeddings
       snake_case    :      Optional[int]      					=    initializer_range
       snake_case    :      Dict      					=    None
       snake_case    :      List[Any]      					=    vocab_size - 1
       snake_case    :      Optional[int]      					=    vocab_size - 1
       snake_case    :      Any      					=    vocab_size - 1







   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       snake_case    :      Dict      					=    ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size						)

       snake_case    :      Optional[int]      					=    None
       if self.use_input_mask:
           snake_case    :      Optional[int]      					=    random_attention_mask([self.batch_size, self.seq_length]						)

       snake_case    :      Tuple      					=    GPTJConfig(
           vocab_size=self.vocab_size						,	n_embd=self.hidden_size						,	n_layer=self.num_hidden_layers						,	n_head=self.num_attention_heads						,	n_positions=self.max_position_embeddings						,	use_cache=UpperCamelCase__						,	bos_token_id=self.bos_token_id						,	eos_token_id=self.eos_token_id						,	pad_token_id=self.pad_token_id						,	rotary_dim=self.rotary_dim						,	)

       return (config, input_ids, input_mask)







   def  lowerCamelCase		(	self						)	->		Dict:

       '''simple docstring'''




       snake_case    :      Union[str, Any]      					=    self.prepare_config_and_inputs()
       snake_case					,snake_case					,snake_case    :      Union[str, Any]      					=    config_and_inputs
       snake_case    :      str      					=    {"input_ids": input_ids, "attention_mask": attention_mask}
       return config, inputs_dict







   def  lowerCamelCase		(	self						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						)	->		Union[str, Any]:

       '''simple docstring'''




       snake_case    :      Optional[Any]      					=    20
       snake_case    :      Optional[Any]      					=    model_class_name(UpperCamelCase__						)

       snake_case    :      List[str]      					=    model.init_cache(input_ids.shape[0]						,	UpperCamelCase__						)
       snake_case    :      List[str]      					=    jnp.ones((input_ids.shape[0], max_decoder_length)						,	dtype="i4"						)

       snake_case    :      str      					=    jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1						)[None, :]						,	(input_ids.shape[0], input_ids.shape[-1] - 1)						)
       snake_case    :      Optional[Any]      					=    model(
           input_ids[:, :-1]						,	attention_mask=UpperCamelCase__						,	past_key_values=UpperCamelCase__						,	position_ids=UpperCamelCase__						,	)

       snake_case    :      Any      					=    jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]						,	dtype="i4"						)
       snake_case    :      Optional[int]      					=    model(
           input_ids[:, -1:]						,	attention_mask=UpperCamelCase__						,	past_key_values=outputs_cache.past_key_values						,	position_ids=UpperCamelCase__						,	)

       snake_case    :      int      					=    model(UpperCamelCase__						)

       snake_case    :      List[str]      					=    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
       self.parent.assertTrue(diff < 1e-3						,	msg=F'Max diff is {diff}'						)







   def  lowerCamelCase		(	self						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						)	->		Dict:

       '''simple docstring'''




       snake_case    :      Dict      					=    20
       snake_case    :      Optional[Any]      					=    model_class_name(UpperCamelCase__						)

       snake_case    :      Tuple      					=    jnp.concatenate(
           [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])						)]						,	axis=-1						,	)

       snake_case    :      str      					=    model.init_cache(input_ids.shape[0]						,	UpperCamelCase__						)
       snake_case    :      List[str]      					=    jnp.broadcast_to(
           jnp.arange(input_ids.shape[-1] - 1						)[None, :]						,	(input_ids.shape[0], input_ids.shape[-1] - 1)						)

       snake_case    :      List[str]      					=    model(
           input_ids[:, :-1]						,	attention_mask=UpperCamelCase__						,	past_key_values=UpperCamelCase__						,	position_ids=UpperCamelCase__						,	)
       snake_case    :      List[str]      					=    jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]						,	dtype="i4"						)
       snake_case    :      Union[str, Any]      					=    model(
           input_ids[:, -1:]						,	past_key_values=outputs_cache.past_key_values						,	attention_mask=UpperCamelCase__						,	position_ids=UpperCamelCase__						,	)

       snake_case    :      Any      					=    model(UpperCamelCase__						,	attention_mask=UpperCamelCase__						)

       snake_case    :      Dict      					=    np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])						)						)
       self.parent.assertTrue(diff < 1e-3						,	msg=F'Max diff is {diff}'						)







@require_flax
class _lowerCAmelCase				(      snake_case_    ,							snake_case_    ,							unittest.TestCase	):
   __UpperCAmelCase :       Any							  =  (FlaxGPTJModel, FlaxGPTJForCausalLM) if is_flax_available() else ()
   __UpperCAmelCase :       int							  =  (FlaxGPTJForCausalLM,) if is_flax_available() else ()







   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      List[Any]      					=    FlaxGPTJModelTester(self						)







   def  lowerCamelCase		(	self						)	->		Tuple:

       '''simple docstring'''




       for model_class_name in self.all_model_classes:
           snake_case					,snake_case					,snake_case    :      Optional[int]      					=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.check_use_cache_forward(UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						)







   def  lowerCamelCase		(	self						)	->		Dict:

       '''simple docstring'''




       for model_class_name in self.all_model_classes:
           snake_case					,snake_case					,snake_case    :      Dict      					=    self.model_tester.prepare_config_and_inputs()
           self.model_tester.check_use_cache_forward_with_attn_mask(
               UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						,	UpperCamelCase__						)







   @tooslow
   def  lowerCamelCase		(	self						)	->		Dict:

       '''simple docstring'''




       snake_case    :      List[Any]      					=    GPTaTokenizer.from_pretrained("gpt2"						,	pad_token="<|endoftext|>"						,	padding_side="left"						)
       snake_case    :      Dict      					=    tokenizer(["Hello this is a long string", "Hey"]						,	return_tensors="np"						,	padding=UpperCamelCase__						,	truncation=UpperCamelCase__						)

       snake_case    :      List[Any]      					=    FlaxGPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B"						)
       snake_case    :      int      					=    False
       snake_case    :      Dict      					=    model.config.eos_token_id

       snake_case    :      Union[str, Any]      					=    jax.jit(model.generate						)

       snake_case    :      List[Any]      					=    jit_generate(
           inputs["input_ids"]						,	attention_mask=inputs["attention_mask"]						,	pad_token_id=tokenizer.pad_token_id						).sequences

       snake_case    :      Optional[int]      					=    tokenizer.batch_decode(UpperCamelCase__						,	skip_special_tokens=UpperCamelCase__						)

       snake_case    :      List[str]      					=    [
           "Hello this is a long string of text.\n\nI'm trying to get the text of the",
           "Hey, I'm a little late to the party. I'm going to",
       ]

       self.assertListEqual(UpperCamelCase__						,	UpperCamelCase__						)







   @is_pt_flax_cross_test
   def  lowerCamelCase		(	self						)	->		Any:

       '''simple docstring'''




       snake_case					,snake_case    :      List[str]      					=    self.model_tester.prepare_config_and_inputs_for_common()

       for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__						):
               # prepare inputs
               snake_case    :      List[Any]      					=    self._prepare_for_class(UpperCamelCase__						,	UpperCamelCase__						)
               snake_case    :      Dict      					=    {k: torch.tensor(v.tolist()						) for k, v in prepared_inputs_dict.items()}

               # load corresponding PyTorch class
               snake_case    :      Optional[Any]      					=    model_class.__name__[4:]  # Skip the "Flax" at the beginning
               snake_case    :      Tuple      					=    getattr(UpperCamelCase__						,	UpperCamelCase__						)

               snake_case					,snake_case    :      Tuple      					=    pt_inputs["input_ids"].shape
               snake_case    :      List[str]      					=    np.random.randint(0						,	seq_length - 1						,	size=(batch_size,)						)
               for batch_idx, start_index in enumerate(UpperCamelCase__						):
                   snake_case    :      List[str]      					=    0
                   snake_case    :      str      					=    1
                   snake_case    :      Optional[int]      					=    0
                   snake_case    :      Optional[int]      					=    1
               snake_case    :      Optional[int]      					=    pt_model_class(UpperCamelCase__						).eval()
               snake_case    :      Any      					=    model_class(UpperCamelCase__						,	dtype=jnp.floataa						)

               snake_case    :      Union[str, Any]      					=    convert_pytorch_state_dict_to_flax(pt_model.state_dict()						,	UpperCamelCase__						)
               snake_case    :      Any      					=    fx_state

               with torch.no_grad():
                   snake_case    :      Optional[Any]      					=    pt_model(**UpperCamelCase__						).to_tuple()

               snake_case    :      List[str]      					=    fx_model(**UpperCamelCase__						).to_tuple()
               self.assertEqual(len(UpperCamelCase__						)						,	len(UpperCamelCase__						)						,	"Output lengths differ between Flax and PyTorch"						)
               for fx_output, pt_output in zip(UpperCamelCase__						,	UpperCamelCase__						):
                   self.assert_almost_equals(fx_output[:, -1]						,	pt_output[:, -1].numpy()						,	4e-2						)

               with tempfile.TemporaryDirectory() as tmpdirname:
                   pt_model.save_pretrained(UpperCamelCase__						)
                   snake_case    :      Optional[Any]      					=    model_class.from_pretrained(UpperCamelCase__						,	from_pt=UpperCamelCase__						)

               snake_case    :      Tuple      					=    fx_model_loaded(**UpperCamelCase__						).to_tuple()
               self.assertEqual(
                   len(UpperCamelCase__						)						,	len(UpperCamelCase__						)						,	"Output lengths differ between Flax and PyTorch"						)
               for fx_output_loaded, pt_output in zip(UpperCamelCase__						,	UpperCamelCase__						):
                   self.assert_almost_equals(fx_output_loaded[:, -1]						,	pt_output[:, -1].numpy()						,	4e-2						)







   @is_pt_flax_cross_test
   def  lowerCamelCase		(	self						)	->		Tuple:

       '''simple docstring'''




       snake_case					,snake_case    :      Tuple      					=    self.model_tester.prepare_config_and_inputs_for_common()
       for model_class in self.all_model_classes:
           with self.subTest(model_class.__name__						):
               # prepare inputs
               snake_case    :      Union[str, Any]      					=    self._prepare_for_class(UpperCamelCase__						,	UpperCamelCase__						)
               snake_case    :      Union[str, Any]      					=    {k: torch.tensor(v.tolist()						) for k, v in prepared_inputs_dict.items()}

               # load corresponding PyTorch class
               snake_case    :      Tuple      					=    model_class.__name__[4:]  # Skip the "Flax" at the beginning
               snake_case    :      Optional[Any]      					=    getattr(UpperCamelCase__						,	UpperCamelCase__						)

               snake_case    :      List[Any]      					=    pt_model_class(UpperCamelCase__						).eval()
               snake_case    :      str      					=    model_class(UpperCamelCase__						,	dtype=jnp.floataa						)

               snake_case    :      Dict      					=    load_flax_weights_in_pytorch_model(UpperCamelCase__						,	fx_model.params						)
               snake_case					,snake_case    :      Optional[Any]      					=    pt_inputs["input_ids"].shape
               snake_case    :      Dict      					=    np.random.randint(0						,	seq_length - 1						,	size=(batch_size,)						)
               for batch_idx, start_index in enumerate(UpperCamelCase__						):
                   snake_case    :      Optional[Any]      					=    0
                   snake_case    :      Optional[Any]      					=    1
                   snake_case    :      List[str]      					=    0
                   snake_case    :      Dict      					=    1

               # make sure weights are tied in PyTorch
               pt_model.tie_weights()

               with torch.no_grad():
                   snake_case    :      int      					=    pt_model(**UpperCamelCase__						).to_tuple()

               snake_case    :      int      					=    fx_model(**UpperCamelCase__						).to_tuple()
               self.assertEqual(len(UpperCamelCase__						)						,	len(UpperCamelCase__						)						,	"Output lengths differ between Flax and PyTorch"						)
               for fx_output, pt_output in zip(UpperCamelCase__						,	UpperCamelCase__						):
                   self.assert_almost_equals(fx_output[:, -1]						,	pt_output[:, -1].numpy()						,	4e-2						)

               with tempfile.TemporaryDirectory() as tmpdirname:
                   fx_model.save_pretrained(UpperCamelCase__						)
                   snake_case    :      int      					=    pt_model_class.from_pretrained(UpperCamelCase__						,	from_flax=UpperCamelCase__						)

               with torch.no_grad():
                   snake_case    :      Dict      					=    pt_model_loaded(**UpperCamelCase__						).to_tuple()

               self.assertEqual(
                   len(UpperCamelCase__						)						,	len(UpperCamelCase__						)						,	"Output lengths differ between Flax and PyTorch"						)
               for fx_output, pt_output in zip(UpperCamelCase__						,	UpperCamelCase__						):
                   self.assert_almost_equals(fx_output[:, -1]						,	pt_output[:, -1].numpy()						,	4e-2						)







   @tooslow
   def  lowerCamelCase		(	self						)	->		Optional[Any]:

       '''simple docstring'''




       for model_class_name in self.all_model_classes:
           snake_case    :      Dict      					=    model_class_name.from_pretrained("EleutherAI/gpt-j-6B"						)
           snake_case    :      Tuple      					=    model(np.ones((1, 1)						)						)
           self.assertIsNotNone(UpperCamelCase__						)









"""simple docstring"""



import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin







class _lowerCAmelCase				(      snake_case_    ,							unittest.TestCase	):
   __UpperCAmelCase :       str							  =  DebertaTokenizer
   __UpperCAmelCase :       Optional[int]							  =  True
   __UpperCAmelCase :       int							  =  DebertaTokenizerFast







   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       snake_case    :      Union[str, Any]      					=    [
           "l",
           "o",
           "w",
           "e",
           "r",
           "s",
           "t",
           "i",
           "d",
           "n",
           "\u0120",
           "\u0120l",
           "\u0120n",
           "\u0120lo",
           "\u0120low",
           "er",
           "\u0120lowest",
           "\u0120newer",
           "\u0120wider",
           "[UNK]",
       ]
       snake_case    :      str      					=    dict(zip(UpperCamelCase__						,	range(len(UpperCamelCase__						)						)						)						)
       snake_case    :      int      					=    ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
       snake_case    :      Optional[Any]      					=    {"unk_token": "[UNK]"}

       snake_case    :      Union[str, Any]      					=    os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["vocab_file"]						)
       snake_case    :      Dict      					=    os.path.join(self.tmpdirname						,	VOCAB_FILES_NAMES["merges_file"]						)
       with open(self.vocab_file						,	"w"						,	encoding="utf-8"						) as fp:
           fp.write(json.dumps(UpperCamelCase__						) + "\n"						)
       with open(self.merges_file						,	"w"						,	encoding="utf-8"						) as fp:
           fp.write("\n".join(UpperCamelCase__						)						)







   def  lowerCamelCase		(	self						,	**UpperCamelCase__						)	->		Any:

       '''simple docstring'''




       kwargs.update(self.special_tokens_map						)
       return self.tokenizer_class.from_pretrained(self.tmpdirname						,	**UpperCamelCase__						)







   def  lowerCamelCase		(	self						,	UpperCamelCase__						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      Tuple      					=    "lower newer"
       snake_case    :      Any      					=    "lower newer"
       return input_text, output_text







   def  lowerCamelCase		(	self						)	->		int:

       '''simple docstring'''




       snake_case    :      Optional[int]      					=    self.get_tokenizer()
       snake_case    :      str      					=    "lower newer"
       snake_case    :      Dict      					=    ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
       snake_case    :      Optional[Any]      					=    tokenizer.tokenize(UpperCamelCase__						)
       self.assertListEqual(UpperCamelCase__						,	UpperCamelCase__						)

       snake_case    :      List[Any]      					=    tokens + [tokenizer.unk_token]
       snake_case    :      Dict      					=    [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__						)						,	UpperCamelCase__						)







   def  lowerCamelCase		(	self						)	->		List[str]:

       '''simple docstring'''




       snake_case    :      Tuple      					=    self.get_tokenizer()
       snake_case    :      Dict      					=    tokenizer("Hello"						,	"World"						)
       snake_case    :      str      					=    [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
       self.assertListEqual(tokd["token_type_ids"]						,	UpperCamelCase__						)







   @slow
   def  lowerCamelCase		(	self						)	->		str:

       '''simple docstring'''




       snake_case    :      List[Any]      					=    self.tokenizer_class.from_pretrained("microsoft/deberta-base"						)

       snake_case    :      Dict      					=    tokenizer.encode("sequence builders"						,	add_special_tokens=UpperCamelCase__						)
       snake_case    :      Tuple      					=    tokenizer.encode("multi-sequence build"						,	add_special_tokens=UpperCamelCase__						)

       snake_case    :      List[str]      					=    tokenizer.encode(
           "sequence builders"						,	add_special_tokens=UpperCamelCase__						,	add_prefix_space=UpperCamelCase__						)
       snake_case    :      Optional[int]      					=    tokenizer.encode(
           "sequence builders"						,	"multi-sequence build"						,	add_special_tokens=UpperCamelCase__						,	add_prefix_space=UpperCamelCase__						)

       snake_case    :      int      					=    tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						)
       snake_case    :      int      					=    tokenizer.build_inputs_with_special_tokens(UpperCamelCase__						,	UpperCamelCase__						)

       assert encoded_sentence == encoded_text_from_decode
       assert encoded_pair == encoded_pair_from_decode







   @slow
   def  lowerCamelCase		(	self						)	->		List[Any]:

       '''simple docstring'''




       snake_case    :      Union[str, Any]      					=    [self.tokenizer_class]
       if self.test_rust_tokenizer:
           tokenizer_classes.append(self.rust_tokenizer_class						)

       for tokenizer_class in tokenizer_classes:
           snake_case    :      str      					=    tokenizer_class.from_pretrained("microsoft/deberta-base"						)

           snake_case    :      Union[str, Any]      					=    [
               "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
               "ALBERT incorporates two parameter reduction techniques",
               "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
               " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
               " vocabulary embedding.",
           ]

           snake_case    :      Optional[int]      					=    tokenizer(UpperCamelCase__						,	padding=UpperCamelCase__						)
           snake_case    :      Optional[int]      					=    [tokenizer.decode(UpperCamelCase__						,	skip_special_tokens=UpperCamelCase__						) for seq in encoding["input_ids"]]

           # fmt: off
           snake_case    :      Dict      					=    {
               "input_ids": [
                   [1, 2118, 1_1126, 565, 35, 83, 2_5191, 163, 1_8854, 13, 1_2156, 12, 1_6101, 2_5376, 1_3807, 9, 2_2205, 2_7893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 2118, 1_1126, 565, 2_4536, 80, 4_3797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 133, 78, 65, 16, 10, 3724, 1538, 3_3183, 1_1303, 4_3797, 1938, 4, 870, 2_4165, 2_9105, 5, 739, 3_2644, 3_3183, 1_1303, 3_6173, 88, 80, 650, 7821, 4_5940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 1_3171, 31, 5, 1836, 9, 3_2644, 3_3183, 1_1303, 4, 2]
               ],
               "token_type_ids": [
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
               ],
               "attention_mask": [
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                   [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
               ]
           }
           # fmt: on

           snake_case    :      int      					=    [
               "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
               "ALBERT incorporates two parameter reduction techniques",
               "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
               " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
               " vocabulary embedding.",
           ]

           self.assertDictEqual(encoding.data						,	UpperCamelCase__						)

           for expected, decoded in zip(UpperCamelCase__						,	UpperCamelCase__						):
               self.assertEqual(UpperCamelCase__						,	UpperCamelCase__						)





