






import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate
# and perform gradient accumulation
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


a		:       Any			     = 16
a		:       Optional[Any]			     = 32






def 				lowerCamelCase__      (    __lowerCamelCase			:     str  ,		__lowerCamelCase			:     str = 16		):
  __UpperCAmelCase							:	Dict           =  AutoTokenizer.from_pretrained("""bert-base-cased"""		)
  __UpperCAmelCase							:	Optional[int]           =  load_dataset("""glue"""  ,		"""mrpc"""		)

  def tokenize_function(__lowerCamelCase			:     Tuple		):
    # max_length=None => use the model max length (it's actually the default)
    __UpperCAmelCase							:	Union[str, Any]           =  tokenizer(examples["""sentence1"""]  ,		examples["""sentence2"""]  ,		truncation=__snake_case  ,		max_length=__snake_case		)
    return outputs

  # Apply the method we just defined to all the examples in all the splits of the dataset
  # starting with the main process first:
  with accelerator.main_process_first():
    __UpperCAmelCase							:	int           =  datasets.map(
        __snake_case  ,		batched=__snake_case  ,		remove_columns=["""idx""", """sentence1""", """sentence2"""]  ,		)

  # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
  # transformers library
  __UpperCAmelCase							:	Union[str, Any]           =  tokenized_datasets.rename_column("""label"""  ,		"""labels"""		)

  def collate_fn(__lowerCamelCase			:     Any		):
    # On TPU it's best to pad everything to the same length or training will be very slow.
    __UpperCAmelCase							:	Optional[int]           =  128 if accelerator.distributed_type == DistributedType.TPU else None
    # When using mixed precision we want round multiples of 8/16
    if accelerator.mixed_precision == "fp8":
      __UpperCAmelCase							:	int           =  16
    elif accelerator.mixed_precision != "no":
      __UpperCAmelCase							:	Union[str, Any]           =  8
    else:
      __UpperCAmelCase							:	List[Any]           =  None

    return tokenizer.pad(
        __snake_case  ,		padding="""longest"""  ,		max_length=__snake_case  ,		pad_to_multiple_of=__snake_case  ,		return_tensors="""pt"""  ,		)

  # Instantiate dataloaders.
  __UpperCAmelCase							:	Union[str, Any]           =  DataLoader(
      tokenized_datasets["""train"""]  ,		shuffle=__snake_case  ,		collate_fn=__snake_case  ,		batch_size=__snake_case		)
  __UpperCAmelCase							:	List[Any]           =  DataLoader(
      tokenized_datasets["""validation"""]  ,		shuffle=__snake_case  ,		collate_fn=__snake_case  ,		batch_size=__snake_case		)

  return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
       from accelerate.test_utils.training import mocked_dataloaders

       a		:       Dict			     = mocked_dataloaders  # noqa: F811






def 				lowerCamelCase__      (    __lowerCamelCase			:     List[Any]  ,		__lowerCamelCase			:     Optional[int]		):





  if os.environ.get("""TESTING_MOCKED_DATALOADERS"""  ,		__snake_case		) == "1":
    __UpperCAmelCase							:	Optional[Any]           =  2
  # New Code #
  __UpperCAmelCase							:	Dict           =  int(args.gradient_accumulation_steps		)
  # Initialize accelerator
  __UpperCAmelCase							:	Any           =  Accelerator(
      cpu=args.cpu  ,		mixed_precision=args.mixed_precision  ,		gradient_accumulation_steps=__snake_case		)
  if accelerator.distributed_type == DistributedType.TPU and gradient_accumulation_steps > 1:
    raise NotImplementedError(
        """Gradient accumulation on TPUs is currently not supported. Pass `gradient_accumulation_steps=1`"""		)
  # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
  __UpperCAmelCase							:	Optional[Any]           =  config["""lr"""]
  __UpperCAmelCase							:	List[Any]           =  int(config["""num_epochs"""]		)
  __UpperCAmelCase							:	List[Any]           =  int(config["""seed"""]		)
  __UpperCAmelCase							:	Optional[int]           =  int(config["""batch_size"""]		)

  __UpperCAmelCase							:	Optional[Any]           =  evaluate.load("""glue"""  ,		"""mrpc"""		)

  set_seed(__snake_case		)
  __UpperCAmelCase      , __UpperCAmelCase							:	str           =  get_dataloaders(__snake_case  ,		__snake_case		)
  # Instantiate the model (we build the model here so that the seed also control new weights initialization)
  __UpperCAmelCase							:	Any           =  AutoModelForSequenceClassification.from_pretrained("""bert-base-cased"""  ,		return_dict=__snake_case		)

  # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
  # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
  # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
  __UpperCAmelCase							:	Union[str, Any]           =  model.to(accelerator.device		)

  # Instantiate optimizer
  __UpperCAmelCase							:	int           =  AdamW(params=model.parameters()  ,		lr=__snake_case		)

  # Instantiate scheduler
  __UpperCAmelCase							:	Optional[int]           =  get_linear_schedule_with_warmup(
      optimizer=__snake_case  ,		num_warmup_steps=100  ,		num_training_steps=(len(__snake_case		) * num_epochs)  ,		)

  # Prepare everything
  # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
  # prepare method.
  __UpperCAmelCase      , __UpperCAmelCase      , __UpperCAmelCase      , __UpperCAmelCase      , __UpperCAmelCase							:	Union[str, Any]           =  accelerator.prepare(
      __snake_case  ,		__snake_case  ,		__snake_case  ,		__snake_case  ,		__snake_case		)

  # Now we train the model
  for epoch in range(__snake_case		):
    model.train()
    for step, batch in enumerate(__snake_case		):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device		)
      # New code #
      # We use the new `accumulate` context manager to perform gradient accumulation
      # We also currently do not support TPUs nor advise it as bugs were found on the XLA side when running our tests.
      with accelerator.accumulate(__snake_case		):
        __UpperCAmelCase							:	Tuple           =  model(**__snake_case		)
        __UpperCAmelCase							:	str           =  output.loss
        accelerator.backward(__snake_case		)
        optimizer.step()
        lr_scheduler.step()
        optimizer.zero_grad()

    model.eval()
    for step, batch in enumerate(__snake_case		):
      # We could avoid this line since we set the accelerator with `device_placement=True`.
      batch.to(accelerator.device		)
      with torch.no_grad():
        __UpperCAmelCase							:	List[str]           =  model(**__snake_case		)
      __UpperCAmelCase							:	Tuple           =  outputs.logits.argmax(dim=-1		)
      __UpperCAmelCase      , __UpperCAmelCase							:	Tuple           =  accelerator.gather_for_metrics((predictions, batch["""labels"""])		)
      metric.add_batch(
          predictions=__snake_case  ,		references=__snake_case  ,		)

    __UpperCAmelCase							:	Dict           =  metric.compute()
    # Use accelerator.print to print only on the main process.
    accelerator.print(f"""epoch {epoch}:"""  ,		__snake_case		)






def 				lowerCamelCase__      (    ):
  __UpperCAmelCase							:	Optional[Any]           =  argparse.ArgumentParser(description="""Simple example of training script."""		)
  parser.add_argument(
      """--mixed_precision"""  ,		type=__snake_case  ,		default=__snake_case  ,		choices=["""no""", """fp16""", """bf16""", """fp8"""]  ,		help="""Whether to use mixed precision. Choose"""
      """between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."""
      """and an Nvidia Ampere GPU."""  ,		)
  # New Code #
  parser.add_argument(
      """--gradient_accumulation_steps"""  ,		type=__snake_case  ,		default=1  ,		help="""The number of minibatches to be ran before gradients are accumulated."""  ,		)
  parser.add_argument("""--cpu"""  ,		action="""store_true"""  ,		help="""If passed, will train on the CPU."""		)
  __UpperCAmelCase							:	Tuple           =  parser.parse_args()
  __UpperCAmelCase							:	Optional[int]           =  {"""lr""": 2E-5, """num_epochs""": 3, """seed""": 42, """batch_size""": 16}
  training_function(__snake_case  ,		__snake_case		)


if __name__ == "__main__":
       main()




"""simple docstring"""





import unittest

from transformers import SPIECE_UNDERLINE
from transformers.models.speechta import SpeechTaTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow
from transformers.tokenization_utils import AddedToken

from ...test_tokenization_common import TokenizerTesterMixin


_a														=		get_tests_dir("""fixtures/test_sentencepiece_bpe_char.model""")






@require_sentencepiece
@require_tokenizers
class     _UpperCAmelCase(    lowerCamelCase      ,     unittest.TestCase	):
		lowercase__						=      SpeechTaTokenizer
		lowercase__						=      False
		lowercase__						=      True



		def 						UpperCAmelCase							(					self)       ->       List[Any]:






							'''simple docstring'''
							super().setUp()

							# We have a SentencePiece fixture for testing
							_UpperCamelCase							 =  SpeechTaTokenizer(__a)

							_UpperCamelCase							 =  AddedToken('''<mask>'''				,		lstrip=__a				,		rstrip=__a)
							_UpperCamelCase							 =  mask_token
							tokenizer.add_special_tokens({'''mask_token''': mask_token})
							tokenizer.add_tokens(['''<ctc_blank>'''])

							tokenizer.save_pretrained(self.tmpdirname)



		def 						UpperCAmelCase							(					self				,		__a)       ->       List[str]:






							'''simple docstring'''
							_UpperCamelCase							 =  '''this is a test'''
							_UpperCamelCase							 =  '''this is a test'''
							return input_text, output_text



		def 						UpperCAmelCase							(					self				,		__a				,		__a=False				,		__a=20				,		__a=5)       ->       List[str]:






							'''simple docstring'''
							_UpperCamelCase ,  _UpperCamelCase							 =  self.get_input_output_texts(__a)
							_UpperCamelCase							 =  tokenizer.encode(__a				,		add_special_tokens=__a)
							_UpperCamelCase							 =  tokenizer.decode(__a				,		clean_up_tokenization_spaces=__a)
							return text, ids



		def 						UpperCAmelCase							(					self)       ->       Optional[Any]:






							'''simple docstring'''
							_UpperCamelCase							 =  '''<pad>'''
							_UpperCamelCase							 =  1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a)				,		__a)
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a)				,		__a)



		def 						UpperCAmelCase							(					self)       ->       Optional[int]:






							'''simple docstring'''
							_UpperCamelCase							 =  list(self.get_tokenizer().get_vocab().keys())

							self.assertEqual(vocab_keys[0]				,		'''<s>''')
							self.assertEqual(vocab_keys[1]				,		'''<pad>''')
							self.assertEqual(vocab_keys[-4]				,		'''œ''')
							self.assertEqual(vocab_keys[-2]				,		'''<mask>''')
							self.assertEqual(vocab_keys[-1]				,		'''<ctc_blank>''')
							self.assertEqual(len(__a)				,		81)



		def 						UpperCAmelCase							(					self)       ->       List[Any]:






							'''simple docstring'''
							self.assertEqual(self.get_tokenizer().vocab_size				,		79)



		def 						UpperCAmelCase							(					self)       ->       int:






							'''simple docstring'''
							_UpperCamelCase							 =  self.get_tokenizers(do_lower_case=__a)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''):
																	_UpperCamelCase							 =  tokenizer.vocab_size
																	_UpperCamelCase							 =  len(__a)

																	self.assertNotEqual(__a				,		0)

																	# We usually have added tokens from the start in tests because our vocab fixtures are
																	# smaller than the original vocabs - let's not assert this
																	# self.assertEqual(vocab_size, all_size)

																	_UpperCamelCase							 =  ['''aaaaa bbbbbb''', '''cccccccccdddddddd''']
																	_UpperCamelCase							 =  tokenizer.add_tokens(__a)
																	_UpperCamelCase							 =  tokenizer.vocab_size
																	_UpperCamelCase							 =  len(__a)

																	self.assertNotEqual(__a				,		0)
																	self.assertEqual(__a				,		__a)
																	self.assertEqual(__a				,		len(__a))
																	self.assertEqual(__a				,		all_size + len(__a))

																	_UpperCamelCase							 =  tokenizer.encode('''aaaaa bbbbbb low cccccccccdddddddd l'''				,		add_special_tokens=__a)

																	self.assertGreaterEqual(len(__a)				,		4)
																	self.assertGreater(tokens[0]				,		tokenizer.vocab_size - 1)
																	self.assertGreater(tokens[-3]				,		tokenizer.vocab_size - 1)

																	_UpperCamelCase							 =  {'''eos_token''': '''>>>>|||<||<<|<<''', '''pad_token''': '''<<<<<|||>|>>>>|>'''}
																	_UpperCamelCase							 =  tokenizer.add_special_tokens(__a)
																	_UpperCamelCase							 =  tokenizer.vocab_size
																	_UpperCamelCase							 =  len(__a)

																	self.assertNotEqual(__a				,		0)
																	self.assertEqual(__a				,		__a)
																	self.assertEqual(__a				,		len(__a))
																	self.assertEqual(__a				,		all_size_a + len(__a))

																	_UpperCamelCase							 =  tokenizer.encode(
																	    '''>>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd <<<<<|||>|>>>>|> l'''				,		add_special_tokens=__a)

																	self.assertGreaterEqual(len(__a)				,		6)
																	self.assertGreater(tokens[0]				,		tokenizer.vocab_size - 1)
																	self.assertGreater(tokens[0]				,		tokens[1])
																	self.assertGreater(tokens[-3]				,		tokenizer.vocab_size - 1)
																	self.assertGreater(tokens[-3]				,		tokens[-4])
																	self.assertEqual(tokens[0]				,		tokenizer.eos_token_id)
																	self.assertEqual(tokens[-3]				,		tokenizer.pad_token_id)



		def 						UpperCAmelCase							(					self)       ->       Any:






							'''simple docstring'''
							pass



		def 						UpperCAmelCase							(					self)       ->       int:






							'''simple docstring'''
							pass



		def 						UpperCAmelCase							(					self)       ->       str:






							'''simple docstring'''
							_UpperCamelCase							 =  self.get_tokenizer()

							_UpperCamelCase							 =  tokenizer.tokenize('''This is a test''')
							# fmt: off
							self.assertListEqual(__a				,		[SPIECE_UNDERLINE, '''T''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''a''', SPIECE_UNDERLINE, '''t''', '''e''', '''s''', '''t'''])
							# fmt: on

							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(__a)				,		[4, 32, 11, 10, 12, 4, 10, 12, 4, 7, 4, 6, 5, 12, 6]				,		)

							_UpperCamelCase							 =  tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
							self.assertListEqual(
							    __a				,		[SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''92000''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.'''])

							_UpperCamelCase							 =  tokenizer.convert_tokens_to_ids(__a)
							# fmt: off
							self.assertListEqual(__a				,		[4, 30, 4, 20, 7, 12, 4, 25, 8, 13, 9, 4, 10, 9, 4, 3, 23, 4, 7, 9, 14, 4, 6, 11, 10, 12, 4, 10, 12, 4, 19, 7, 15, 12, 73, 26])
							# fmt: on

							_UpperCamelCase							 =  tokenizer.convert_ids_to_tokens(__a)
							self.assertListEqual(
							    __a				,		[SPIECE_UNDERLINE, '''I''', SPIECE_UNDERLINE, '''w''', '''a''', '''s''', SPIECE_UNDERLINE, '''b''', '''o''', '''r''', '''n''', SPIECE_UNDERLINE, '''i''', '''n''', SPIECE_UNDERLINE, '''<unk>''', ''',''', SPIECE_UNDERLINE, '''a''', '''n''', '''d''', SPIECE_UNDERLINE, '''t''', '''h''', '''i''', '''s''', SPIECE_UNDERLINE, '''i''', '''s''', SPIECE_UNDERLINE, '''f''', '''a''', '''l''', '''s''', '''é''', '''.'''])




		@slow
		def 						UpperCAmelCase							(					self)       ->       Optional[int]:






							'''simple docstring'''
							# Use custom sequence because this tokenizer does not handle numbers.
							_UpperCamelCase							 =  [
							    '''Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides '''
							    '''general-purpose architectures (BERT, GPT, RoBERTa, XLM, DistilBert, XLNet...) for Natural '''
							    '''Language Understanding (NLU) and Natural Language Generation (NLG) with over thirty-two pretrained '''
							    '''models in one hundred plus languages and deep interoperability between Jax, PyTorch and TensorFlow.''',
							    '''BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly '''
							    '''conditioning on both left and right context in all layers.''',
							    '''The quick brown fox jumps over the lazy dog.''',
							]

							# fmt: off
							_UpperCamelCase							 =  {
							    '''input_ids''': [
							        [4, 32, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 64, 19, 8, 13, 18, 5, 13, 15, 22, 4, 28, 9, 8, 20, 9, 4, 7, 12, 4, 24, 22, 6, 8, 13, 17, 11, 39, 6, 13, 7, 9, 12, 19, 8, 13, 18, 5, 13, 12, 4, 7, 9, 14, 4, 24, 22, 6, 8, 13, 17, 11, 39, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 39, 25, 5, 13, 6, 63, 4, 24, 13, 8, 27, 10, 14, 5, 12, 4, 21, 5, 9, 5, 13, 7, 15, 39, 24, 16, 13, 24, 8, 12, 5, 4, 7, 13, 17, 11, 10, 6, 5, 17, 6, 16, 13, 5, 12, 4, 64, 40, 47, 54, 32, 23, 4, 53, 49, 32, 23, 4, 54, 8, 40, 47, 54, 32, 7, 23, 4, 69, 52, 43, 23, 4, 51, 10, 12, 6, 10, 15, 40, 5, 13, 6, 23, 4, 69, 52, 48, 5, 6, 26, 26, 26, 63, 4, 19, 8, 13, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 61, 9, 14, 5, 13, 12, 6, 7, 9, 14, 10, 9, 21, 4, 64, 48, 52, 61, 63, 4, 7, 9, 14, 4, 48, 7, 6, 16, 13, 7, 15, 4, 52, 7, 9, 21, 16, 7, 21, 5, 4, 53, 5, 9, 5, 13, 7, 6, 10, 8, 9, 4, 64, 48, 52, 53, 63, 4, 20, 10, 6, 11, 4, 8, 27, 5, 13, 4, 6, 11, 10, 13, 6, 22, 39, 6, 20, 8, 4, 24, 13, 5, 6, 13, 7, 10, 9, 5, 14, 4, 18, 8, 14, 5, 15, 12, 4, 10, 9, 4, 8, 9, 5, 4, 11, 16, 9, 14, 13, 5, 14, 4, 24, 15, 16, 12, 4, 15, 7, 9, 21, 16, 7, 21, 5, 12, 4, 7, 9, 14, 4, 14, 5, 5, 24, 4, 10, 9, 6, 5, 13, 8, 24, 5, 13, 7, 25, 10, 15, 10, 6, 22, 4, 25, 5, 6, 20, 5, 5, 9, 4, 58, 7, 37, 23, 4, 49, 22, 32, 8, 13, 17, 11, 4, 7, 9, 14, 4, 32, 5, 9, 12, 8, 13, 55, 15, 8, 20, 26, 2],
							        [4, 40, 47, 54, 32, 4, 10, 12, 4, 14, 5, 12, 10, 21, 9, 5, 14, 4, 6, 8, 4, 24, 13, 5, 39, 6, 13, 7, 10, 9, 4, 14, 5, 5, 24, 4, 25, 10, 14, 10, 13, 5, 17, 6, 10, 8, 9, 7, 15, 4, 13, 5, 24, 13, 5, 12, 5, 9, 6, 7, 6, 10, 8, 9, 12, 4, 19, 13, 8, 18, 4, 16, 9, 15, 7, 25, 5, 15, 5, 14, 4, 6, 5, 37, 6, 4, 25, 22, 4, 46, 8, 10, 9, 6, 15, 22, 4, 17, 8, 9, 14, 10, 6, 10, 8, 9, 10, 9, 21, 4, 8, 9, 4, 25, 8, 6, 11, 4, 15, 5, 19, 6, 4, 7, 9, 14, 4, 13, 10, 21, 11, 6, 4, 17, 8, 9, 6, 5, 37, 6, 4, 10, 9, 4, 7, 15, 15, 4, 15, 7, 22, 5, 13, 12, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [4, 32, 11, 5, 4, 45, 16, 10, 17, 28, 4, 25, 13, 8, 20, 9, 4, 19, 8, 37, 4, 46, 16, 18, 24, 12, 4, 8, 27, 5, 13, 4, 6, 11, 5, 4, 15, 7, 57, 22, 4, 14, 8, 21, 26, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							    ],
							    '''attention_mask''': [
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							        [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
							    ]
							}
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=__a				,		model_name='''microsoft/speecht5_asr'''				,		revision='''c5ef64c71905caeccde0e4462ef3f9077224c524'''				,		sequences=__a				,		)






