
import logging
import os
import sys
from dataclasses import dataclass, field
from importlib import import_module
from typing import Dict, List, Optional, Tuple

import numpy as np
from seqeval.metrics import accuracy_score, fa_score, precision_score, recall_score
from torch import nn
from utils_ner import Split, TokenClassificationDataset, TokenClassificationTask

import transformers
from transformers import (
    AutoConfig,
    AutoModelForTokenClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    set_seed,
)
from transformers.trainer_utils import is_main_process


lowerCamelCase__          =		logging.getLogger(__name__)







@dataclass
class      A__							:
				lowercase			    =  field(
				    metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}	)
				lowercase			    =  field(
				    default=__magic_name__						,   metadata={'help': 'Pretrained config name or path if not the same as model_name'}	)
				lowercase			    =  field(
				    default='NER'						,   metadata={'help': 'Task type to fine tune in training (e.g. NER, POS, etc)'}	)
				lowercase			    =  field(
				    default=__magic_name__						,   metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}	)
				lowercase			    =  field(default=__magic_name__						,   metadata={'help': 'Set this flag to use fast tokenization.'}	)
				# If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
				# or just modify its tokenizer_config.json.
				lowercase			    =  field(
				    default=__magic_name__						,   metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}						,   )







@dataclass
class      A__							:
				lowercase			    =  field(
				    metadata={'help': 'The input data dir. Should contain the .txt files for a CoNLL-2003-formatted task.'}	)
				lowercase			    =  field(
				    default=__magic_name__						,   metadata={'help': 'Path to a file containing all labels. If not specified, CoNLL-2003 labels are used.'}						,   )
				lowercase			    =  field(
				    default=128						,   metadata={
				        'help': (
				            'The maximum total input sequence length after tokenization. Sequences longer '
				            'than this will be truncated, sequences shorter will be padded.'
				        )
				    }						,   )
				lowercase			    =  field(
				    default=__magic_name__						,   metadata={'help': 'Overwrite the cached training and evaluation sets'}	)






def  lowerCAmelCase__			(  )				->		int:
			# See all possible arguments in src/transformers/training_args.py
			# or by passing the --help flag to this script.
			# We now keep distinct sets of args, for a cleaner separation of concerns.

			lowerCAmelCase__      :      str         =		HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )
			if len(sys.argv     ) == 2 and sys.argv[1].endswith('.json'     ):
						# If we pass only one argument to the script and it's the path to a json file,
						# let's parse it to get our arguments.
						lowerCAmelCase__      ,				lowerCAmelCase__      ,				lowerCAmelCase__      :      List[Any]         =		parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]     )     )
			else:
						lowerCAmelCase__      ,				lowerCAmelCase__      ,				lowerCAmelCase__      :      Optional[Any]         =		parser.parse_args_into_dataclasses()

			if (
			    os.path.exists(training_args.output_dir     )
			    and os.listdir(training_args.output_dir     )
			    and training_args.do_train
			    and not training_args.overwrite_output_dir
			):
						raise ValueError(
						    F'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
						    ' --overwrite_output_dir to overcome.'     )

			lowerCAmelCase__      :      List[str]         =		import_module('tasks'     )
			try:
						lowerCAmelCase__      :      str         =		getattr(SCREAMING_SNAKE_CASE_       ,							model_args.task_type     )
						lowerCAmelCase__      :      TokenClassificationTask         =		token_classification_task_clazz()
			except AttributeError:
						raise ValueError(
						    F'''Task {model_args.task_type} needs to be defined as a TokenClassificationTask subclass in {module}. '''
						    F'''Available tasks classes are: {TokenClassificationTask.__subclasses__()}'''     )

			# Setup logging
			logging.basicConfig(
			    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'       ,							datefmt='%m/%d/%Y %H:%M:%S'       ,							level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN       ,							)
			logger.warning(
			    'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'       ,							training_args.local_rank       ,							training_args.device       ,							training_args.n_gpu       ,							bool(training_args.local_rank != -1     )       ,							training_args.fpaa       ,							)
			# Set the verbosity to info of the Transformers logger (on main process only):
			if is_main_process(training_args.local_rank     ):
						transformers.utils.logging.set_verbosity_info()
						transformers.utils.logging.enable_default_handler()
						transformers.utils.logging.enable_explicit_format()
			logger.info('Training/evaluation parameters %s'       ,							SCREAMING_SNAKE_CASE_     )

			# Set seed
			set_seed(training_args.seed     )

			# Prepare CONLL-2003 task
			lowerCAmelCase__      :      str         =		token_classification_task.get_labels(data_args.labels     )
			lowerCAmelCase__      :      Dict[int, str]         =		dict(enumerate(SCREAMING_SNAKE_CASE_     )     )
			lowerCAmelCase__      :      Any         =		len(SCREAMING_SNAKE_CASE_     )

			# Load pretrained model and tokenizer
			#
			# Distributed training:
			# The .from_pretrained methods guarantee that only one local process can concurrently
			# download model & vocab.

			lowerCAmelCase__      :      Optional[int]         =		AutoConfig.from_pretrained(
			    model_args.config_name if model_args.config_name else model_args.model_name_or_path       ,							num_labels=SCREAMING_SNAKE_CASE_       ,							idalabel=SCREAMING_SNAKE_CASE_       ,							labelaid={label: i for i, label in enumerate(SCREAMING_SNAKE_CASE_     )}       ,							cache_dir=model_args.cache_dir       ,							)
			lowerCAmelCase__      :      int         =		AutoTokenizer.from_pretrained(
			    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path       ,							cache_dir=model_args.cache_dir       ,							use_fast=model_args.use_fast       ,							)
			lowerCAmelCase__      :      Union[str, Any]         =		AutoModelForTokenClassification.from_pretrained(
			    model_args.model_name_or_path       ,							from_tf=bool('.ckpt' in model_args.model_name_or_path     )       ,							config=SCREAMING_SNAKE_CASE_       ,							cache_dir=model_args.cache_dir       ,							)

			# Get datasets
			lowerCAmelCase__      :      str         =		(
			    TokenClassificationDataset(
			        token_classification_task=SCREAMING_SNAKE_CASE_       ,							data_dir=data_args.data_dir       ,							tokenizer=SCREAMING_SNAKE_CASE_       ,							labels=SCREAMING_SNAKE_CASE_       ,							model_type=config.model_type       ,							max_seq_length=data_args.max_seq_length       ,							overwrite_cache=data_args.overwrite_cache       ,							mode=Split.train       ,							)
			    if training_args.do_train
			    else None
			)
			lowerCAmelCase__      :      Optional[Any]         =		(
			    TokenClassificationDataset(
			        token_classification_task=SCREAMING_SNAKE_CASE_       ,							data_dir=data_args.data_dir       ,							tokenizer=SCREAMING_SNAKE_CASE_       ,							labels=SCREAMING_SNAKE_CASE_       ,							model_type=config.model_type       ,							max_seq_length=data_args.max_seq_length       ,							overwrite_cache=data_args.overwrite_cache       ,							mode=Split.dev       ,							)
			    if training_args.do_eval
			    else None
			)

			def align_predictions(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ) -> Tuple[List[int], List[int]]:
						lowerCAmelCase__      :      Dict         =		np.argmax(SCREAMING_SNAKE_CASE_       ,							axis=2     )

						lowerCAmelCase__      ,				lowerCAmelCase__      :      Optional[Any]         =		preds.shape

						lowerCAmelCase__      :      List[Any]         =		[[] for _ in range(SCREAMING_SNAKE_CASE_     )]
						lowerCAmelCase__      :      int         =		[[] for _ in range(SCREAMING_SNAKE_CASE_     )]

						for i in range(SCREAMING_SNAKE_CASE_     ):
									for j in range(SCREAMING_SNAKE_CASE_     ):
												if label_ids[i, j] != nn.CrossEntropyLoss().ignore_index:
															out_label_list[i].append(label_map[label_ids[i][j]]     )
															preds_list[i].append(label_map[preds[i][j]]     )

						return preds_list, out_label_list

			def compute_metrics(SCREAMING_SNAKE_CASE_     ) -> Dict:
						lowerCAmelCase__      ,				lowerCAmelCase__      :      int         =		align_predictions(p.predictions       ,							p.label_ids     )
						return {
						    "accuracy_score": accuracy_score(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ),
						    "precision": precision_score(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ),
						    "recall": recall_score(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ),
						    "f1": fa_score(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ),
						}

			# Data collator
			lowerCAmelCase__      :      Optional[int]         =		DataCollatorWithPadding(SCREAMING_SNAKE_CASE_       ,							pad_to_multiple_of=8     ) if training_args.fpaa else None

			# Initialize our Trainer
			lowerCAmelCase__      :      Optional[Any]         =		Trainer(
			    model=SCREAMING_SNAKE_CASE_       ,							args=SCREAMING_SNAKE_CASE_       ,							train_dataset=SCREAMING_SNAKE_CASE_       ,							eval_dataset=SCREAMING_SNAKE_CASE_       ,							compute_metrics=SCREAMING_SNAKE_CASE_       ,							data_collator=SCREAMING_SNAKE_CASE_       ,							)

			# Training
			if training_args.do_train:
						trainer.train(
						    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
						trainer.save_model()
						# For convenience, we also re-save the tokenizer to the same directory,
						# so that you can share your model easily on huggingface.co/models =)
						if trainer.is_world_process_zero():
									tokenizer.save_pretrained(training_args.output_dir     )

    # Evaluation
			lowerCAmelCase__      :      Optional[Any]         =		{}
			if training_args.do_eval:
						logger.info('*** Evaluate ***'     )

						lowerCAmelCase__      :      List[str]         =		trainer.evaluate()

						lowerCAmelCase__      :      Any         =		os.path.join(training_args.output_dir       ,							'eval_results.txt'     )
						if trainer.is_world_process_zero():
									with open(SCREAMING_SNAKE_CASE_       ,							'w'     ) as writer:
												logger.info('***** Eval results *****'     )
												for key, value in result.items():
															logger.info('  %s = %s'       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
															writer.write('%s = %s\n' % (key, value)     )

									results.update(SCREAMING_SNAKE_CASE_     )

    # Predict
			if training_args.do_predict:
						lowerCAmelCase__      :      Any         =		TokenClassificationDataset(
						    token_classification_task=SCREAMING_SNAKE_CASE_       ,							data_dir=data_args.data_dir       ,							tokenizer=SCREAMING_SNAKE_CASE_       ,							labels=SCREAMING_SNAKE_CASE_       ,							model_type=config.model_type       ,							max_seq_length=data_args.max_seq_length       ,							overwrite_cache=data_args.overwrite_cache       ,							mode=Split.test       ,							)

						lowerCAmelCase__      ,				lowerCAmelCase__      ,				lowerCAmelCase__      :      Dict         =		trainer.predict(SCREAMING_SNAKE_CASE_     )
						lowerCAmelCase__      ,				lowerCAmelCase__      :      Any         =		align_predictions(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )

						lowerCAmelCase__      :      List[str]         =		os.path.join(training_args.output_dir       ,							'test_results.txt'     )
						if trainer.is_world_process_zero():
									with open(SCREAMING_SNAKE_CASE_       ,							'w'     ) as writer:
												for key, value in metrics.items():
															logger.info('  %s = %s'       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
															writer.write('%s = %s\n' % (key, value)     )

        # Save predictions
						lowerCAmelCase__      :      Dict         =		os.path.join(training_args.output_dir       ,							'test_predictions.txt'     )
						if trainer.is_world_process_zero():
									with open(SCREAMING_SNAKE_CASE_       ,							'w'     ) as writer:
												with open(os.path.join(data_args.data_dir       ,							'test.txt'     )       ,							'r'     ) as f:
															token_classification_task.write_predictions_to_file(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )

			return results
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		List[str]:
			# For xla_spawn (TPUs)
			main()


if __name__ == "__main__":
					main()

import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__          =		get_tests_dir("""fixtures/spiece.model""")







@require_sentencepiece
@require_tokenizers
class      A__							(     __magic_name__						,   unittest.TestCase	):
				lowercase			    =  AlbertTokenizer
				lowercase			    =  AlbertTokenizerFast
				lowercase			    =  True
				lowercase			    =  True
				lowercase			    =  True






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							super().setUp()

							# We have a SentencePiece fixture for testing
							lowerCAmelCase__      :      int         =		AlbertTokenizer(a   )
							tokenizer.save_pretrained(self.tmpdirname   )






				def 							_lowerCamelCase		(						self					:      List[str]    ,     a					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		'this is a test'
							lowerCAmelCase__      :      List[Any]         =		'this is a test'
							return input_text, output_text






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		'<pad>'
							lowerCAmelCase__      :      Optional[Any]         =		0

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(a   )    ,     a   )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(a   )    ,     a   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		list(self.get_tokenizer().get_vocab().keys()   )

							self.assertEqual(vocab_keys[0]    ,     '<pad>'   )
							self.assertEqual(vocab_keys[1]    ,     '<unk>'   )
							self.assertEqual(vocab_keys[-1]    ,     '▁eloquent'   )
							self.assertEqual(len(a   )    ,     30_000   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							self.assertEqual(self.get_tokenizer().vocab_size    ,     30_000   )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							if not self.test_rust_tokenizer:
										return

							lowerCAmelCase__      :      str         =		self.get_tokenizer()
							lowerCAmelCase__      :      str         =		self.get_rust_tokenizer()

							lowerCAmelCase__      :      List[Any]         =		'I was born in 92000, and this is falsé.'

							lowerCAmelCase__      :      str         =		tokenizer.tokenize(a   )
							lowerCAmelCase__      :      Optional[int]         =		rust_tokenizer.tokenize(a   )
							self.assertListEqual(a    ,     a   )

							lowerCAmelCase__      :      Tuple         =		tokenizer.encode(a    ,     add_special_tokens=a   )
							lowerCAmelCase__      :      Union[str, Any]         =		rust_tokenizer.encode(a    ,     add_special_tokens=a   )
							self.assertListEqual(a    ,     a   )

							lowerCAmelCase__      :      Optional[Any]         =		self.get_rust_tokenizer()
							lowerCAmelCase__      :      Dict         =		tokenizer.encode(a   )
							lowerCAmelCase__      :      List[Any]         =		rust_tokenizer.encode(a   )
							self.assertListEqual(a    ,     a   )






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		AlbertTokenizer(a    ,     keep_accents=a   )

							lowerCAmelCase__      :      Union[str, Any]         =		tokenizer.tokenize('This is a test'   )
							self.assertListEqual(a    ,     ['▁this', '▁is', '▁a', '▁test']   )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(a   )    ,     [48, 25, 21, 1_289]   )

							lowerCAmelCase__      :      Tuple         =		tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
							self.assertListEqual(
							    a    ,     ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', 'é', '.']   )
							lowerCAmelCase__      :      Any         =		tokenizer.convert_tokens_to_ids(a   )
							self.assertListEqual(a    ,     [31, 23, 386, 19, 561, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]   )

							lowerCAmelCase__      :      Any         =		tokenizer.convert_ids_to_tokens(a   )
							self.assertListEqual(
							    a    ,     ['▁i', '▁was', '▁born', '▁in', '▁9', '2000', ',', '▁and', '▁this', '▁is', '▁fal', 's', '<unk>', '.']    ,     )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		AlbertTokenizer(a   )

							lowerCAmelCase__      :      Tuple         =		tokenizer.encode('sequence builders'   )
							lowerCAmelCase__      :      Any         =		tokenizer.encode('multi-sequence build'   )

							lowerCAmelCase__      :      Dict         =		tokenizer.build_inputs_with_special_tokens(a   )
							lowerCAmelCase__      :      Tuple         =		tokenizer.build_inputs_with_special_tokens(a    ,     a   )

							assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
							assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
							    tokenizer.sep_token_id
							]






				@slow
				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Dict         =		{'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'input_ids': [[2, 21_970, 13, 5, 6_092, 167, 28, 7_103, 2_153, 673, 8, 7_028, 12_051, 18, 17, 7_103, 2_153, 673, 8, 3_515, 18_684, 8, 4_461, 6, 1_927, 297, 8, 12_060, 2_607, 18, 13, 5, 4_461, 15, 10_538, 38, 8, 135, 15, 822, 58, 15, 993, 10_363, 15, 1_460, 8_005, 4_461, 15, 993, 255, 2_328, 9, 9, 9, 6, 26, 1_112, 816, 3_260, 13, 5, 103, 2_377, 6, 17, 1_112, 816, 2_782, 13, 5, 103, 10_641, 6, 29, 84, 2_512, 2_430, 782, 18_684, 2_761, 19, 808, 2_430, 2_556, 17, 855, 1_480, 9_477, 4_091, 128, 11_712, 15, 7_103, 2_153, 673, 17, 24_883, 9_990, 9, 3], [2, 11_502, 25, 1_006, 20, 782, 8, 11_809, 855, 1_732, 19_393, 18_667, 37, 367, 21_018, 69, 1_854, 34, 11_860, 19_124, 27, 156, 225, 17, 193, 4_141, 19, 65, 9_124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2_231, 886, 2_385, 17_659, 84, 14, 16_792, 1_952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=a    ,     model_name='albert-base-v2'    ,     revision='6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'    ,     )