





'''simple docstring'''







import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase_					:    Optional[int]      	=			get_tests_dir('fixtures/spiece.model')
@require_sentencepiece
@require_tokenizers
class 				__SCREAMING_SNAKE_CASE  (snake_case_       ,							unittest.TestCase      ):






				"""simple docstring"""







				__a            =AlbertTokenizer
				__a            =AlbertTokenizerFast
				__a            =True
				__a            =True
				__a            =True




				def   UpperCamelCase__   (  self     :					int					):
								super().setUp()

								# We have a SentencePiece fixture for testing
								_a								= AlbertTokenizer(__a					)
								tokenizer.save_pretrained(self.tmpdirname					)




				def   UpperCamelCase__   (  self     :					Dict					,				__a     :					Dict					):
								_a								= "this is a test"
								_a								= "this is a test"
								return input_text, output_text




				def   UpperCamelCase__   (  self     :					Any					):
								_a								= "<pad>"
								_a								= 0

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a					)					,				__a					)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a					)					,				__a					)




				def   UpperCamelCase__   (  self     :					List[Any]					):
								_a								= list(self.get_tokenizer().get_vocab().keys()					)

								self.assertEqual(vocab_keys[0]					,				"<pad>"					)
								self.assertEqual(vocab_keys[1]					,				"<unk>"					)
								self.assertEqual(vocab_keys[-1]					,				"▁eloquent"					)
								self.assertEqual(len(__a					)					,				3_00_00					)




				def   UpperCamelCase__   (  self     :					Optional[int]					):
								self.assertEqual(self.get_tokenizer().vocab_size					,				3_00_00					)




				def   UpperCamelCase__   (  self     :					Optional[int]					):
								if not self.test_rust_tokenizer:
												return

								_a								= self.get_tokenizer()
								_a								= self.get_rust_tokenizer()

								_a								= "I was born in 92000, and this is falsé."

								_a								= tokenizer.tokenize(__a					)
								_a								= rust_tokenizer.tokenize(__a					)
								self.assertListEqual(__a					,				__a					)

								_a								= tokenizer.encode(__a					,				add_special_tokens=__a					)
								_a								= rust_tokenizer.encode(__a					,				add_special_tokens=__a					)
								self.assertListEqual(__a					,				__a					)

								_a								= self.get_rust_tokenizer()
								_a								= tokenizer.encode(__a					)
								_a								= rust_tokenizer.encode(__a					)
								self.assertListEqual(__a					,				__a					)




				def   UpperCamelCase__   (  self     :					Dict					):
								_a								= AlbertTokenizer(__a					,				keep_accents=__a					)

								_a								= tokenizer.tokenize("This is a test"					)
								self.assertListEqual(__a					,				["▁this", "▁is", "▁a", "▁test"]					)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(__a					)					,				[48, 25, 21, 12_89]					)

								_a								= tokenizer.tokenize("I was born in 92000, and this is falsé."					)
								self.assertListEqual(
								    __a					,				["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]					)
								_a								= tokenizer.convert_tokens_to_ids(__a					)
								self.assertListEqual(__a					,				[31, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]					)

								_a								= tokenizer.convert_ids_to_tokens(__a					)
								self.assertListEqual(
								    __a					,				["▁i", "▁was", "▁born", "▁in", "▁9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."]					,				)




				def   UpperCamelCase__   (  self     :					Tuple					):
								_a								= AlbertTokenizer(__a					)

								_a								= tokenizer.encode("sequence builders"					)
								_a								= tokenizer.encode("multi-sequence build"					)

								_a								= tokenizer.build_inputs_with_special_tokens(__a					)
								_a								= tokenizer.build_inputs_with_special_tokens(__a					,				__a					)

								assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
								assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
								    tokenizer.sep_token_id
								]




				@slow
				def   UpperCamelCase__   (  self     :					Dict					):
								# fmt: off
								_a								= {"attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "input_ids": [[2, 2_19_70, 13, 5, 60_92, 1_67, 28, 71_03, 21_53, 6_73, 8, 70_28, 1_20_51, 18, 17, 71_03, 21_53, 6_73, 8, 35_15, 1_86_84, 8, 44_61, 6, 19_27, 2_97, 8, 1_20_60, 26_07, 18, 13, 5, 44_61, 15, 1_05_38, 38, 8, 1_35, 15, 8_22, 58, 15, 9_93, 1_03_63, 15, 14_60, 80_05, 44_61, 15, 9_93, 2_55, 23_28, 9, 9, 9, 6, 26, 11_12, 8_16, 32_60, 13, 5, 1_03, 23_77, 6, 17, 11_12, 8_16, 27_82, 13, 5, 1_03, 1_06_41, 6, 29, 84, 25_12, 24_30, 7_82, 1_86_84, 27_61, 19, 8_08, 24_30, 25_56, 17, 8_55, 14_80, 94_77, 40_91, 1_28, 1_17_12, 15, 71_03, 21_53, 6_73, 17, 2_48_83, 99_90, 9, 3], [2, 1_15_02, 25, 10_06, 20, 7_82, 8, 1_18_09, 8_55, 17_32, 1_93_93, 1_86_67, 37, 3_67, 2_10_18, 69, 18_54, 34, 1_18_60, 1_91_24, 27, 1_56, 2_25, 17, 1_93, 41_41, 19, 65, 91_24, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 22_31, 8_86, 23_85, 1_76_59, 84, 14, 1_67_92, 19_52, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=__a					,				model_name="albert-base-v2"					,				revision="6b6560eaf5ff2e250b00c50f380c5389a9c2d82e"					,				)








import argparse
import os

import torch

from transformers import (
    XLNetConfig,
    XLNetForQuestionAnswering,
    XLNetForSequenceClassification,
    XLNetLMHeadModel,
    load_tf_weights_in_xlnet,
)
from transformers.utils import CONFIG_NAME, WEIGHTS_NAME, logging


lowercase  							=					{
    "cola": 2,
    "mnli": 3,
    "mrpc": 2,
    "sst-2": 2,
    "sts-b": 1,
    "qqp": 2,
    "qnli": 2,
    "rte": 2,
    "wnli": 2,
}


logging.set_verbosity_info()





def 						__UpperCAmelCase			(     a_							,     a_							,     a_							,     a_=None):
							# Initialise PyTorch model
							snake_case_				=							XLNetConfig.from_json_file(a_)

							snake_case_				=							finetuning_task.lower() if finetuning_task is not None else ''
							if finetuning_task in GLUE_TASKS_NUM_LABELS:
														print(f'''Building PyTorch XLNetForSequenceClassification model from configuration: {config}''')
														snake_case_				=							finetuning_task
														snake_case_				=							GLUE_TASKS_NUM_LABELS[finetuning_task]
														snake_case_				=							XLNetForSequenceClassification(a_)
							elif "squad" in finetuning_task:
														snake_case_				=							finetuning_task
														snake_case_				=							XLNetForQuestionAnswering(a_)
							else:
														snake_case_				=							XLNetLMHeadModel(a_)

							# Load weights from tf checkpoint
							load_tf_weights_in_xlnet(a_							,     a_							,     a_)

							# Save pytorch-model
							snake_case_				=							os.path.join(a_							,     a_)
							snake_case_				=							os.path.join(a_							,     a_)
							print(f'''Save PyTorch model to {os.path.abspath(a_)}''')
							torch.save(model.state_dict()							,     a_)
							print(f'''Save configuration file to {os.path.abspath(a_)}''')
							with open(a_							,     'w'							,     encoding='utf-8') as f:
														f.write(config.to_json_string())


if __name__ == "__main__":
	lowercase  							=					argparse.ArgumentParser()
	# Required parameters
	parser.add_argument(
	    "--tf_checkpoint_path", default=None, type=str, required=True, help="Path to the TensorFlow checkpoint path."
	)
	parser.add_argument(
	    "--xlnet_config_file",
	    default=None,
	    type=str,
	    required=True,
	    help=(
	        "The config json file corresponding to the pre-trained XLNet model. \n"
	        "This specifies the model architecture."
	    ),
	)
	parser.add_argument(
	    "--pytorch_dump_folder_path",
	    default=None,
	    type=str,
	    required=True,
	    help="Path to the folder to store the PyTorch model or dataset/vocab.",
	)
	parser.add_argument(
	    "--finetuning_task",
	    default=None,
	    type=str,
	    help="Name of a task on which the XLNet TensorFlow model was fine-tuned",
	)
	lowercase  							=					parser.parse_args()
	print(args)

	convert_xlnet_checkpoint_to_pytorch(
	    args.tf_checkpoint_path, args.xlnet_config_file, args.pytorch_dump_folder_path, args.finetuning_task
	)





