


from argparse import ArgumentParser, Namespace

from ..utils import logging
from . import BaseTransformersCLICommand






def 							__lowercase   ( __lowerCAmelCase							:				Tuple   ):
							return ConvertCommand(
							    args.model_type					,    args.tf_checkpoint					,    args.pytorch_dump_output					,    args.config					,    args.finetuning_task_name   )


snake_case							:       Optional[int]     			=      '''
transformers can only be used from the commandline to convert TensorFlow models in PyTorch, In that case, it requires
TensorFlow to be installed. Please see https://www.tensorflow.org/install/ for installation instructions.
'''







class        snake_case_   (_UpperCAmelCase      ):
					@staticmethod
					def  lowerCamelCase__(      __snake_case   :ArgumentParser      )					->    Optional[Any]:
												a__             =     parser.add_parser(
												    'convert'			,help='CLI tool to run convert model from original author checkpoints to Transformers PyTorch checkpoints.'			,)
												train_parser.add_argument('--model_type'			,type=UpperCAmelCase_			,required=UpperCAmelCase_			,help='Model\'s type.'      )
												train_parser.add_argument(
												    '--tf_checkpoint'			,type=UpperCAmelCase_			,required=UpperCAmelCase_			,help='TensorFlow checkpoint path or folder.'      )
												train_parser.add_argument(
												    '--pytorch_dump_output'			,type=UpperCAmelCase_			,required=UpperCAmelCase_			,help='Path to the PyTorch saved model output.'      )
												train_parser.add_argument('--config'			,type=UpperCAmelCase_			,default=''			,help='Configuration file path or folder.'      )
												train_parser.add_argument(
												    '--finetuning_task_name'			,type=UpperCAmelCase_			,default=UpperCAmelCase_			,help='Optional fine-tuning task name if the TF model was a finetuned model.'			,)
												train_parser.set_defaults(func=UpperCAmelCase_      )
					def __init__(      self   :str			,__snake_case   :str			,__snake_case   :str			,__snake_case   :str			,__snake_case   :str			,__snake_case   :str			,*__snake_case   :Optional[int]			,)					->    Dict:
												a__             =     logging.get_logger('transformers-cli/converting'      )

												self._logger.info(F'Loading model {model_type}'      )
												a__             =     model_type
												a__             =     tf_checkpoint
												a__             =     pytorch_dump_output
												a__             =     config
												a__             =     finetuning_task_name



					def  lowerCamelCase__(      self   :str      )					->    int:
												if self._model_type == "albert":
																			try:
																										from ..models.albert.convert_albert_original_tf_checkpoint_to_pytorch import (
																										    convert_tf_checkpoint_to_pytorch,
																										)
																			except ImportError:
																										raise ImportError(UpperCAmelCase_      )

																			convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,self._config			,self._pytorch_dump_output      )
												elif self._model_type == "bert":
																			try:
																										from ..models.bert.convert_bert_original_tf_checkpoint_to_pytorch import (
																										    convert_tf_checkpoint_to_pytorch,
																										)
																			except ImportError:
																										raise ImportError(UpperCAmelCase_      )

																			convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,self._config			,self._pytorch_dump_output      )
												elif self._model_type == "funnel":
																			try:
																										from ..models.funnel.convert_funnel_original_tf_checkpoint_to_pytorch import (
																										    convert_tf_checkpoint_to_pytorch,
																										)
																			except ImportError:
																										raise ImportError(UpperCAmelCase_      )

																			convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,self._config			,self._pytorch_dump_output      )
												elif self._model_type == "t5":
																			try:
																										from ..models.ta.convert_ta_original_tf_checkpoint_to_pytorch import convert_tf_checkpoint_to_pytorch
																			except ImportError:
																										raise ImportError(UpperCAmelCase_      )

																			convert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,self._config			,self._pytorch_dump_output      )
												elif self._model_type == "gpt":
																			from ..models.openai.convert_openai_original_tf_checkpoint_to_pytorch import (
																			    convert_openai_checkpoint_to_pytorch,
																			)

																			convert_openai_checkpoint_to_pytorch(self._tf_checkpoint			,self._config			,self._pytorch_dump_output      )
												elif self._model_type == "transfo_xl":
																			try:
																										from ..models.transfo_xl.convert_transfo_xl_original_tf_checkpoint_to_pytorch import (
																										    convert_transfo_xl_checkpoint_to_pytorch,
																										)
																			except ImportError:
																										raise ImportError(UpperCAmelCase_      )

																			if "ckpt" in self._tf_checkpoint.lower():
																										a__             =     self._tf_checkpoint
																										a__             =     ''
																			else:
																										a__             =     self._tf_checkpoint
																										a__             =     ''
																			convert_transfo_xl_checkpoint_to_pytorch(
																			    UpperCAmelCase_			,self._config			,self._pytorch_dump_output			,UpperCAmelCase_      )
												elif self._model_type == "gpt2":
																			try:
																										from ..models.gpta.convert_gpta_original_tf_checkpoint_to_pytorch import (
																										    convert_gpta_checkpoint_to_pytorch,
																										)
																			except ImportError:
																										raise ImportError(UpperCAmelCase_      )

																			convert_gpta_checkpoint_to_pytorch(self._tf_checkpoint			,self._config			,self._pytorch_dump_output      )
												elif self._model_type == "xlnet":
																			try:
																										from ..models.xlnet.convert_xlnet_original_tf_checkpoint_to_pytorch import (
																										    convert_xlnet_checkpoint_to_pytorch,
																										)
																			except ImportError:
																										raise ImportError(UpperCAmelCase_      )

																			convert_xlnet_checkpoint_to_pytorch(
																			    self._tf_checkpoint			,self._config			,self._pytorch_dump_output			,self._finetuning_task_name      )
												elif self._model_type == "xlm":
																			from ..models.xlm.convert_xlm_original_pytorch_checkpoint_to_pytorch import (
																			    convert_xlm_checkpoint_to_pytorch,
																			)

																			convert_xlm_checkpoint_to_pytorch(self._tf_checkpoint			,self._pytorch_dump_output      )
												elif self._model_type == "lxmert":
																			from ..models.lxmert.convert_lxmert_original_tf_checkpoint_to_pytorch import (
																			    convert_lxmert_checkpoint_to_pytorch,
																			)

																			convert_lxmert_checkpoint_to_pytorch(self._tf_checkpoint			,self._pytorch_dump_output      )
												elif self._model_type == "rembert":
																			from ..models.rembert.convert_rembert_tf_checkpoint_to_pytorch import (
																			    convert_rembert_tf_checkpoint_to_pytorch,
																			)

																			convert_rembert_tf_checkpoint_to_pytorch(self._tf_checkpoint			,self._config			,self._pytorch_dump_output      )
												else:
																			raise ValueError(
																			    '--model_type should be selected in the list [bert, gpt, gpt2, t5, transfo_xl, xlnet, xlm, lxmert]'      )






import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


__snake_case								=  get_tests_dir("""fixtures/test_sentencepiece.model""")




@require_sentencepiece
class 						lowercase__  (						_UpperCAmelCase     ,       unittest.TestCase  ):
	A__    :   List[str]				       =XLMProphetNetTokenizer
	A__    :   List[Any]				       =False
	A__    :   Tuple				       =True
	def      A_				(			self							:   str							):
						super().setUp()

						# We have a SentencePiece fixture for testing
						SCREAMING_SNAKE_CASE__													= XLMProphetNetTokenizer(UpperCAmelCase_       ,   keep_accents=UpperCAmelCase_							)
						tokenizer.save_pretrained(self.tmpdirname							)
	def      A_				(			self							:   Tuple							):
						SCREAMING_SNAKE_CASE__													= '[PAD]'
						SCREAMING_SNAKE_CASE__													= 0

						self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase_							)       ,   UpperCAmelCase_							)
						self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase_							)       ,   UpperCAmelCase_							)
	def      A_				(			self							:   Tuple							):
						SCREAMING_SNAKE_CASE__													= list(self.get_tokenizer().get_vocab().keys()							)

						self.assertEqual(vocab_keys[0]       ,   '[PAD]'							)
						self.assertEqual(vocab_keys[1]       ,   '[CLS]'							)
						self.assertEqual(vocab_keys[-1]       ,   'j'							)
						self.assertEqual(len(UpperCAmelCase_							)       ,   1012							)
	def      A_				(			self							:   List[Any]							):
						self.assertEqual(self.get_tokenizer().vocab_size       ,   1012							)
	def      A_				(			self							:   Optional[int]							):
						SCREAMING_SNAKE_CASE__													= XLMProphetNetTokenizer(UpperCAmelCase_       ,   keep_accents=UpperCAmelCase_							)

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize('This is a test'							)
						self.assertListEqual(UpperCAmelCase_       ,   ['▁This', '▁is', '▁a', '▁t', 'est']							)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)       ,   [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,   )

						SCREAMING_SNAKE_CASE__													= tokenizer.tokenize('I was born in 92000, and this is falsé.'							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '9',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        'é',
						        '.',
						    ]       ,   )
						SCREAMING_SNAKE_CASE__													= tokenizer.convert_tokens_to_ids(UpperCAmelCase_							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
						    ]       ,   )

						SCREAMING_SNAKE_CASE__													= tokenizer.convert_ids_to_tokens(UpperCAmelCase_							)
						self.assertListEqual(
						    UpperCAmelCase_       ,   [
						        SPIECE_UNDERLINE + 'I',
						        SPIECE_UNDERLINE + 'was',
						        SPIECE_UNDERLINE + 'b',
						        'or',
						        'n',
						        SPIECE_UNDERLINE + 'in',
						        SPIECE_UNDERLINE + '',
						        '[UNK]',
						        '2',
						        '0',
						        '0',
						        '0',
						        ',',
						        SPIECE_UNDERLINE + 'and',
						        SPIECE_UNDERLINE + 'this',
						        SPIECE_UNDERLINE + 'is',
						        SPIECE_UNDERLINE + 'f',
						        'al',
						        's',
						        '[UNK]',
						        '.',
						    ]       ,   )
	@cached_property
	def      A_				(			self							:   Optional[Any]							):
						return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'							)
	@slow
	def      A_				(			self							:   Any							):
						SCREAMING_SNAKE_CASE__													= 'Hello World!'
						SCREAMING_SNAKE_CASE__													= [35389, 6672, 49, 2]
						self.assertListEqual(UpperCAmelCase_       ,   self.big_tokenizer.encode(UpperCAmelCase_							)							)





	@slow
	def      A_				(			self							:   Tuple							):
						# fmt: off
						SCREAMING_SNAKE_CASE__													= {'input_ids': [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
						# fmt: on

						self.tokenizer_integration_test_util(
						    expected_encoding=UpperCAmelCase_       ,   model_name='microsoft/xprophetnet-large-wiki100-cased'       ,   revision='1acad1643ddd54a44df6a1b797ada8373685d90e'       ,   )





