



import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase__	=       get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
class       SCREAMING_SNAKE_CASE	(							__UpperCAmelCase	,					unittest.TestCase    ):
							__lowerCamelCase		:		List[Any]			=XLMProphetNetTokenizer
							__lowerCamelCase		:		Any			=False
							__lowerCamelCase		:		List[Any]			=True





							def   UpperCamelCase_    (      self						:						int		):

											'''simple docstring'''
											super().setUp()

											# We have a SentencePiece fixture for testing
											__a        =						XLMProphetNetTokenizer(__lowercase					, keep_accents=__lowercase		)
											tokenizer.save_pretrained(self.tmpdirname		)





							def   UpperCamelCase_    (      self						:						str		):

											'''simple docstring'''
											__a        =						"""[PAD]"""
											__a        =						0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowercase		)					, __lowercase		)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowercase		)					, __lowercase		)





							def   UpperCamelCase_    (      self						:						int		):

											'''simple docstring'''
											__a        =						list(self.get_tokenizer().get_vocab().keys()		)

											self.assertEqual(vocab_keys[0]					, """[PAD]"""		)
											self.assertEqual(vocab_keys[1]					, """[CLS]"""		)
											self.assertEqual(vocab_keys[-1]					, """j"""		)
											self.assertEqual(len(__lowercase		)					, 1012		)





							def   UpperCamelCase_    (      self						:						Dict		):

											'''simple docstring'''
											self.assertEqual(self.get_tokenizer().vocab_size					, 1012		)





							def   UpperCamelCase_    (      self						:						int		):

											'''simple docstring'''
											__a        =						XLMProphetNetTokenizer(__lowercase					, keep_accents=__lowercase		)

											__a        =						tokenizer.tokenize("""This is a test"""		)
											self.assertListEqual(__lowercase					, ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]		)

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(__lowercase		)					, [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]					, )

											__a        =						tokenizer.tokenize("""I was born in 92000, and this is falsé."""		)
											self.assertListEqual(
											    __lowercase					, [
											        SPIECE_UNDERLINE + """I""",
											        SPIECE_UNDERLINE + """was""",
											        SPIECE_UNDERLINE + """b""",
											        """or""",
											        """n""",
											        SPIECE_UNDERLINE + """in""",
											        SPIECE_UNDERLINE + """""",
											        """9""",
											        """2""",
											        """0""",
											        """0""",
											        """0""",
											        """,""",
											        SPIECE_UNDERLINE + """and""",
											        SPIECE_UNDERLINE + """this""",
											        SPIECE_UNDERLINE + """is""",
											        SPIECE_UNDERLINE + """f""",
											        """al""",
											        """s""",
											        """é""",
											        """.""",
											    ]					, )
											__a        =						tokenizer.convert_tokens_to_ids(__lowercase		)
											self.assertListEqual(
											    __lowercase					, [
											        value + tokenizer.fairseq_offset
											        for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
											    ]					, )

											__a        =						tokenizer.convert_ids_to_tokens(__lowercase		)
											self.assertListEqual(
											    __lowercase					, [
											        SPIECE_UNDERLINE + """I""",
											        SPIECE_UNDERLINE + """was""",
											        SPIECE_UNDERLINE + """b""",
											        """or""",
											        """n""",
											        SPIECE_UNDERLINE + """in""",
											        SPIECE_UNDERLINE + """""",
											        """[UNK]""",
											        """2""",
											        """0""",
											        """0""",
											        """0""",
											        """,""",
											        SPIECE_UNDERLINE + """and""",
											        SPIECE_UNDERLINE + """this""",
											        SPIECE_UNDERLINE + """is""",
											        SPIECE_UNDERLINE + """f""",
											        """al""",
											        """s""",
											        """[UNK]""",
											        """.""",
											    ]					, )





							@cached_property
							def   UpperCamelCase_    (      self						:						str		):

											'''simple docstring'''
											return XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased"""		)





							@slow
							def   UpperCamelCase_    (      self						:						Tuple		):

											'''simple docstring'''
											__a        =						"""Hello World!"""
											__a        =						[35389, 6672, 49, 2]
											self.assertListEqual(__lowercase					, self.big_tokenizer.encode(__lowercase		)		)







							@slow
							def   UpperCamelCase_    (      self						:						List[Any]		):

											'''simple docstring'''
											# fmt: off
											__a        =						{"""input_ids""": [[11073, 82783, 18, 26, 82783, 549, 51540, 248, 17209, 1301, 217, 20, 215186, 1325, 147, 17209, 1301, 217, 20, 56370, 53, 122020, 20, 16477, 27, 87355, 4548, 20, 4728, 78392, 17, 159969, 18, 26, 24491, 629, 15, 538, 22704, 5439, 15, 2788, 24491, 9885, 15, 43534, 605, 15, 814, 18403, 33200, 29, 15, 43534, 24458, 12410, 111, 24966, 83669, 9637, 144068, 26, 850, 22346, 27, 147, 24966, 83669, 83490, 26, 39113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 122020, 115785, 34, 816, 1339, 46887, 18, 147, 53905, 1951, 42238, 41170, 17732, 834, 436, 15, 27523, 98733, 217, 147, 5542, 4981, 930, 17347, 16, 2], [20091, 629, 94, 82786, 58, 490, 20, 1528, 84, 53905, 344, 80592, 110128, 18822, 5267, 1306, 62, 152537, 308, 7997, 401, 124427, 549, 35442, 225, 109, 15055, 25748, 147, 7119, 43712, 34, 767, 135366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 63784, 119466, 17, 147808, 88214, 18, 656, 81, 32, 3296, 10280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__lowercase					, model_name="""microsoft/xprophetnet-large-wiki100-cased"""					, revision="""1acad1643ddd54a44df6a1b797ada8373685d90e"""					, )





'''simple docstring'''




from typing import List, Union

import numpy as np

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, logging
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline


lowerCamelCase							:Tuple				    =					logging.get_logger(__name__)







class 	_lowerCAmelCase							(			__UpperCAmelCase ):


	def     _a	(self       ,      lowercase							):
				if isinstance(lowercase       ,      lowercase							):
							A_	:						Tuple	=     [label.strip() for label in labels.split(""","""							) if label.strip()]
				return labels


	def __call__(self       ,      lowercase       ,      lowercase       ,      lowercase							):
				if len(lowercase							) == 0 or len(lowercase							) == 0:
							raise ValueError("""You must include at least one label and at least one sequence."""							)
				if hypothesis_template.format(labels[0]							) == hypothesis_template:
							raise ValueError(
							    (
							        """The provided hypothesis_template \"{}\" was not able to be formatted with the target labels. """
							        """Make sure the passed template includes formatting syntax such as {{}} where the label should go."""
							    ).format(lowercase							)							)

				if isinstance(lowercase       ,      lowercase							):
							A_	:						Optional[Any]	=     [sequences]

				A_	:						List[Any]	=     []
				for sequence in sequences:
							sequence_pairs.extend([[sequence, hypothesis_template.format(lowercase							)] for label in labels]							)

				return sequence_pairs, sequences







@add_end_docstrings(__UpperCAmelCase )
class 	_lowerCAmelCase							(			__UpperCAmelCase ):


	def __init__(self       ,      lowercase=ZeroShotClassificationArgumentHandler()       ,      *lowercase       ,      **lowercase							):
				A_	:						str	=     args_parser
				super().__init__(*lowercase       ,      **lowercase							)
				if self.entailment_id == -1:
							logger.warning(
							    """Failed to determine 'entailment' label id from the label2id mapping in the model config. Setting to """
							    """-1. Define a descriptive label2id mapping in the model config to ensure correct outputs."""							)


	@property
	def     _a	(self							):
				for label, ind in self.model.config.labelaid.items():
							if label.lower().startswith("""entail"""							):
										return ind
				return -1


	def     _a	(self       ,      lowercase       ,      lowercase=True       ,      lowercase=True       ,      lowercase=TruncationStrategy.ONLY_FIRST       ,      **lowercase							):
				A_	:						Optional[int]	=     self.framework
				if self.tokenizer.pad_token is None:
							# Override for tokenizers not supporting padding
							logger.error(
							    """Tokenizer was not supporting padding necessary for zero-shot, attempting to use """
							    """ `pad_token=eos_token`"""							)
							A_	:						Any	=     self.tokenizer.eos_token
				try:
							A_	:						Optional[Any]	=     self.tokenizer(
							    lowercase       ,      add_special_tokens=lowercase       ,      return_tensors=lowercase       ,      padding=lowercase       ,      truncation=lowercase       ,      )
				except Exception as e:
							if "too short" in str(lowercase							):
										# tokenizers might yell that we want to truncate
										# to a value that is not even reached by the input.
										# In that case we don't want to truncate.
										# It seems there's not a really better way to catch that
										# exception.

										A_	:						str	=     self.tokenizer(
										    lowercase       ,      add_special_tokens=lowercase       ,      return_tensors=lowercase       ,      padding=lowercase       ,      truncation=TruncationStrategy.DO_NOT_TRUNCATE       ,      )
							else:
										raise e

				return inputs


	def     _a	(self       ,      **lowercase							):
				if kwargs.get("""multi_class"""       ,      lowercase							) is not None:
							A_	:						Union[str, Any]	=     kwargs["""multi_class"""]
							logger.warning(
							    """The `multi_class` argument has been deprecated and renamed to `multi_label`. """
							    """`multi_class` will be removed in a future version of Transformers."""							)
				A_	:						Tuple	=     {}
				if "candidate_labels" in kwargs:
							A_	:						Optional[int]	=     self._args_parser._parse_labels(kwargs["""candidate_labels"""]							)
				if "hypothesis_template" in kwargs:
							A_	:						List[str]	=     kwargs["""hypothesis_template"""]

				A_	:						Union[str, Any]	=     {}
				if "multi_label" in kwargs:
							A_	:						Optional[int]	=     kwargs["""multi_label"""]
				return preprocess_params, {}, postprocess_params


	def __call__(self       ,      lowercase       ,      *lowercase       ,      **lowercase       ,      ):
				if len(lowercase							) == 0:
							pass
				elif len(lowercase							) == 1 and "candidate_labels" not in kwargs:
							A_	:						Any	=     args[0]
				else:
							raise ValueError(F'Unable to understand extra arguments {args}'							)

				return super().__call__(lowercase       ,      **lowercase							)


	def     _a	(self       ,      lowercase       ,      lowercase=None       ,      lowercase="This example is {}."							):
				A_,    A_	:						Any	=     self._args_parser(lowercase       ,      lowercase       ,      lowercase							)

				for i, (candidate_label, sequence_pair) in enumerate(zip(lowercase       ,      lowercase							)							):
							A_	:						List[Any]	=     self._parse_and_tokenize([sequence_pair]							)

							yield {
							    "candidate_label": candidate_label,
							    "sequence": sequences[0],
							    "is_last": i == len(lowercase							) - 1,
							    **model_input,
							}


	def     _a	(self       ,      lowercase							):
				A_	:						Dict	=     inputs["""candidate_label"""]
				A_	:						Any	=     inputs["""sequence"""]
				A_	:						List[Any]	=     {k: inputs[k] for k in self.tokenizer.model_input_names}
				A_	:						Optional[int]	=     self.model(**lowercase							)

				A_	:						Optional[Any]	=     {
				    """candidate_label""": candidate_label,
				    """sequence""": sequence,
				    """is_last""": inputs["""is_last"""],
				    **outputs,
				}
				return model_outputs


	def     _a	(self       ,      lowercase       ,      lowercase=False							):
				A_	:						Tuple	=     [outputs["""candidate_label"""] for outputs in model_outputs]
				A_	:						Optional[int]	=     [outputs["""sequence"""] for outputs in model_outputs]
				A_	:						Union[str, Any]	=     np.concatenate([output["""logits"""].numpy() for output in model_outputs]							)
				A_	:						List[str]	=     logits.shape[0]
				A_	:						Optional[int]	=     len(lowercase							)
				A_	:						int	=     N // n
				A_	:						int	=     logits.reshape((num_sequences, n, -1)							)

				if multi_label or len(lowercase							) == 1:
							# softmax over the entailment vs. contradiction dim for each label independently
							A_	:						Dict	=     self.entailment_id
							A_	:						Dict	=     -1 if entailment_id == 0 else 0
							A_	:						str	=     reshaped_outputs[..., [contradiction_id, entailment_id]]
							A_	:						Optional[Any]	=     np.exp(lowercase							) / np.exp(lowercase							).sum(-1       ,      keepdims=lowercase							)
							A_	:						Tuple	=     scores[..., 1]
				else:
							# softmax the "entailment" logits over all candidate labels
							A_	:						Optional[int]	=     reshaped_outputs[..., self.entailment_id]
							A_	:						Any	=     np.exp(lowercase							) / np.exp(lowercase							).sum(-1       ,      keepdims=lowercase							)

				A_	:						List[str]	=     list(reversed(scores[0].argsort()							)							)
				return {
				    "sequence": sequences[0],
				    "labels": [candidate_labels[i] for i in top_inds],
				    "scores": scores[0, top_inds].tolist(),
				}