



'''simple docstring'''




import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCamelCase							:Optional[Any]				    =					logging.get_logger(__name__)

lowerCamelCase							:int				    =					'''▁'''

lowerCamelCase							:Dict				    =					{'''vocab_file''': '''sentencepiece.bpe.model'''}

lowerCamelCase							:Tuple				    =					{
    '''vocab_file''': {
        '''xlm-roberta-base''': '''https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large''': '''https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model''',
        '''xlm-roberta-large-finetuned-conll02-dutch''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll02-spanish''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-english''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model'''
        ),
        '''xlm-roberta-large-finetuned-conll03-german''': (
            '''https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model'''
        ),
    }
}

lowerCamelCase							:Dict				    =					{
    '''xlm-roberta-base''': 5_1_2,
    '''xlm-roberta-large''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-dutch''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll02-spanish''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-english''': 5_1_2,
    '''xlm-roberta-large-finetuned-conll03-german''': 5_1_2,
}







class 	_lowerCAmelCase							(			__UpperCAmelCase ):
	__SCREAMING_SNAKE_CASE   :		Any					=					VOCAB_FILES_NAMES
	__SCREAMING_SNAKE_CASE   :		str					=					PRETRAINED_VOCAB_FILES_MAP
	__SCREAMING_SNAKE_CASE   :		List[Any]					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__SCREAMING_SNAKE_CASE   :		List[Any]					=					['input_ids', 'attention_mask']


	def __init__(self       ,      lowercase       ,      lowercase="<s>"       ,      lowercase="</s>"       ,      lowercase="</s>"       ,      lowercase="<s>"       ,      lowercase="<unk>"       ,      lowercase="<pad>"       ,      lowercase="<mask>"       ,      lowercase = None       ,      **lowercase       ,      ):
				# Mask token behave like a normal word, i.e. include the space before it
				A_	:						Tuple	=     AddedToken(lowercase       ,      lstrip=lowercase       ,      rstrip=lowercase							) if isinstance(lowercase       ,      lowercase							) else mask_token

				A_	:						int	=     {} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=lowercase       ,      eos_token=lowercase       ,      unk_token=lowercase       ,      sep_token=lowercase       ,      cls_token=lowercase       ,      pad_token=lowercase       ,      mask_token=lowercase       ,      sp_model_kwargs=self.sp_model_kwargs       ,      **lowercase       ,      )

				A_	:						Optional[Any]	=     spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.Load(str(lowercase							)							)
				A_	:						Dict	=     vocab_file

				# Original fairseq vocab and spm vocab must be "aligned":
				# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
				# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
				# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
				# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

				# Mimic fairseq token-to-id alignment for the first 4 token
				A_	:						Optional[int]	=     {"""<s>""": 0, """<pad>""": 1, """</s>""": 2, """<unk>""": 3}

				# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
				A_	:						Tuple	=     1

				A_	:						Optional[Any]	=     len(self.sp_model							) + self.fairseq_offset
				A_	:						Any	=     {v: k for k, v in self.fairseq_tokens_to_ids.items()}


	def __getstate__(self							):
				A_	:						List[Any]	=     self.__dict__.copy()
				A_	:						str	=     None
				A_	:						Any	=     self.sp_model.serialized_model_proto()
				return state


	def __setstate__(self       ,      lowercase							):
				A_	:						Optional[Any]	=     d

				# for backward compatibility
				if not hasattr(self       ,      """sp_model_kwargs"""							):
							A_	:						str	=     {}

				A_	:						Union[str, Any]	=     spm.SentencePieceProcessor(**self.sp_model_kwargs							)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto							)


	def     _a	(self       ,      lowercase       ,      lowercase = None							):

				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				A_	:						Dict	=     [self.cls_token_id]
				A_	:						Any	=     [self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep


	def     _a	(self       ,      lowercase       ,      lowercase = None       ,      lowercase = False							):

				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowercase       ,      token_ids_a=lowercase       ,      already_has_special_tokens=lowercase							)

				if token_ids_a is None:
							return [1] + ([0] * len(lowercase							)) + [1]
				return [1] + ([0] * len(lowercase							)) + [1, 1] + ([0] * len(lowercase							)) + [1]


	def     _a	(self       ,      lowercase       ,      lowercase = None							):
				A_	:						Tuple	=     [self.sep_token_id]
				A_	:						Optional[int]	=     [self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep							) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]


	@property
	def     _a	(self							):
				return len(self.sp_model							) + self.fairseq_offset + 1  # Add the <mask> token


	def     _a	(self							):
				A_	:						Tuple	=     {self.convert_ids_to_tokens(lowercase							): i for i in range(self.vocab_size							)}
				vocab.update(self.added_tokens_encoder							)
				return vocab


	def     _a	(self       ,      lowercase							):
				return self.sp_model.encode(lowercase       ,      out_type=lowercase							)


	def     _a	(self       ,      lowercase							):
				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				A_	:						str	=     self.sp_model.PieceToId(lowercase							)

				# Need to return unknown token if the SP model returned 0
				return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


	def     _a	(self       ,      lowercase							):
				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset							)


	def     _a	(self       ,      lowercase							):
				A_	:						int	=     """""".join(lowercase							).replace(lowercase       ,      """ """							).strip()
				return out_string


	def     _a	(self       ,      lowercase       ,      lowercase = None							):
				if not os.path.isdir(lowercase							):
							logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
							return
				A_	:						Optional[int]	=     os.path.join(
				    lowercase       ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

				if os.path.abspath(self.vocab_file							) != os.path.abspath(lowercase							) and os.path.isfile(self.vocab_file							):
							copyfile(self.vocab_file       ,      lowercase							)
				elif not os.path.isfile(self.vocab_file							):
							with open(lowercase       ,      """wb"""							) as fi:
										A_	:						Tuple	=     self.sp_model.serialized_model_proto()
										fi.write(lowercase							)

				return (out_vocab_file,)




'''simple docstring'''




from typing import List, Union

import numpy as np

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, logging
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, ChunkPipeline


lowerCamelCase							:Tuple				    =					logging.get_logger(__name__)







class 	_lowerCAmelCase							(			__UpperCAmelCase ):


	def     _a	(self       ,      lowercase							):
				if isinstance(lowercase       ,      lowercase							):
							A_	:						Tuple	=     [label.strip() for label in labels.split(""","""							) if label.strip()]
				return labels


	def __call__(self       ,      lowercase       ,      lowercase       ,      lowercase							):
				if len(lowercase							) == 0 or len(lowercase							) == 0:
							raise ValueError("""You must include at least one label and at least one sequence."""							)
				if hypothesis_template.format(labels[0]							) == hypothesis_template:
							raise ValueError(
							    (
							        """The provided hypothesis_template \"{}\" was not able to be formatted with the target labels. """
							        """Make sure the passed template includes formatting syntax such as {{}} where the label should go."""
							    ).format(lowercase							)							)

				if isinstance(lowercase       ,      lowercase							):
							A_	:						Optional[Any]	=     [sequences]

				A_	:						List[Any]	=     []
				for sequence in sequences:
							sequence_pairs.extend([[sequence, hypothesis_template.format(lowercase							)] for label in labels]							)

				return sequence_pairs, sequences







@add_end_docstrings(__UpperCAmelCase )
class 	_lowerCAmelCase							(			__UpperCAmelCase ):


	def __init__(self       ,      lowercase=ZeroShotClassificationArgumentHandler()       ,      *lowercase       ,      **lowercase							):
				A_	:						str	=     args_parser
				super().__init__(*lowercase       ,      **lowercase							)
				if self.entailment_id == -1:
							logger.warning(
							    """Failed to determine 'entailment' label id from the label2id mapping in the model config. Setting to """
							    """-1. Define a descriptive label2id mapping in the model config to ensure correct outputs."""							)


	@property
	def     _a	(self							):
				for label, ind in self.model.config.labelaid.items():
							if label.lower().startswith("""entail"""							):
										return ind
				return -1


	def     _a	(self       ,      lowercase       ,      lowercase=True       ,      lowercase=True       ,      lowercase=TruncationStrategy.ONLY_FIRST       ,      **lowercase							):
				A_	:						Optional[int]	=     self.framework
				if self.tokenizer.pad_token is None:
							# Override for tokenizers not supporting padding
							logger.error(
							    """Tokenizer was not supporting padding necessary for zero-shot, attempting to use """
							    """ `pad_token=eos_token`"""							)
							A_	:						Any	=     self.tokenizer.eos_token
				try:
							A_	:						Optional[Any]	=     self.tokenizer(
							    lowercase       ,      add_special_tokens=lowercase       ,      return_tensors=lowercase       ,      padding=lowercase       ,      truncation=lowercase       ,      )
				except Exception as e:
							if "too short" in str(lowercase							):
										# tokenizers might yell that we want to truncate
										# to a value that is not even reached by the input.
										# In that case we don't want to truncate.
										# It seems there's not a really better way to catch that
										# exception.

										A_	:						str	=     self.tokenizer(
										    lowercase       ,      add_special_tokens=lowercase       ,      return_tensors=lowercase       ,      padding=lowercase       ,      truncation=TruncationStrategy.DO_NOT_TRUNCATE       ,      )
							else:
										raise e

				return inputs


	def     _a	(self       ,      **lowercase							):
				if kwargs.get("""multi_class"""       ,      lowercase							) is not None:
							A_	:						Union[str, Any]	=     kwargs["""multi_class"""]
							logger.warning(
							    """The `multi_class` argument has been deprecated and renamed to `multi_label`. """
							    """`multi_class` will be removed in a future version of Transformers."""							)
				A_	:						Tuple	=     {}
				if "candidate_labels" in kwargs:
							A_	:						Optional[int]	=     self._args_parser._parse_labels(kwargs["""candidate_labels"""]							)
				if "hypothesis_template" in kwargs:
							A_	:						List[str]	=     kwargs["""hypothesis_template"""]

				A_	:						Union[str, Any]	=     {}
				if "multi_label" in kwargs:
							A_	:						Optional[int]	=     kwargs["""multi_label"""]
				return preprocess_params, {}, postprocess_params


	def __call__(self       ,      lowercase       ,      *lowercase       ,      **lowercase       ,      ):
				if len(lowercase							) == 0:
							pass
				elif len(lowercase							) == 1 and "candidate_labels" not in kwargs:
							A_	:						Any	=     args[0]
				else:
							raise ValueError(F'Unable to understand extra arguments {args}'							)

				return super().__call__(lowercase       ,      **lowercase							)


	def     _a	(self       ,      lowercase       ,      lowercase=None       ,      lowercase="This example is {}."							):
				A_,    A_	:						Any	=     self._args_parser(lowercase       ,      lowercase       ,      lowercase							)

				for i, (candidate_label, sequence_pair) in enumerate(zip(lowercase       ,      lowercase							)							):
							A_	:						List[Any]	=     self._parse_and_tokenize([sequence_pair]							)

							yield {
							    "candidate_label": candidate_label,
							    "sequence": sequences[0],
							    "is_last": i == len(lowercase							) - 1,
							    **model_input,
							}


	def     _a	(self       ,      lowercase							):
				A_	:						Dict	=     inputs["""candidate_label"""]
				A_	:						Any	=     inputs["""sequence"""]
				A_	:						List[Any]	=     {k: inputs[k] for k in self.tokenizer.model_input_names}
				A_	:						Optional[int]	=     self.model(**lowercase							)

				A_	:						Optional[Any]	=     {
				    """candidate_label""": candidate_label,
				    """sequence""": sequence,
				    """is_last""": inputs["""is_last"""],
				    **outputs,
				}
				return model_outputs


	def     _a	(self       ,      lowercase       ,      lowercase=False							):
				A_	:						Tuple	=     [outputs["""candidate_label"""] for outputs in model_outputs]
				A_	:						Optional[int]	=     [outputs["""sequence"""] for outputs in model_outputs]
				A_	:						Union[str, Any]	=     np.concatenate([output["""logits"""].numpy() for output in model_outputs]							)
				A_	:						List[str]	=     logits.shape[0]
				A_	:						Optional[int]	=     len(lowercase							)
				A_	:						int	=     N // n
				A_	:						int	=     logits.reshape((num_sequences, n, -1)							)

				if multi_label or len(lowercase							) == 1:
							# softmax over the entailment vs. contradiction dim for each label independently
							A_	:						Dict	=     self.entailment_id
							A_	:						Dict	=     -1 if entailment_id == 0 else 0
							A_	:						str	=     reshaped_outputs[..., [contradiction_id, entailment_id]]
							A_	:						Optional[Any]	=     np.exp(lowercase							) / np.exp(lowercase							).sum(-1       ,      keepdims=lowercase							)
							A_	:						Tuple	=     scores[..., 1]
				else:
							# softmax the "entailment" logits over all candidate labels
							A_	:						Optional[int]	=     reshaped_outputs[..., self.entailment_id]
							A_	:						Any	=     np.exp(lowercase							) / np.exp(lowercase							).sum(-1       ,      keepdims=lowercase							)

				A_	:						List[str]	=     list(reversed(scores[0].argsort()							)							)
				return {
				    "sequence": sequences[0],
				    "labels": [candidate_labels[i] for i in top_inds],
				    "scores": scores[0, top_inds].tolist(),
				}