"""simple docstring"""







import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin







class        UpperCamelCase		(			lowercase_						,      unittest.TestCase  ):
   lowercase							=  DebertaTokenizer
   lowercase							=  True
   lowercase							=  DebertaTokenizerFast
   def 			_UpperCAmelCase   (			self			)	->      Tuple:


        '''simple docstring'''


        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        lowercase_	:				str													=	[
            'l',
            'o',
            'w',
            'e',
            'r',
            's',
            't',
            'i',
            'd',
            'n',
            '\u0120',
            '\u0120l',
            '\u0120n',
            '\u0120lo',
            '\u0120low',
            'er',
            '\u0120lowest',
            '\u0120newer',
            '\u0120wider',
            '[UNK]',
        ]
        lowercase_	:				str													=	dict(zip(__UpperCamelCase     ,range(len(__UpperCamelCase			)			)			)			)
        lowercase_	:				List[Any]													=	['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
        lowercase_	:				Dict													=	{'unk_token': '[UNK]'}

        lowercase_	:				int													=	os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['vocab_file']			)
        lowercase_	:				List[Any]													=	os.path.join(self.tmpdirname     ,VOCAB_FILES_NAMES['merges_file']			)
        with open(self.vocab_file     ,'w'     ,encoding='utf-8'			) as fp:
             fp.write(json.dumps(__UpperCamelCase			) + '\n'			)
        with open(self.merges_file     ,'w'     ,encoding='utf-8'			) as fp:
             fp.write('\n'.join(__UpperCamelCase			)			)
   def 			_UpperCAmelCase   (			self     ,**__UpperCamelCase			)	->      Any:


        '''simple docstring'''


        kwargs.update(self.special_tokens_map			)
        return self.tokenizer_class.from_pretrained(self.tmpdirname     ,**__UpperCamelCase			)
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase			)	->      List[str]:


        '''simple docstring'''


        lowercase_	:				Any													=	'lower newer'
        lowercase_	:				Dict													=	'lower newer'
        return input_text, output_text
   def 			_UpperCAmelCase   (			self			)	->      int:


        '''simple docstring'''


        lowercase_	:				Union[str, Any]													=	self.get_tokenizer()
        lowercase_	:				Dict													=	'lower newer'
        lowercase_	:				Tuple													=	['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
        lowercase_	:				Optional[Any]													=	tokenizer.tokenize(__UpperCamelCase			)
        self.assertListEqual(__UpperCamelCase     ,__UpperCamelCase			)

        lowercase_	:				Optional[Any]													=	tokens + [tokenizer.unk_token]
        lowercase_	:				Dict													=	[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCamelCase			)     ,__UpperCamelCase			)
   def 			_UpperCAmelCase   (			self			)	->      List[str]:


        '''simple docstring'''


        lowercase_	:				List[Any]													=	self.get_tokenizer()
        lowercase_	:				Optional[int]													=	tokenizer('Hello'     ,'World'			)
        lowercase_	:				Tuple													=	[0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
        self.assertListEqual(tokd['token_type_ids']     ,__UpperCamelCase			)
   @slow
   def 			_UpperCAmelCase   (			self			)	->      Any:


        '''simple docstring'''


        lowercase_	:				Optional[int]													=	self.tokenizer_class.from_pretrained('microsoft/deberta-base'			)

        lowercase_	:				int													=	tokenizer.encode('sequence builders'     ,add_special_tokens=__UpperCamelCase			)
        lowercase_	:				Dict													=	tokenizer.encode('multi-sequence build'     ,add_special_tokens=__UpperCamelCase			)

        lowercase_	:				str													=	tokenizer.encode(
            'sequence builders'     ,add_special_tokens=__UpperCamelCase     ,add_prefix_space=__UpperCamelCase			)
        lowercase_	:				Optional[Any]													=	tokenizer.encode(
            'sequence builders'     ,'multi-sequence build'     ,add_special_tokens=__UpperCamelCase     ,add_prefix_space=__UpperCamelCase			)

        lowercase_	:				Optional[int]													=	tokenizer.build_inputs_with_special_tokens(__UpperCamelCase			)
        lowercase_	:				List[str]													=	tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     ,__UpperCamelCase			)

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode







   @slow
   def 			_UpperCAmelCase   (			self			)	->      Optional[Any]:


        '''simple docstring'''


        lowercase_	:				Any													=	[self.tokenizer_class]
        if self.test_rust_tokenizer:
             tokenizer_classes.append(self.rust_tokenizer_class			)

        for tokenizer_class in tokenizer_classes:
             lowercase_	:				Optional[int]													=	tokenizer_class.from_pretrained('microsoft/deberta-base'			)

             lowercase_	:				List[Any]													=	[
                 'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                 'ALBERT incorporates two parameter reduction techniques',
                 'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                 ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                 ' vocabulary embedding.',
             ]

             lowercase_	:				Optional[Any]													=	tokenizer(__UpperCamelCase     ,padding=__UpperCamelCase			)
             lowercase_	:				List[str]													=	[tokenizer.decode(__UpperCamelCase     ,skip_special_tokens=__UpperCamelCase			) for seq in encoding['input_ids']]

             # fmt: off
             lowercase_	:				Optional[int]													=	{
                 'input_ids': [
                     [1, 2118, 1_1126, 565, 35, 83, 2_5191, 163, 1_8854, 13, 1_2156, 12, 1_6101, 2_5376, 1_3807, 9, 2_2205, 2_7893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 2118, 1_1126, 565, 2_4536, 80, 4_3797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 133, 78, 65, 16, 10, 3724, 1538, 3_3183, 1_1303, 4_3797, 1938, 4, 870, 2_4165, 2_9105, 5, 739, 3_2644, 3_3183, 1_1303, 3_6173, 88, 80, 650, 7821, 4_5940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 1_3171, 31, 5, 1836, 9, 3_2644, 3_3183, 1_1303, 4, 2]
                 ],
                 'token_type_ids': [
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                 ],
                 'attention_mask': [
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                 ]
             }
             # fmt: on

             lowercase_	:				Union[str, Any]													=	[
                 'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                 'ALBERT incorporates two parameter reduction techniques',
                 'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                 ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                 ' vocabulary embedding.',
             ]

             self.assertDictEqual(encoding.data     ,__UpperCamelCase			)

             for expected, decoded in zip(__UpperCamelCase     ,__UpperCamelCase			):
                  self.assertEqual(__UpperCamelCase     ,__UpperCamelCase			)


"""simple docstring"""







import enum
import warnings

from ..tokenization_utils import TruncationStrategy
from ..utils import add_end_docstrings, is_tf_available, is_torch_available, logging
from .base import PIPELINE_INIT_ARGS, Pipeline


if is_tf_available():
       import tensorflow as tf

       from ..models.auto.modeling_tf_auto import TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

if is_torch_available():
       from ..models.auto.modeling_auto import MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING

__SCREAMING_SNAKE_CASE    	=logging.get_logger(__name__)







class        UpperCamelCase		(			enum.Enum  ):
   lowercase							=  0
   lowercase							=  1







@add_end_docstrings(lowercase_  )
class        UpperCamelCase		(			lowercase_  ):
   lowercase							=  'generated'
   def __init__(			self     ,*__UpperCamelCase     ,**__UpperCamelCase			)	->      Tuple:


        '''simple docstring'''


        super().__init__(*__UpperCamelCase     ,**__UpperCamelCase			)

        self.check_model_type(
            TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
            if self.framework == 'tf'
            else MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING			)
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase=None     ,__UpperCamelCase=None     ,__UpperCamelCase=None     ,__UpperCamelCase=None     ,__UpperCamelCase=None     ,__UpperCamelCase=None     ,**__UpperCamelCase     ,)	->      Optional[Any]:


        '''simple docstring'''


        lowercase_	:				List[Any]													=	{}
        if truncation is not None:
             lowercase_	:				int													=	truncation

        lowercase_	:				Dict													=	generate_kwargs

        lowercase_	:				List[Any]													=	{}
        if return_tensors is not None and return_type is None:
             lowercase_	:				Union[str, Any]													=	ReturnType.TENSORS if return_tensors else ReturnType.TEXT
        if return_type is not None:
             lowercase_	:				str													=	return_type

        if clean_up_tokenization_spaces is not None:
             lowercase_	:				Dict													=	clean_up_tokenization_spaces

        if stop_sequence is not None:
             lowercase_	:				Union[str, Any]													=	self.tokenizer.encode(__UpperCamelCase     ,add_special_tokens=__UpperCamelCase			)
             if len(__UpperCamelCase			) > 1:
                  warnings.warn(
                      'Stopping on a multiple token sequence is not yet supported on transformers. The first token of'
                      ' the stop sequence will be used as the stop sequence string in the interim.'			)
             lowercase_	:				Optional[int]													=	stop_sequence_ids[0]

        return preprocess_params, forward_params, postprocess_params
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase     ,__UpperCamelCase     ,__UpperCamelCase			)	->      Optional[int]:


        '''simple docstring'''


        return True
   def 			_UpperCAmelCase   (			self     ,*__UpperCamelCase     ,__UpperCamelCase			)	->      Dict:


        '''simple docstring'''


        lowercase_	:				Dict													=	self.model.config.prefix if self.model.config.prefix is not None else ''
        if isinstance(args[0]     ,__UpperCamelCase			):
             if self.tokenizer.pad_token_id is None:
                  raise ValueError('Please make sure that the tokenizer has a pad_token_id when using a batch input'			)
             lowercase_	:				str													=	([prefix + arg for arg in args[0]],)
             lowercase_	:				Union[str, Any]													=	True

        elif isinstance(args[0]     ,__UpperCamelCase			):
             lowercase_	:				Union[str, Any]													=	(prefix + args[0],)
             lowercase_	:				Union[str, Any]													=	False
        else:
             raise ValueError(
                 f''' `args[0]`: {args[0]} have the wrong format. The should be either of type `str` or type `list`'''			)
        lowercase_	:				List[Any]													=	self.tokenizer(*__UpperCamelCase     ,padding=__UpperCamelCase     ,truncation=__UpperCamelCase     ,return_tensors=self.framework			)
        # This is produced by tokenizers but is an invalid generate kwargs
        if "token_type_ids" in inputs:
             del inputs["token_type_ids"]
        return inputs
   def __call__(			self     ,*__UpperCamelCase     ,**__UpperCamelCase			)	->      Dict:


        '''simple docstring'''


        lowercase_	:				Optional[int]													=	super().__call__(*__UpperCamelCase     ,**__UpperCamelCase			)
        if (
            isinstance(args[0]     ,__UpperCamelCase			)
            and all(isinstance(__UpperCamelCase     ,__UpperCamelCase			) for el in args[0]			)
            and all(len(__UpperCamelCase			) == 1 for res in result			)
        ):
             return [res[0] for res in result]
        return result
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase     ,__UpperCamelCase=TruncationStrategy.DO_NOT_TRUNCATE     ,**__UpperCamelCase			)	->      List[str]:


        '''simple docstring'''


        lowercase_	:				Any													=	self._parse_and_tokenize(__UpperCamelCase     ,truncation=__UpperCamelCase     ,**__UpperCamelCase			)
        return inputs
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase     ,**__UpperCamelCase			)	->      Optional[int]:


        '''simple docstring'''


        if self.framework == "pt":
             lowercase_							,  lowercase_	:				Optional[int]													=	model_inputs['input_ids'].shape
        elif self.framework == "tf":
             lowercase_							,  lowercase_	:				Union[str, Any]													=	tf.shape(model_inputs['input_ids']			).numpy()

        lowercase_	:				str													=	generate_kwargs.get('min_length'     ,self.model.config.min_length			)
        lowercase_	:				List[Any]													=	generate_kwargs.get('max_length'     ,self.model.config.max_length			)
        self.check_inputs(__UpperCamelCase     ,generate_kwargs['min_length']     ,generate_kwargs['max_length']			)
        lowercase_	:				Tuple													=	self.model.generate(**__UpperCamelCase     ,**__UpperCamelCase			)
        lowercase_	:				str													=	output_ids.shape[0]
        if self.framework == "pt":
             lowercase_	:				List[Any]													=	output_ids.reshape(__UpperCamelCase     ,out_b // in_b     ,*output_ids.shape[1:]			)
        elif self.framework == "tf":
             lowercase_	:				List[Any]													=	tf.reshape(__UpperCamelCase     ,(in_b, out_b // in_b, *output_ids.shape[1:])			)
        return {"output_ids": output_ids}







   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase     ,__UpperCamelCase=ReturnType.TEXT     ,__UpperCamelCase=False			)	->      Dict:


        '''simple docstring'''


        lowercase_	:				Dict													=	[]
        for output_ids in model_outputs["output_ids"][0]:
             if return_type == ReturnType.TENSORS:
                  lowercase_	:				List[Any]													=	{f'''{self.return_name}_token_ids''': output_ids}
             elif return_type == ReturnType.TEXT:
                  lowercase_	:				str													=	{
                      f'''{self.return_name}_text''': self.tokenizer.decode(
                          __UpperCamelCase     ,skip_special_tokens=__UpperCamelCase     ,clean_up_tokenization_spaces=__UpperCamelCase     ,)
                  }
             records.append(__UpperCamelCase			)
        return records







@add_end_docstrings(lowercase_  )
class        UpperCamelCase		(			lowercase_  ):
   lowercase							=  'summary'
   def __call__(			self     ,*__UpperCamelCase     ,**__UpperCamelCase			)	->      Union[str, Any]:


        '''simple docstring'''


        return super().__call__(*__UpperCamelCase     ,**__UpperCamelCase			)







   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase     ,__UpperCamelCase     ,__UpperCamelCase			)	->      bool:


        '''simple docstring'''


        if max_length < min_length:
             logger.warning(f'''Your min_length={min_length} must be inferior than your max_length={max_length}.'''			)

        if input_length < max_length:
             logger.warning(
                 f'''Your max_length is set to {max_length}, but your input_length is only {input_length}. Since this is '''
                 'a summarization task, where outputs shorter than the input are typically wanted, you might '
                 f'''consider decreasing max_length manually, e.g. summarizer(\'...\', max_length={input_length//2})'''			)







@add_end_docstrings(lowercase_  )
class        UpperCamelCase		(			lowercase_  ):
   lowercase							=  'translation'
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase     ,__UpperCamelCase     ,__UpperCamelCase			)	->      Dict:


        '''simple docstring'''


        if input_length > 0.9 * max_length:
             logger.warning(
                 f'''Your input_length: {input_length} is bigger than 0.9 * max_length: {max_length}. You might consider '''
                 'increasing your max_length manually, e.g. translator(\'...\', max_length=400)'			)
        return True
   def 			_UpperCAmelCase   (			self     ,*__UpperCamelCase     ,__UpperCamelCase=TruncationStrategy.DO_NOT_TRUNCATE     ,__UpperCamelCase=None     ,__UpperCamelCase=None			)	->      int:


        '''simple docstring'''


        if getattr(self.tokenizer     ,'_build_translation_inputs'     ,__UpperCamelCase			):
             return self.tokenizer._build_translation_inputs(
                 *__UpperCamelCase     ,return_tensors=self.framework     ,truncation=__UpperCamelCase     ,src_lang=__UpperCamelCase     ,tgt_lang=__UpperCamelCase			)
        else:
             return super()._parse_and_tokenize(*__UpperCamelCase     ,truncation=__UpperCamelCase			)
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase=None     ,__UpperCamelCase=None     ,**__UpperCamelCase			)	->      str:


        '''simple docstring'''


        lowercase_							,  lowercase_							,  lowercase_	:				int													=	super()._sanitize_parameters(**__UpperCamelCase			)
        if src_lang is not None:
             lowercase_	:				str													=	src_lang
        if tgt_lang is not None:
             lowercase_	:				Optional[Any]													=	tgt_lang
        if src_lang is None and tgt_lang is None:
             # Backward compatibility, direct arguments use is preferred.
             lowercase_	:				Tuple													=	kwargs.get('task'     ,self.task			)
             lowercase_	:				List[str]													=	task.split('_'			)
             if task and len(__UpperCamelCase			) == 4:
                  # translation, XX, to YY
                  lowercase_	:				Union[str, Any]													=	items[1]
                  lowercase_	:				Tuple													=	items[3]
        return preprocess_params, forward_params, postprocess_params







   def __call__(			self     ,*__UpperCamelCase     ,**__UpperCamelCase			)	->      Dict:


        '''simple docstring'''


        return super().__call__(*__UpperCamelCase     ,**__UpperCamelCase			)

