





import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TextaTextGenerationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, require_tf, require_torch
from transformers.utils import is_torch_available

from .test_pipelines_common import ANY


if is_torch_available():
     import torch



@is_pipeline_test
class 				_UpperCAmelCase    (    unittest.TestCase    ):

  """simple docstring"""


  a_        = MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  a_        = TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING






  def 				lowercase		(				self :		str					,  lowerCAmelCase_ :		List[str]					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		List[str]     )							->						List[str]:
     __lowerCAmelCase								=	TextaTextGenerationPipeline(model=__lowerCamelCase					,  tokenizer=__lowerCamelCase     )
     return generator, ["Something to write", "Something else"]






  def 				lowercase		(				self :		Tuple					,  lowerCAmelCase_ :		Optional[Any]					,  lowerCAmelCase_ :		int     )							->						str:
     __lowerCAmelCase								=	generator('Something there'     )
     self.assertEqual(__lowerCamelCase					,  [{'generated_text': ANY(__lowerCamelCase     )}]     )
     # These are encoder decoder, they don't just append to incoming string
     self.assertFalse(outputs[0]['generated_text'].startswith('Something there'     )     )

     __lowerCAmelCase								=	generator(['This is great !', 'Something else']					,  num_return_sequences=2					,  do_sample=__lowerCamelCase     )
     self.assertEqual(
         __lowerCamelCase					,  [
             [{'generated_text': ANY(__lowerCamelCase     )}, {'generated_text': ANY(__lowerCamelCase     )}],
             [{'generated_text': ANY(__lowerCamelCase     )}, {'generated_text': ANY(__lowerCamelCase     )}],
         ]					,  )

     __lowerCAmelCase								=	generator(
         ['This is great !', 'Something else']					,  num_return_sequences=2					,  batch_size=2					,  do_sample=__lowerCamelCase     )
     self.assertEqual(
         __lowerCamelCase					,  [
             [{'generated_text': ANY(__lowerCamelCase     )}, {'generated_text': ANY(__lowerCamelCase     )}],
             [{'generated_text': ANY(__lowerCamelCase     )}, {'generated_text': ANY(__lowerCamelCase     )}],
         ]					,  )

     with self.assertRaises(__lowerCamelCase     ):
        generator(4     )






  @require_torch
  def 				lowercase		(				self :		Union[str, Any]     )							->						Tuple:
     __lowerCAmelCase								=	pipeline('text2text-generation'					,  model='patrickvonplaten/t5-tiny-random'					,  framework='pt'     )
     # do_sample=False necessary for reproducibility
     __lowerCAmelCase								=	generator('Something there'					,  do_sample=__lowerCamelCase     )
     self.assertEqual(__lowerCamelCase					,  [{'generated_text': ''}]     )

     __lowerCAmelCase								=	3
     __lowerCAmelCase								=	generator(
         'Something there'					,  num_return_sequences=__lowerCamelCase					,  num_beams=__lowerCamelCase					,  )
     __lowerCAmelCase								=	[
         {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide Beide'},
         {'generated_text': 'Beide Beide Beide Beide Beide Beide Beide Beide'},
         {'generated_text': ''},
     ]
     self.assertEqual(__lowerCamelCase					,  __lowerCamelCase     )

     __lowerCAmelCase								=	generator('This is a test'					,  do_sample=__lowerCamelCase					,  num_return_sequences=2					,  return_tensors=__lowerCamelCase     )
     self.assertEqual(
         __lowerCamelCase					,  [
             {'generated_token_ids': ANY(torch.Tensor     )},
             {'generated_token_ids': ANY(torch.Tensor     )},
         ]					,  )
     __lowerCAmelCase								=	generator.model.config.eos_token_id
     __lowerCAmelCase								=	'<pad>'
     __lowerCAmelCase								=	generator(
         ['This is a test', 'This is a second test']					,  do_sample=__lowerCamelCase					,  num_return_sequences=2					,  batch_size=2					,  return_tensors=__lowerCamelCase					,  )
     self.assertEqual(
         __lowerCamelCase					,  [
             [
                 {'generated_token_ids': ANY(torch.Tensor     )},
                 {'generated_token_ids': ANY(torch.Tensor     )},
             ],
             [
                 {'generated_token_ids': ANY(torch.Tensor     )},
                 {'generated_token_ids': ANY(torch.Tensor     )},
             ],
         ]					,  )






  @require_tf
  def 				lowercase		(				self :		Optional[Any]     )							->						List[Any]:
     __lowerCAmelCase								=	pipeline('text2text-generation'					,  model='patrickvonplaten/t5-tiny-random'					,  framework='tf'     )
     # do_sample=False necessary for reproducibility
     __lowerCAmelCase								=	generator('Something there'					,  do_sample=__lowerCamelCase     )
     self.assertEqual(__lowerCamelCase					,  [{'generated_text': ''}]     )








import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


a__:      Dict									=		logging.get_logger(__name__)

a__:      str									=		{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

a__:      Any									=		{
    'vocab_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt'
        ),
        'distilbert-base-german-cased': 'https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt',
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'distilbert-base-uncased': 'https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json',
        'distilbert-base-uncased-distilled-squad': (
            'https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-cased': 'https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json',
        'distilbert-base-cased-distilled-squad': (
            'https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json'
        ),
        'distilbert-base-german-cased': (
            'https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json'
        ),
        'distilbert-base-multilingual-cased': (
            'https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json'
        ),
    },
}

a__:      Any									=		{
    'distilbert-base-uncased': 512,
    'distilbert-base-uncased-distilled-squad': 512,
    'distilbert-base-cased': 512,
    'distilbert-base-cased-distilled-squad': 512,
    'distilbert-base-german-cased': 512,
    'distilbert-base-multilingual-cased': 512,
}


a__:      Optional[Any]									=		{
    'distilbert-base-uncased': {'do_lower_case': True},
    'distilbert-base-uncased-distilled-squad': {'do_lower_case': True},
    'distilbert-base-cased': {'do_lower_case': False},
    'distilbert-base-cased-distilled-squad': {'do_lower_case': False},
    'distilbert-base-german-cased': {'do_lower_case': False},
    'distilbert-base-multilingual-cased': {'do_lower_case': False},
}




class     SCREAMING_SNAKE_CASE__ (						UpperCamelCase__      ):
 __SCREAMING_SNAKE_CASE							       =	VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_INIT_CONFIGURATION
 __SCREAMING_SNAKE_CASE							       =	['''input_ids''', '''attention_mask''']
 __SCREAMING_SNAKE_CASE							       =	DistilBertTokenizer



 def __init__(							self,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=True,__lowerCamelCase="[UNK]",__lowerCamelCase="[SEP]",__lowerCamelCase="[PAD]",__lowerCamelCase="[CLS]",__lowerCamelCase="[MASK]",__lowerCamelCase=True,__lowerCamelCase=None,**__lowerCamelCase,):
        super().__init__(
            __lowerCamelCase,tokenizer_file=__lowerCamelCase,do_lower_case=__lowerCamelCase,unk_token=__lowerCamelCase,sep_token=__lowerCamelCase,pad_token=__lowerCamelCase,cls_token=__lowerCamelCase,mask_token=__lowerCamelCase,tokenize_chinese_chars=__lowerCamelCase,strip_accents=__lowerCamelCase,**__lowerCamelCase,)

        A__											=						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            normalizer_state.get('''lowercase''',__lowerCamelCase ) != do_lower_case
            or normalizer_state.get('''strip_accents''',__lowerCamelCase ) != strip_accents
            or normalizer_state.get('''handle_chinese_chars''',__lowerCamelCase ) != tokenize_chinese_chars
        ):
               A__											=						getattr(__lowerCamelCase,normalizer_state.pop('''type''' ) )
               A__											=						do_lower_case
               A__											=						strip_accents
               A__											=						tokenize_chinese_chars
               A__											=						normalizer_class(**__lowerCamelCase )

        A__											=						do_lower_case



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase=None ):
        A__											=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

        return output



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase = None ):
        A__											=						[self.sep_token_id]
        A__											=						[self.cls_token_id]
        if token_ids_a is None:
               return len(cls + token_ids_a + sep ) * [0]
        return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase = None ):
        A__											=						self._tokenizer.model.save(__lowerCamelCase,name=__lowerCamelCase )
        return tuple(__lowerCamelCase )





