




import unittest

from transformers import (
    MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
    SummarizationPipeline,
    TFPreTrainedModel,
    pipeline,
)
from transformers.testing_utils import get_gpu_count, is_pipeline_test, require_tf, require_torch, slow, torch_device
from transformers.tokenization_utils import TruncationStrategy

from .test_pipelines_common import ANY


snake_case__ :						str     =			-1 if torch_device == '''cpu''' else 0





@is_pipeline_test
class     snake_case  (	unittest.TestCase							):


			'''simple docstring'''







			UpperCamelCase__	:       Optional[Any]     =      MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
			UpperCamelCase__	:       Optional[int]     =      TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING




			def 	UpperCAmelCase       (					self     :	Dict						,    lowerCamelCase_     :	Optional[int]						,    lowerCamelCase_     :	List[str]						,    lowerCamelCase_     :	Union[str, Any]			)					->Union[str, Any]:
					'''simple docstring'''







					UpperCAmelCase__    						=   SummarizationPipeline(model=lowerCamelCase_						,    tokenizer=lowerCamelCase_			)
					return summarizer, ["(CNN)The Palestinian Authority officially became", "Some other text"]




			def 	UpperCAmelCase       (					self     :	Dict						,    lowerCamelCase_     :	Tuple						,    lowerCamelCase_     :	str			)					->Tuple:
					'''simple docstring'''







					UpperCAmelCase__    						=   summarizer.model

					UpperCAmelCase__    						=   summarizer("""(CNN)The Palestinian Authority officially became"""			)
					self.assertEqual(lowerCamelCase_						,    [{"""summary_text""": ANY(lowerCamelCase_			)}]			)

					UpperCAmelCase__    						=   summarizer(
					    """(CNN)The Palestinian Authority officially became """						,    num_beams=2						,    min_length=2						,    max_length=5						,    )
					self.assertEqual(lowerCamelCase_						,    [{"""summary_text""": ANY(lowerCamelCase_			)}]			)

					# Some models (Switch Transformers, LED, T5, LongT5, etc) can handle long sequences.
					UpperCAmelCase__    						=   [
					    """SwitchTransformersConfig""",
					    """T5Config""",
					    """LongT5Config""",
					    """LEDConfig""",
					    """PegasusXConfig""",
					    """FSMTConfig""",
					    """M2M100Config""",
					    """ProphetNetConfig""",  # positional embeddings up to a fixed maximum size (otherwise clamping the values)
					]
					if model.config.__class__.__name__ not in model_can_handle_longer_seq:
							# Too long and exception is expected.
							# For TF models, if the weights are initialized in GPU context, we won't get expected index error from
							# the embedding layer.
							if not (
							    isinstance(lowerCamelCase_						,    lowerCamelCase_			)
							    and get_gpu_count() > 0
							    and len(summarizer.model.trainable_weights			) > 0
							):
									with self.assertRaises(lowerCamelCase_			):
											UpperCAmelCase__    						=   summarizer("""This """ * 1000			)
					UpperCAmelCase__    						=   summarizer("""This """ * 1000						,    truncation=TruncationStrategy.ONLY_FIRST			)




			@require_torch
			def 	UpperCAmelCase       (					self     :	Optional[Any]			)					->List[str]:
					'''simple docstring'''







					UpperCAmelCase__    						=   pipeline(task="""summarization"""						,    model="""sshleifer/tiny-mbart"""						,    framework="""pt"""			)
					UpperCAmelCase__    						=   summarizer("""This is a small test"""			)
					self.assertEqual(
					    lowerCamelCase_						,    [
					        {
					            """summary_text""": """เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป"""
					        }
					    ]						,    )




			@require_tf
			def 	UpperCAmelCase       (					self     :	str			)					->str:
					'''simple docstring'''







					UpperCAmelCase__    						=   pipeline(task="""summarization"""						,    model="""sshleifer/tiny-mbart"""						,    framework="""tf"""			)
					UpperCAmelCase__    						=   summarizer("""This is a small test"""			)
					self.assertEqual(
					    lowerCamelCase_						,    [
					        {
					            """summary_text""": """เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป"""
					        }
					    ]						,    )







			@require_torch
			@slow
			def 	UpperCAmelCase       (					self     :	Union[str, Any]			)					->Tuple:
					'''simple docstring'''







					UpperCAmelCase__    						=   pipeline(task="""summarization"""						,    device=lowerCamelCase_			)
					UpperCAmelCase__    						=   (
					    """ (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on"""
					    """ Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The"""
					    """ formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based."""
					    """ The Palestinians signed the ICC's founding Rome Statute in January, when they also accepted its"""
					    """ jurisdiction over alleged crimes committed \"in the occupied Palestinian territory, including East"""
					    """ Jerusalem, since June 13, 2014.\" Later that month, the ICC opened a preliminary examination into the"""
					    """ situation in Palestinian territories, paving the way for possible war crimes investigations against"""
					    """ Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and"""
					    """ the United States, neither of which is an ICC member, opposed the Palestinians' efforts to join the"""
					    """ body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday's ceremony, said it was a"""
					    """ move toward greater justice. \"As Palestine formally becomes a State Party to the Rome Statute today, the"""
					    """ world is also a step closer to ending a long era of impunity and injustice,\" he said, according to an"""
					    """ ICC news release. \"Indeed, today brings us closer to our shared goals of justice and peace.\" Judge"""
					    """ Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the"""
					    """ Palestinians. \"As the Rome Statute today enters into force for the State of Palestine, Palestine"""
					    """ acquires all the rights as well as responsibilities that come with being a State Party to the Statute."""
					    """ These are substantive commitments, which cannot be taken lightly,\" she said. Rights group Human Rights"""
					    """ Watch welcomed the development. \"Governments seeking to penalize Palestine for joining the ICC should"""
					    """ immediately end their pressure, and countries that support universal acceptance of the court's treaty"""
					    """ should speak out to welcome its membership,\" said Balkees Jarrah, international justice counsel for the"""
					    """ group. \"What's objectionable is the attempts to undermine international justice, not Palestine's"""
					    """ decision to join a treaty to which over 100 countries around the world are members.\" In January, when"""
					    """ the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an"""
					    """ outrage, saying the court was overstepping its boundaries. The United States also said it \"strongly\""""
					    """ disagreed with the court's decision. \"As we have said repeatedly, we do not believe that Palestine is a"""
					    """ state and therefore we do not believe that it is eligible to join the ICC,\" the State Department said in"""
					    """ a statement. It urged the warring sides to resolve their differences through direct negotiations. \"We"""
					    """ will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace,\""""
					    """ it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the"""
					    """ territories as \"Palestine.\" While a preliminary examination is not a formal investigation, it allows the"""
					    """ court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou"""
					    """ Bensouda said her office would \"conduct its analysis in full independence and impartiality.\" The war"""
					    """ between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry"""
					    """ will include alleged war crimes committed since June. The International Criminal Court was set up in"""
					    """ 2002 to prosecute genocide, crimes against humanity and war crimes. CNN's Vasco Cotovio, Kareem Khadder"""
					    """ and Faith Karimi contributed to this report."""
					)
					UpperCAmelCase__    						=   (
					    """ The Palestinian Authority becomes the 123rd member of the International Criminal Court . The move gives"""
					    """ the court jurisdiction over alleged crimes in Palestinian territories . Israel and the United States"""
					    """ opposed the Palestinians' efforts to join the court . Rights group Human Rights Watch welcomes the move,"""
					    """ says governments seeking to penalize Palestine should end pressure ."""
					)
					UpperCAmelCase__    						=   summarizer(lowerCamelCase_			)
					self.assertEqual(result[0]["""summary_text"""]						,    lowerCamelCase_			)










import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


snake_case__ :						str     =			logging.get_logger(__name__)

snake_case__ :						List[str]     =			{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

snake_case__ :						Any     =			{
    '''vocab_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''Salesforce/codegen-350M-mono''': '''https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt''',
    },
}

snake_case__ :						Tuple     =			{
    '''Salesforce/codegen-350M-mono''': 2_0_4_8,
}


@lru_cache()
def 						lowercase		(     ):
  UpperCAmelCase__    						=   (
      list(range(ord("""!"""      )   ,     ord("""~"""      ) + 1      )      ) + list(range(ord("""¡"""      )   ,     ord("""¬"""      ) + 1      )      ) + list(range(ord("""®"""      )   ,     ord("""ÿ"""      ) + 1      )      )
  )
  UpperCAmelCase__    						=   bs[:]
  UpperCAmelCase__    						=   0
  for b in range(2**8      ):
    if b not in bs:
      bs.append(_lowerCAmelCase      )
      cs.append(2**8 + n      )
      n += 1
  UpperCAmelCase__    						=   [chr(_lowerCAmelCase      ) for n in cs]
  return dict(zip(_lowerCAmelCase   ,     _lowerCAmelCase      )      )


def 						lowercase		(     _lowerCAmelCase      ):
  UpperCAmelCase__    						=   set()
  UpperCAmelCase__    						=   word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)      )
    UpperCAmelCase__    						=   char
  return pairs





class     snake_case  (	_snake_case							):


   '''simple docstring'''







   UpperCamelCase__	:       List[Any]     =      VOCAB_FILES_NAMES
   UpperCamelCase__	:       Union[str, Any]     =      PRETRAINED_VOCAB_FILES_MAP
   UpperCamelCase__	:       int     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   UpperCamelCase__	:       int     =      ["input_ids", "attention_mask"]




   def __init__(					self     :	Union[str, Any]						,    lowerCamelCase_     :	Tuple						,    lowerCamelCase_     :	Any						,    lowerCamelCase_     :	Union[str, Any]="replace"						,    lowerCamelCase_     :	List[str]="<|endoftext|>"						,    lowerCamelCase_     :	Dict="<|endoftext|>"						,    lowerCamelCase_     :	List[str]="<|endoftext|>"						,    lowerCamelCase_     :	str=None						,    lowerCamelCase_     :	Any=False						,    lowerCamelCase_     :	Optional[int]=False						,    **lowerCamelCase_     :	str						,    )					->Dict:
     '''simple docstring'''







     UpperCAmelCase__    						=   AddedToken(lowerCamelCase_						,    lstrip=lowerCamelCase_						,    rstrip=lowerCamelCase_			) if isinstance(lowerCamelCase_						,    lowerCamelCase_			) else bos_token
     UpperCAmelCase__    						=   AddedToken(lowerCamelCase_						,    lstrip=lowerCamelCase_						,    rstrip=lowerCamelCase_			) if isinstance(lowerCamelCase_						,    lowerCamelCase_			) else eos_token
     UpperCAmelCase__    						=   AddedToken(lowerCamelCase_						,    lstrip=lowerCamelCase_						,    rstrip=lowerCamelCase_			) if isinstance(lowerCamelCase_						,    lowerCamelCase_			) else unk_token
     UpperCAmelCase__    						=   AddedToken(lowerCamelCase_						,    lstrip=lowerCamelCase_						,    rstrip=lowerCamelCase_			) if isinstance(lowerCamelCase_						,    lowerCamelCase_			) else pad_token
     super().__init__(
         errors=lowerCamelCase_						,    unk_token=lowerCamelCase_						,    bos_token=lowerCamelCase_						,    eos_token=lowerCamelCase_						,    pad_token=lowerCamelCase_						,    add_prefix_space=lowerCamelCase_						,    add_bos_token=lowerCamelCase_						,    **lowerCamelCase_						,    )
     UpperCAmelCase__    						=   add_bos_token

     with open(lowerCamelCase_						,    encoding="""utf-8"""			) as vocab_handle:
       UpperCAmelCase__    						=   json.load(lowerCamelCase_			)
     UpperCAmelCase__    						=   {v: k for k, v in self.encoder.items()}
     UpperCAmelCase__    						=   errors  # how to handle errors in decoding
     UpperCAmelCase__    						=   bytes_to_unicode()
     UpperCAmelCase__    						=   {v: k for k, v in self.byte_encoder.items()}
     with open(lowerCamelCase_						,    encoding="""utf-8"""			) as merges_handle:
       UpperCAmelCase__    						=   merges_handle.read().split("""\n"""			)[1:-1]
     UpperCAmelCase__    						=   [tuple(merge.split()			) for merge in bpe_merges]
     UpperCAmelCase__    						=   dict(zip(lowerCamelCase_						,    range(len(lowerCamelCase_			)			)			)			)
     UpperCAmelCase__    						=   {}
     UpperCAmelCase__    						=   add_prefix_space

     # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
     UpperCAmelCase__    						=   re.compile(R"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"""			)




   @property
   def 	UpperCAmelCase       (					self     :	Dict			)					->Any:
     '''simple docstring'''







     return len(self.encoder			)




   def 	UpperCAmelCase       (					self     :	Dict			)					->Optional[int]:
     '''simple docstring'''







     return dict(self.encoder						,    **self.added_tokens_encoder			)




   def 	UpperCAmelCase       (					self     :	Tuple						,    lowerCamelCase_     :	List[str]			)					->Optional[Any]:
     '''simple docstring'''







     if token in self.cache:
       return self.cache[token]
     UpperCAmelCase__    						=   tuple(lowerCamelCase_			)
     UpperCAmelCase__    						=   get_pairs(lowerCamelCase_			)

     if not pairs:
       return token

     while True:
       UpperCAmelCase__    						=   min(lowerCamelCase_						,    key=lambda lowerCamelCase_			: self.bpe_ranks.get(lowerCamelCase_						,    float("""inf"""			)			)			)
       if bigram not in self.bpe_ranks:
         break
       UpperCAmelCase__						,					UpperCAmelCase__    						=   bigram
       UpperCAmelCase__    						=   []
       UpperCAmelCase__    						=   0
       while i < len(lowerCamelCase_			):
         try:
           UpperCAmelCase__    						=   word.index(lowerCamelCase_						,    lowerCamelCase_			)
         except ValueError:
           new_word.extend(word[i:]			)
           break
         else:
           new_word.extend(word[i:j]			)
           UpperCAmelCase__    						=   j

         if word[i] == first and i < len(lowerCamelCase_			) - 1 and word[i + 1] == second:
           new_word.append(first + second			)
           i += 2
         else:
           new_word.append(word[i]			)
           i += 1
       UpperCAmelCase__    						=   tuple(lowerCamelCase_			)
       UpperCAmelCase__    						=   new_word
       if len(lowerCamelCase_			) == 1:
         break
       else:
         UpperCAmelCase__    						=   get_pairs(lowerCamelCase_			)
     UpperCAmelCase__    						=   """ """.join(lowerCamelCase_			)
     UpperCAmelCase__    						=   word
     return word




   def 	UpperCAmelCase       (					self     :	int						,    lowerCamelCase_     :	Optional[int]						,    lowerCamelCase_     :	Optional[Any]=None			)					->str:
     '''simple docstring'''







     if self.add_bos_token:
       UpperCAmelCase__    						=   [self.bos_token_id]
     else:
       UpperCAmelCase__    						=   []

     UpperCAmelCase__    						=   bos_token_ids + token_ids_a

     if token_ids_a is None:
       return output

     return output + bos_token_ids + token_ids_a




   def 	UpperCAmelCase       (					self     :	Any						,    lowerCamelCase_     :	Optional[Any]			)					->Union[str, Any]:
     '''simple docstring'''







     UpperCAmelCase__    						=   []
     for token in re.findall(self.pat						,    lowerCamelCase_			):
       UpperCAmelCase__    						=   """""".join(
           self.byte_encoder[b] for b in token.encode("""utf-8"""			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowerCamelCase_			).split(""" """			)			)
     return bpe_tokens




   def 	UpperCAmelCase       (					self     :	Dict						,    lowerCamelCase_     :	Optional[Any]			)					->int:
     '''simple docstring'''







     return self.encoder.get(lowerCamelCase_						,    self.encoder.get(self.unk_token			)			)




   def 	UpperCAmelCase       (					self     :	Union[str, Any]						,    lowerCamelCase_     :	Dict			)					->Union[str, Any]:
     '''simple docstring'''







     return self.decoder.get(lowerCamelCase_			)




   def 	UpperCAmelCase       (					self     :	int						,    lowerCamelCase_     :	Union[str, Any]			)					->Union[str, Any]:
     '''simple docstring'''







     UpperCAmelCase__    						=   """""".join(lowerCamelCase_			)
     UpperCAmelCase__    						=   bytearray([self.byte_decoder[c] for c in text]			).decode("""utf-8"""						,    errors=self.errors			)
     return text




   def 	UpperCAmelCase       (					self     :	Dict						,    lowerCamelCase_     :	str						,    lowerCamelCase_     :	Optional[str] = None			)					->Tuple[str]:
     '''simple docstring'''







     if not os.path.isdir(lowerCamelCase_			):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
       return
     UpperCAmelCase__    						=   os.path.join(
         lowerCamelCase_						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)
     UpperCAmelCase__    						=   os.path.join(
         lowerCamelCase_						,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]			)

     with open(lowerCamelCase_						,    """w"""						,    encoding="""utf-8"""			) as f:
       f.write(json.dumps(self.encoder						,    indent=2						,    sort_keys=lowerCamelCase_						,    ensure_ascii=lowerCamelCase_			) + """\n"""			)

     UpperCAmelCase__    						=   0
     with open(lowerCamelCase_						,    """w"""						,    encoding="""utf-8"""			) as writer:
       writer.write("""#version: 0.2\n"""			)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()						,    key=lambda lowerCamelCase_			: kv[1]			):
         if index != token_index:
           logger.warning(
               f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
               """ Please check that the tokenizer is not corrupted!"""			)
           UpperCAmelCase__    						=   token_index
         writer.write(""" """.join(lowerCamelCase_			) + """\n"""			)
         index += 1

     return vocab_file, merge_file




   def 	UpperCAmelCase       (					self     :	List[Any]						,    lowerCamelCase_     :	int						,    lowerCamelCase_     :	Tuple=False						,    **lowerCamelCase_     :	Any			)					->Any:
     '''simple docstring'''







     UpperCAmelCase__    						=   kwargs.pop("""add_prefix_space"""						,    self.add_prefix_space			)
     if is_split_into_words or add_prefix_space:
       UpperCAmelCase__    						=   """ """ + text
     return (text, kwargs)




   def 	UpperCAmelCase       (					self     :	List[Any]						,    lowerCamelCase_     :	Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"]						,    lowerCamelCase_     :	bool = False						,    lowerCamelCase_     :	bool = None						,    lowerCamelCase_     :	Optional[List[str]] = None						,    **lowerCamelCase_     :	Any						,    )					->str:
     '''simple docstring'''







     UpperCAmelCase__    						=   super()._decode(
         token_ids=lowerCamelCase_						,    skip_special_tokens=lowerCamelCase_						,    clean_up_tokenization_spaces=lowerCamelCase_						,    **lowerCamelCase_						,    )

     if truncate_before_pattern is not None and len(lowerCamelCase_			) > 0:
       UpperCAmelCase__    						=   self.truncate(lowerCamelCase_						,    lowerCamelCase_			)

     return decoded_text







   def 	UpperCAmelCase       (					self     :	Tuple						,    lowerCamelCase_     :	Any						,    lowerCamelCase_     :	str			)					->Optional[int]:
     '''simple docstring'''







     def find_re(lowerCamelCase_     :	List[str]						,    lowerCamelCase_     :	Union[str, Any]						,    lowerCamelCase_     :	int			):
       UpperCAmelCase__    						=   pattern.search(lowerCamelCase_						,    lowerCamelCase_			)
       return m.start() if m else -1

     UpperCAmelCase__    						=   [re.compile(lowerCamelCase_						,    re.MULTILINE			) for pattern in truncate_before_pattern]

     UpperCAmelCase__    						=   list(re.finditer("""^print"""						,    lowerCamelCase_						,    re.MULTILINE			)			)

     if len(lowerCamelCase_			) > 1:
       UpperCAmelCase__    						=   completion[: prints[1].start()]

     UpperCAmelCase__    						=   list(re.finditer("""^def"""						,    lowerCamelCase_						,    re.MULTILINE			)			)

     if len(lowerCamelCase_			) > 1:
       UpperCAmelCase__    						=   completion[: defs[1].start()]

     UpperCAmelCase__    						=   0

     UpperCAmelCase__    						=   [
         pos for pos in [find_re(lowerCamelCase_						,    lowerCamelCase_						,    lowerCamelCase_			) for terminal in terminals] if pos != -1
     ]

     if len(lowerCamelCase_			) > 0:
       return completion[: min(lowerCamelCase_			)]
     else:
       return completion




