code: 
  '''simple docstring'''
  
  
  
  
  
  
  
  import unittest
  
  from transformers import (
      MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
      TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
      SummarizationPipeline,
      TFPreTrainedModel,
      pipeline,
  )
  from transformers.testing_utils import get_gpu_count, is_pipeline_test, require_tf, require_torch, slow, torch_device
  from transformers.tokenization_utils import TruncationStrategy
  
  from .test_pipelines_common import ANY
  
  
  _lowerCamelCase							:		Any										=						-1 if torch_device == 'cpu' else 0
  
  
  
  
  
  @is_pipeline_test
  class   lowerCamelCase_							( unittest.TestCase							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  					lowercase__					=				TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase,						__UpperCamelCase				):
  							_snake_case			 = SummarizationPipeline(model=__UpperCamelCase,						tokenizer=__UpperCamelCase				)
  							return summarizer, ["(CNN)The Palestinian Authority officially became", "Some other text"]
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase				):
  							_snake_case			 = summarizer.model
  
  							_snake_case			 = summarizer("(CNN)The Palestinian Authority officially became"				)
  							self.assertEqual(__UpperCamelCase,						[{"summary_text": ANY(__UpperCamelCase				)}]				)
  
  							_snake_case			 = summarizer(
  							    "(CNN)The Palestinian Authority officially became ",						num_beams=2,						min_length=2,						max_length=5,						)
  							self.assertEqual(__UpperCamelCase,						[{"summary_text": ANY(__UpperCamelCase				)}]				)
  
  							# Some models (Switch Transformers, LED, T5, LongT5, etc) can handle long sequences.
  							_snake_case			 = [
  							    "SwitchTransformersConfig",
  							    "T5Config",
  							    "LongT5Config",
  							    "LEDConfig",
  							    "PegasusXConfig",
  							    "FSMTConfig",
  							    "M2M100Config",
  							    "ProphetNetConfig",  # positional embeddings up to a fixed maximum size (otherwise clamping the values)
  							]
  							if model.config.__class__.__name__ not in model_can_handle_longer_seq:
  									# Too long and exception is expected.
  									# For TF models, if the weights are initialized in GPU context, we won't get expected index error from
  									# the embedding layer.
  									if not (
  									    isinstance(__UpperCamelCase,						__UpperCamelCase				)
  									    and get_gpu_count() > 0
  									    and len(summarizer.model.trainable_weights				) > 0
  									):
  											with self.assertRaises(__UpperCamelCase				):
  													_snake_case			 = summarizer("This " * 1_000				)
  							_snake_case			 = summarizer("This " * 1_000,						truncation=TruncationStrategy.ONLY_FIRST				)
  
  
  
  
  
  
  					@require_torch
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = pipeline(task="summarization",						model="sshleifer/tiny-mbart",						framework="pt"				)
  							_snake_case			 = summarizer("This is a small test"				)
  							self.assertEqual(
  							    __UpperCamelCase,						[
  							        {
  							            "summary_text": "เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป"
  							        }
  							    ],						)
  
  
  
  
  
  
  					@require_tf
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = pipeline(task="summarization",						model="sshleifer/tiny-mbart",						framework="tf"				)
  							_snake_case			 = summarizer("This is a small test"				)
  							self.assertEqual(
  							    __UpperCamelCase,						[
  							        {
  							            "summary_text": "เข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไปเข้าไป"
  							        }
  							    ],						)
  
  
  
  
  
  
  					@require_torch
  					@slow
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = pipeline(task="summarization",						device=__UpperCamelCase				)
  							_snake_case			 = (
  							    " (CNN)The Palestinian Authority officially became the 123rd member of the International Criminal Court on"
  							    " Wednesday, a step that gives the court jurisdiction over alleged crimes in Palestinian territories. The"
  							    " formal accession was marked with a ceremony at The Hague, in the Netherlands, where the court is based."
  							    " The Palestinians signed the ICC's founding Rome Statute in January, when they also accepted its"
  							    " jurisdiction over alleged crimes committed \"in the occupied Palestinian territory, including East"
  							    " Jerusalem, since June 13, 2014.\" Later that month, the ICC opened a preliminary examination into the"
  							    " situation in Palestinian territories, paving the way for possible war crimes investigations against"
  							    " Israelis. As members of the court, Palestinians may be subject to counter-charges as well. Israel and"
  							    " the United States, neither of which is an ICC member, opposed the Palestinians' efforts to join the"
  							    " body. But Palestinian Foreign Minister Riad al-Malki, speaking at Wednesday's ceremony, said it was a"
  							    " move toward greater justice. \"As Palestine formally becomes a State Party to the Rome Statute today, the"
  							    " world is also a step closer to ending a long era of impunity and injustice,\" he said, according to an"
  							    " ICC news release. \"Indeed, today brings us closer to our shared goals of justice and peace.\" Judge"
  							    " Kuniko Ozaki, a vice president of the ICC, said acceding to the treaty was just the first step for the"
  							    " Palestinians. \"As the Rome Statute today enters into force for the State of Palestine, Palestine"
  							    " acquires all the rights as well as responsibilities that come with being a State Party to the Statute."
  							    " These are substantive commitments, which cannot be taken lightly,\" she said. Rights group Human Rights"
  							    " Watch welcomed the development. \"Governments seeking to penalize Palestine for joining the ICC should"
  							    " immediately end their pressure, and countries that support universal acceptance of the court's treaty"
  							    " should speak out to welcome its membership,\" said Balkees Jarrah, international justice counsel for the"
  							    " group. \"What's objectionable is the attempts to undermine international justice, not Palestine's"
  							    " decision to join a treaty to which over 100 countries around the world are members.\" In January, when"
  							    " the preliminary ICC examination was opened, Israeli Prime Minister Benjamin Netanyahu described it as an"
  							    " outrage, saying the court was overstepping its boundaries. The United States also said it \"strongly\""
  							    " disagreed with the court's decision. \"As we have said repeatedly, we do not believe that Palestine is a"
  							    " state and therefore we do not believe that it is eligible to join the ICC,\" the State Department said in"
  							    " a statement. It urged the warring sides to resolve their differences through direct negotiations. \"We"
  							    " will continue to oppose actions against Israel at the ICC as counterproductive to the cause of peace,\""
  							    " it said. But the ICC begs to differ with the definition of a state for its purposes and refers to the"
  							    " territories as \"Palestine.\" While a preliminary examination is not a formal investigation, it allows the"
  							    " court to review evidence and determine whether to investigate suspects on both sides. Prosecutor Fatou"
  							    " Bensouda said her office would \"conduct its analysis in full independence and impartiality.\" The war"
  							    " between Israel and Hamas militants in Gaza last summer left more than 2,000 people dead. The inquiry"
  							    " will include alleged war crimes committed since June. The International Criminal Court was set up in"
  							    " 2002 to prosecute genocide, crimes against humanity and war crimes. CNN's Vasco Cotovio, Kareem Khadder"
  							    " and Faith Karimi contributed to this report."
  							)
  							_snake_case			 = (
  							    " The Palestinian Authority becomes the 123rd member of the International Criminal Court . The move gives"
  							    " the court jurisdiction over alleged crimes in Palestinian territories . Israel and the United States"
  							    " opposed the Palestinians' efforts to join the court . Rights group Human Rights Watch welcomes the move,"
  							    " says governments seeking to penalize Palestine should end pressure ."
  							)
  							_snake_case			 = summarizer(__UpperCamelCase				)
  							self.assertEqual(result[0]["summary_text"],						__UpperCamelCase				)
  
  
  
  
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import json
  from typing import List, Optional, Tuple
  
  from tokenizers import normalizers
  
  from ...tokenization_utils import AddedToken
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import logging
  from .tokenization_mpnet import MPNetTokenizer
  
  
  _lowerCamelCase							:		int										=						logging.get_logger(__name__)
  
  _lowerCamelCase							:		Any										=						{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}
  
  _lowerCamelCase							:		Dict										=						{
      'vocab_file': {
          'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt',
      },
      'tokenizer_file': {
          'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json',
      },
  }
  
  _lowerCamelCase							:		List[str]										=						{
      'microsoft/mpnet-base': 512,
  }
  
  _lowerCamelCase							:		Tuple										=						{
      'microsoft/mpnet-base': {'do_lower_case': True},
  }
  
  
  
  
  
  class   lowerCamelCase_							( UpperCAmelCase_							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				VOCAB_FILES_NAMES
  					lowercase__					=				PRETRAINED_VOCAB_FILES_MAP
  					lowercase__					=				PRETRAINED_INIT_CONFIGURATION
  					lowercase__					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  					lowercase__					=				MPNetTokenizer
  					lowercase__					=				["""input_ids""", """attention_mask"""]
  
  
  
  
  
  
  					def __init__(					self,						__UpperCamelCase=None,						__UpperCamelCase=None,						__UpperCamelCase=True,						__UpperCamelCase="<s>",						__UpperCamelCase="</s>",						__UpperCamelCase="</s>",						__UpperCamelCase="<s>",						__UpperCamelCase="[UNK]",						__UpperCamelCase="<pad>",						__UpperCamelCase="<mask>",						__UpperCamelCase=True,						__UpperCamelCase=None,						**__UpperCamelCase,						):
  							super().__init__(
  							    __UpperCamelCase,						tokenizer_file=__UpperCamelCase,						do_lower_case=__UpperCamelCase,						bos_token=__UpperCamelCase,						eos_token=__UpperCamelCase,						sep_token=__UpperCamelCase,						cls_token=__UpperCamelCase,						unk_token=__UpperCamelCase,						pad_token=__UpperCamelCase,						mask_token=__UpperCamelCase,						tokenize_chinese_chars=__UpperCamelCase,						strip_accents=__UpperCamelCase,						**__UpperCamelCase,						)
  
  							_snake_case			 = json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
  							if (
  							    pre_tok_state.get("lowercase",						__UpperCamelCase				) != do_lower_case
  							    or pre_tok_state.get("strip_accents",						__UpperCamelCase				) != strip_accents
  							):
  									_snake_case			 = getattr(__UpperCamelCase,						pre_tok_state.pop("type"				)				)
  									_snake_case			 = do_lower_case
  									_snake_case			 = strip_accents
  									_snake_case			 = pre_tok_class(**__UpperCamelCase				)
  
  							_snake_case			 = do_lower_case
  
  
  
  
  
  
  					@property
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							if self._mask_token is None:
  									if self.verbose:
  											logger.error("Using mask_token, but it is not set yet."				)
  									return None
  							return str(self._mask_token				)
  
  
  
  
  
  
  					@mask_token.setter
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case			 = AddedToken(__UpperCamelCase,						lstrip=__UpperCamelCase,						rstrip=__UpperCamelCase				) if isinstance(__UpperCamelCase,						__UpperCamelCase				) else value
  							_snake_case			 = value
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase=None				):
  							_snake_case			 = [self.bos_token_id] + token_ids_a + [self.eos_token_id]
  							if token_ids_a is None:
  									return output
  
  							return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = None				):
  							_snake_case			 = [self.sep_token_id]
  							_snake_case			 = [self.cls_token_id]
  
  							if token_ids_a is None:
  									return len(cls + token_ids_a + sep				) * [0]
  							return len(cls + token_ids_a + sep + sep + token_ids_a + sep				) * [0]
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						__UpperCamelCase = None				):
  							_snake_case			 = self._tokenizer.model.save(__UpperCamelCase,						name=__UpperCamelCase				)
  							return tuple(__UpperCamelCase				)
  
  
  
  
  
  