code: 
  
  
  
  
  """simple docstring"""
  
  
  
  
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  
  if is_torch_available():
    import torch
  
    from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer
  @require_torch
  class 							lowerCamelCase		( unittest.TestCase				):
  
  
  
  
  
  
  
     @slow
     def       a_							(							self ):
         UpperCamelCase				:   Tuple          =							XLMProphetNetForConditionalGeneration.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased""" )
         model.to(SCREAMING_SNAKE_CASE_ )
  
         # encoder-decoder outputs
         UpperCamelCase				:   List[Any]          =							torch.tensor([[17, 9_6208, 10_3471, 2]] ).to(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Optional[int]          =							torch.tensor(
             [[2, 250, 9953, 34, 6_9489, 1620, 32, 11_8424, 624, 210, 105, 2913, 1032, 351]] ).to(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Dict          =							model(
             input_ids=SCREAMING_SNAKE_CASE_    ,      attention_mask=SCREAMING_SNAKE_CASE_    ,      encoder_outputs=SCREAMING_SNAKE_CASE_    ,      decoder_input_ids=SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   str          =							output[0]
         UpperCamelCase				:   str          =							torch.Size((1, 14, 25_0012) )
         self.assertEqual(output_predited_logis.shape    ,      SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   str          =							torch.tensor(
             [[[-6.3986, -8.2391, 12.5189], [-6.3289, -8.0864, 12.6211], [-6.2418, -8.0445, 12.7968]]] ).to(SCREAMING_SNAKE_CASE_ )
         self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )
  
         # encoder outputs
         UpperCamelCase				:   str          =							model.prophetnet.encoder(SCREAMING_SNAKE_CASE_ )[0]
         UpperCamelCase				:   str          =							torch.tensor(
             [[[-1.4260, -0.7628, 0.8453], [-1.4719, -0.1391, 0.7807], [-1.7678, 0.0114, 0.4646]]] ).to(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   int          =							torch.Size((1, 4, 1024) )
         self.assertEqual(encoder_outputs.shape    ,      SCREAMING_SNAKE_CASE_ )
         self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )
  
         # decoder outputs
         UpperCamelCase				:   Union[str, Any]          =							model.prophetnet.decoder(
             SCREAMING_SNAKE_CASE_    ,      encoder_hidden_states=SCREAMING_SNAKE_CASE_    ,      )
         UpperCamelCase				:   str          =							decoder_outputs[1].view(1    ,      model.config.ngram    ,      14    ,      -1 )
         UpperCamelCase				:   Optional[int]          =							model.lm_head(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   str          =							predicting_streams_logits[:, 0]
         self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )
  
  
  
  
  
  
  
     @slow
     def       a_							(							self ):
         UpperCamelCase				:   Optional[int]          =							XLMProphetNetForConditionalGeneration.from_pretrained(
             """microsoft/xprophetnet-large-wiki100-cased-xglue-ntg""" )
         model.to(SCREAMING_SNAKE_CASE_ )
  
         UpperCamelCase				:   Union[str, Any]          =							torch.tensor([[17, 9_6208, 10_3471, 2]] ).to(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   str          =							torch.tensor(
             [[2, 250, 9953, 34, 6_9489, 1620, 32, 11_8424, 624, 210, 105, 2913, 1032, 351]] ).to(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   str          =							model(
             input_ids=SCREAMING_SNAKE_CASE_    ,      attention_mask=SCREAMING_SNAKE_CASE_    ,      encoder_outputs=SCREAMING_SNAKE_CASE_    ,      decoder_input_ids=SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   List[Any]          =							output[0]
         UpperCamelCase				:   List[str]          =							torch.Size((1, 14, 25_0012) )
         self.assertEqual(output_predited_logis.shape    ,      SCREAMING_SNAKE_CASE_ )
         # compare the actual values for a slice.
         UpperCamelCase				:   Optional[int]          =							torch.tensor(
             [[[-9.2253, -9.7173, -6.3529], [-7.6701, -9.0145, -1.9382], [-8.0195, -7.0004, -0.1523]]] ).to(SCREAMING_SNAKE_CASE_ )
  
         self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]    ,      SCREAMING_SNAKE_CASE_    ,      atol=1e-4 ) )
  
  
  
  
  
  
  
     @slow
     def       a_							(							self ):
         UpperCamelCase				:   Optional[int]          =							XLMProphetNetForConditionalGeneration.from_pretrained(
             """microsoft/xprophetnet-large-wiki100-cased-xglue-ntg""" )
         model.to(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Optional[int]          =							512
  
         UpperCamelCase				:   Optional[Any]          =							XLMProphetNetTokenizer.from_pretrained("""microsoft/xprophetnet-large-wiki100-cased-xglue-ntg""" )
  
         UpperCamelCase				:   Optional[int]          =							(
             """Microsoft Corporation intends to officially end free support for the Windows 7 operating system after"""
             """ January 14, 2020, according to the official portal of the organization. From that day, users of this"""
             """ system will not be able to receive security updates, which could make their computers vulnerable to"""
             """ cyber attacks."""
         )
         UpperCamelCase				:   int          =							(
             """орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7"""
             """ после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи"""
             """ этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми"""
             """ к кибератакам."""
         )
         UpperCamelCase				:   Optional[Any]          =							(
             """根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"""
         )
  
         UpperCamelCase				:   Union[str, Any]          =							tokenizer(
             [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]    ,      padding=SCREAMING_SNAKE_CASE_    ,      max_length=255    ,      return_tensors="""pt""" ).input_ids
         UpperCamelCase				:   List[str]          =							input_ids.to(SCREAMING_SNAKE_CASE_ )
  
         UpperCamelCase				:   int          =							model.generate(
             SCREAMING_SNAKE_CASE_    ,      num_beams=10    ,      length_penalty=1.0    ,      no_repeat_ngram_size=3    ,      early_stopping=SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Tuple          =							[tokenizer.decode(SCREAMING_SNAKE_CASE_    ,      skip_special_tokens=SCREAMING_SNAKE_CASE_ ) for g in summary_ids]
         UpperCamelCase				:   Tuple          =							"""Microsoft to end Windows 7 free support after January 14, 2020"""
         UpperCamelCase				:   str          =							"""Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года"""
         UpperCamelCase				:   Optional[Any]          =							"""微软打算终止对Windows 7操作系统的免费支持"""
         self.assertListEqual(
             [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]    ,      SCREAMING_SNAKE_CASE_    ,      )
  
         UpperCamelCase				:   Optional[Any]          =							model.generate(
             SCREAMING_SNAKE_CASE_    ,      num_beams=1    ,      length_penalty=1.0    ,      no_repeat_ngram_size=3    ,      early_stopping=SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Optional[int]          =							[
             tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_    ,      skip_special_tokens=SCREAMING_SNAKE_CASE_ ) for g in summary_ids_beama
         ]
         UpperCamelCase				:   List[str]          =							"""▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7""".split(""" """ )
         UpperCamelCase				:   Optional[int]          =							"""▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года""".split(
             """ """ )
         UpperCamelCase				:   Any          =							"""微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持""".split(""" """ )
         self.assertListEqual(
             [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]    ,      SCREAMING_SNAKE_CASE_    ,      )
  
  
  
  
  
style_context: 
  
  
  
  
  """simple docstring"""
  
  
  
  
  import os
  from shutil import copyfile
  from typing import List, Optional, Tuple
  
  from tokenizers import processors
  
  from ...tokenization_utils import AddedToken, BatchEncoding
  from ...tokenization_utils_fast import PreTrainedTokenizerFast
  from ...utils import is_sentencepiece_available, logging
  
  
  if is_sentencepiece_available():
    from .tokenization_mbartaa import MBartaaTokenizer
  else:
    __A :		Tuple      							=    None
  
  
  __A :		Optional[Any]      							=    logging.get_logger(__name__)
  
  __A :		Optional[Any]      							=    {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}
  
  __A :		Optional[Any]      							=    {
      '''vocab_file''': {
          '''facebook/mbart-large-50-one-to-many-mmt''': (
              '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
          ),
      },
      '''tokenizer_file''': {
          '''facebook/mbart-large-50-one-to-many-mmt''': (
              '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
          ),
      },
  }
  
  __A :		List[Any]      							=    {
      '''facebook/mbart-large-50-one-to-many-mmt''': 1024,
  }
  
  # fmt: off
  __A :		Dict      							=    ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']
  class 							lowerCamelCase		( _UpperCAmelCase				):
     lowercase       :  List[Any]      =				VOCAB_FILES_NAMES
     lowercase       :  Optional[Any]      =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowercase       :  List[Any]      =				PRETRAINED_VOCAB_FILES_MAP
     lowercase       :  Tuple      =				['input_ids', 'attention_mask']
     lowercase       :  Optional[int]      =				MBartaaTokenizer
  
     lowercase       :  List[int]      =				[]
     lowercase       :  List[int]      =				[]
  
  
  
  
  
  
  
     def __init__(							self    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_=None    ,      SCREAMING_SNAKE_CASE_="</s>"    ,      SCREAMING_SNAKE_CASE_="</s>"    ,      SCREAMING_SNAKE_CASE_="<s>"    ,      SCREAMING_SNAKE_CASE_="<unk>"    ,      SCREAMING_SNAKE_CASE_="<pad>"    ,      SCREAMING_SNAKE_CASE_="<mask>"    ,      **SCREAMING_SNAKE_CASE_    ,      ):
         # Mask token behave like a normal word, i.e. include the space before it
         UpperCamelCase				:   Optional[int]          =							AddedToken(SCREAMING_SNAKE_CASE_    ,      lstrip=SCREAMING_SNAKE_CASE_    ,      rstrip=SCREAMING_SNAKE_CASE_ ) if isinstance(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ ) else mask_token
  
         UpperCamelCase				:   Dict          =							kwargs.get("""additional_special_tokens"""    ,      [] )
         kwargs["additional_special_tokens"] += [
             code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
         ]
  
         super().__init__(
             SCREAMING_SNAKE_CASE_    ,      src_lang=SCREAMING_SNAKE_CASE_    ,      tgt_lang=SCREAMING_SNAKE_CASE_    ,      tokenizer_file=SCREAMING_SNAKE_CASE_    ,      eos_token=SCREAMING_SNAKE_CASE_    ,      sep_token=SCREAMING_SNAKE_CASE_    ,      cls_token=SCREAMING_SNAKE_CASE_    ,      unk_token=SCREAMING_SNAKE_CASE_    ,      pad_token=SCREAMING_SNAKE_CASE_    ,      mask_token=SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_    ,      )
  
         UpperCamelCase				:   int          =							vocab_file
         UpperCamelCase				:   Optional[Any]          =							False if not self.vocab_file else True
  
         UpperCamelCase				:   List[Any]          =							{
             lang_code: self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ ) for lang_code in FAIRSEQ_LANGUAGE_CODES
         }
  
         UpperCamelCase				:   List[str]          =							src_lang if src_lang is not None else """en_XX"""
         UpperCamelCase				:   Union[str, Any]          =							tgt_lang
         UpperCamelCase				:   Union[str, Any]          =							self.lang_code_to_id[self._src_lang]
         self.set_src_lang_special_tokens(self._src_lang )
  
  
  
  
  
  
  
     @property
     def       a_							(							self ):
         return self._src_lang
  
  
  
  
  
  
  
     @src_lang.setter
     def       a_							(							self    ,      SCREAMING_SNAKE_CASE_ ):
         UpperCamelCase				:   str          =							new_src_lang
         self.set_src_lang_special_tokens(self._src_lang )
  
  
  
  
  
  
  
     def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ = None ):
         if token_ids_a is None:
             return self.prefix_tokens + token_ids_a + self.suffix_tokens
         # We don't expect to process pairs, but leave the pair logic for API consistency
         return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens
  
  
  
  
  
  
  
     def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ = "en_XX"    ,      SCREAMING_SNAKE_CASE_ = None    ,      SCREAMING_SNAKE_CASE_ = "ro_RO"    ,      **SCREAMING_SNAKE_CASE_    ,      ):
         UpperCamelCase				:   str          =							src_lang
         UpperCamelCase				:   Optional[Any]          =							tgt_lang
         return super().prepare_seqaseq_batch(SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_ )
  
  
  
  
  
  
  
     def       a_							(							self ):
         return self.set_src_lang_special_tokens(self.src_lang )
  
  
  
  
  
  
  
     def       a_							(							self ):
         return self.set_tgt_lang_special_tokens(self.tgt_lang )
  
  
  
  
  
  
  
     def       a_							(							self    ,      SCREAMING_SNAKE_CASE_ ):
         UpperCamelCase				:   Optional[Any]          =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Tuple          =							[self.cur_lang_code_id]
         UpperCamelCase				:   Optional[int]          =							[self.eos_token_id]
  
         UpperCamelCase				:   Dict          =							self.convert_ids_to_tokens(self.prefix_tokens )
         UpperCamelCase				:   Any          =							self.convert_ids_to_tokens(self.suffix_tokens )
  
         UpperCamelCase				:   Any          =							processors.TemplateProcessing(
             single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,      pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,      self.prefix_tokens + self.suffix_tokens ) )    ,      )
  
  
  
  
  
  
  
     def       a_							(							self    ,      SCREAMING_SNAKE_CASE_ ):
         UpperCamelCase				:   Any          =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Optional[Any]          =							[self.cur_lang_code_id]
         UpperCamelCase				:   Tuple          =							[self.eos_token_id]
  
         UpperCamelCase				:   str          =							self.convert_ids_to_tokens(self.prefix_tokens )
         UpperCamelCase				:   Dict          =							self.convert_ids_to_tokens(self.suffix_tokens )
  
         UpperCamelCase				:   Dict          =							processors.TemplateProcessing(
             single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str    ,      pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str    ,      special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str    ,      self.prefix_tokens + self.suffix_tokens ) )    ,      )
  
  
  
  
  
  
  
     def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_ ):
         if src_lang is None or tgt_lang is None:
             raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model""" )
         UpperCamelCase				:   Optional[int]          =							src_lang
         UpperCamelCase				:   Optional[int]          =							self(SCREAMING_SNAKE_CASE_    ,      add_special_tokens=SCREAMING_SNAKE_CASE_    ,      return_tensors=SCREAMING_SNAKE_CASE_    ,      **SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   Dict          =							self.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_ )
         UpperCamelCase				:   str          =							tgt_lang_id
         return inputs
  
  
  
  
  
  
  
     def       a_							(							self    ,      SCREAMING_SNAKE_CASE_    ,      SCREAMING_SNAKE_CASE_ = None ):
         if not self.can_save_slow_tokenizer:
             raise ValueError(
                 """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                 """tokenizer.""" )
  
         if not os.path.isdir(SCREAMING_SNAKE_CASE_ ):
             logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
             return
         UpperCamelCase				:   Optional[int]          =							os.path.join(
             SCREAMING_SNAKE_CASE_    ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )
  
         if os.path.abspath(self.vocab_file ) != os.path.abspath(SCREAMING_SNAKE_CASE_ ):
             copyfile(self.vocab_file    ,      SCREAMING_SNAKE_CASE_ )
  
         return (out_vocab_file,)
  
  
  
  
  