code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  import unittest
  
  from transformers import is_torch_available
  from transformers.testing_utils import require_torch, slow, torch_device
  
  
  if is_torch_available():
  			import torch
  
  			from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer
  
  
  
  
  
  
  @require_torch
  class 							__snake_case    (				unittest.TestCase       ):
  
  
  
  
  
  	@slow
  	def      _snake_case      (		self		)			->		int:
  		snake_case__  					=      XLMProphetNetForConditionalGeneration.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'		)
  		model.to(UpperCamelCase_		)
  
  		# encoder-decoder outputs
  		snake_case__  					=      torch.tensor([[17, 9_6208, 10_3471, 2]]		).to(UpperCamelCase_		)
  		snake_case__  					=      torch.tensor(
  		    [[2, 250, 9953, 34, 6_9489, 1620, 32, 11_8424, 624, 210, 105, 2913, 1032, 351]]		).to(UpperCamelCase_		)
  		snake_case__  					=      model(
  		    input_ids=UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					encoder_outputs=UpperCamelCase_			,					decoder_input_ids=UpperCamelCase_		)
  		snake_case__  					=      output[0]
  		snake_case__  					=      torch.Size((1, 14, 25_0012)		)
  		self.assertEqual(output_predited_logis.shape			,					UpperCamelCase_		)
  		snake_case__  					=      torch.tensor(
  		    [[[-6.3_9_8_6, -8.2_3_9_1, 1_2.5_1_8_9], [-6.3_2_8_9, -8.0_8_6_4, 1_2.6_2_1_1], [-6.2_4_1_8, -8.0_4_4_5, 1_2.7_9_6_8]]]		).to(UpperCamelCase_		)
  		self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]			,					UpperCamelCase_			,					atol=1E-4		)		)
  
  		# encoder outputs
  		snake_case__  					=      model.prophetnet.encoder(UpperCamelCase_		)[0]
  		snake_case__  					=      torch.tensor(
  		    [[[-1.4_2_6_0, -0.7_6_2_8, 0.8_4_5_3], [-1.4_7_1_9, -0.1_3_9_1, 0.7_8_0_7], [-1.7_6_7_8, 0.0_1_1_4, 0.4_6_4_6]]]		).to(UpperCamelCase_		)
  		snake_case__  					=      torch.Size((1, 4, 1024)		)
  		self.assertEqual(encoder_outputs.shape			,					UpperCamelCase_		)
  		self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]			,					UpperCamelCase_			,					atol=1E-4		)		)
  
  		# decoder outputs
  		snake_case__  					=      model.prophetnet.decoder(
  		    UpperCamelCase_			,					encoder_hidden_states=UpperCamelCase_			,					)
  		snake_case__  					=      decoder_outputs[1].view(1			,					model.config.ngram			,					14			,					-1		)
  		snake_case__  					=      model.lm_head(UpperCamelCase_		)
  		snake_case__  					=      predicting_streams_logits[:, 0]
  		self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]			,					UpperCamelCase_			,					atol=1E-4		)		)
  
  
  
  
  
  	@slow
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      XLMProphetNetForConditionalGeneration.from_pretrained(
  		    'microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'		)
  		model.to(UpperCamelCase_		)
  
  		snake_case__  					=      torch.tensor([[17, 9_6208, 10_3471, 2]]		).to(UpperCamelCase_		)
  		snake_case__  					=      torch.tensor(
  		    [[2, 250, 9953, 34, 6_9489, 1620, 32, 11_8424, 624, 210, 105, 2913, 1032, 351]]		).to(UpperCamelCase_		)
  		snake_case__  					=      model(
  		    input_ids=UpperCamelCase_			,					attention_mask=UpperCamelCase_			,					encoder_outputs=UpperCamelCase_			,					decoder_input_ids=UpperCamelCase_		)
  		snake_case__  					=      output[0]
  		snake_case__  					=      torch.Size((1, 14, 25_0012)		)
  		self.assertEqual(output_predited_logis.shape			,					UpperCamelCase_		)
  		# compare the actual values for a slice.
  		snake_case__  					=      torch.tensor(
  		    [[[-9.2_2_5_3, -9.7_1_7_3, -6.3_5_2_9], [-7.6_7_0_1, -9.0_1_4_5, -1.9_3_8_2], [-8.0_1_9_5, -7.0_0_0_4, -0.1_5_2_3]]]		).to(UpperCamelCase_		)
  
  		self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]			,					UpperCamelCase_			,					atol=1E-4		)		)
  
  
  
  
  
  	@slow
  	def      _snake_case      (		self		)			->		Tuple:
  		snake_case__  					=      XLMProphetNetForConditionalGeneration.from_pretrained(
  		    'microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'		)
  		model.to(UpperCamelCase_		)
  		snake_case__  					=      512
  
  		snake_case__  					=      XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased-xglue-ntg'		)
  
  		snake_case__  					=      (
  		    'Microsoft Corporation intends to officially end free support for the Windows 7 operating system after'
  		    ' January 14, 2020, according to the official portal of the organization. From that day, users of this'
  		    ' system will not be able to receive security updates, which could make their computers vulnerable to'
  		    ' cyber attacks.'
  		)
  		snake_case__  					=      (
  		    'орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7'
  		    ' после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи'
  		    ' этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми'
  		    ' к кибератакам.'
  		)
  		snake_case__  					=      (
  		    '根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。'
  		)
  
  		snake_case__  					=      tokenizer(
  		    [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]			,					padding=UpperCamelCase_			,					max_length=255			,					return_tensors='pt'		).input_ids
  		snake_case__  					=      input_ids.to(UpperCamelCase_		)
  
  		snake_case__  					=      model.generate(
  		    UpperCamelCase_			,					num_beams=10			,					length_penalty=1.0			,					no_repeat_ngram_size=3			,					early_stopping=UpperCamelCase_		)
  		snake_case__  					=      [tokenizer.decode(UpperCamelCase_			,					skip_special_tokens=UpperCamelCase_		) for g in summary_ids]
  		snake_case__  					=      'Microsoft to end Windows 7 free support after January 14, 2020'
  		snake_case__  					=      'Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года'
  		snake_case__  					=      '微软打算终止对Windows 7操作系统的免费支持'
  		self.assertListEqual(
  		    [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]			,					UpperCamelCase_			,					)
  
  		snake_case__  					=      model.generate(
  		    UpperCamelCase_			,					num_beams=1			,					length_penalty=1.0			,					no_repeat_ngram_size=3			,					early_stopping=UpperCamelCase_		)
  		snake_case__  					=      [
  		    tokenizer.convert_ids_to_tokens(UpperCamelCase_			,					skip_special_tokens=UpperCamelCase_		) for g in summary_ids_beama
  		]
  		snake_case__  					=      '▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7'.split(' '		)
  		snake_case__  					=      '▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года'.split(
  		    ' '		)
  		snake_case__  					=      '微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持'.split(' '		)
  		self.assertListEqual(
  		    [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]			,					UpperCamelCase_			,					)
  
  
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  
  
  import shutil
  import tempfile
  import unittest
  
  import pytest
  
  from transformers import WhisperTokenizer, is_speech_available
  from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio
  
  from .test_feature_extraction_whisper import floats_list
  
  
  if is_speech_available():
  			from transformers import WhisperFeatureExtractor, WhisperProcessor
  
  
  a__				: Tuple									=    5_0358
  a__				: str									=    5_0362
  
  
  
  
  
  
  @require_torch
  @require_torchaudio
  @require_sentencepiece
  class 							__snake_case    (				unittest.TestCase       ):
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      'openai/whisper-small.en'
  		snake_case__  					=      tempfile.mkdtemp()
  
  
  
  
  
  	def      _snake_case      (		self			,					**UpperCamelCase_		)			->		Optional[int]:
  		return WhisperTokenizer.from_pretrained(self.checkpoint			,					**UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self			,					**UpperCamelCase_		)			->		Any:
  		return WhisperFeatureExtractor.from_pretrained(self.checkpoint			,					**UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Tuple:
  		shutil.rmtree(self.tmpdirname		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Any:
  		snake_case__  					=      self.get_tokenizer()
  		snake_case__  					=      self.get_feature_extractor()
  
  		snake_case__  					=      WhisperProcessor(tokenizer=UpperCamelCase_			,					feature_extractor=UpperCamelCase_		)
  
  		processor.save_pretrained(self.tmpdirname		)
  		snake_case__  					=      WhisperProcessor.from_pretrained(self.tmpdirname		)
  
  		self.assertEqual(processor.tokenizer.get_vocab()			,					tokenizer.get_vocab()		)
  		self.assertIsInstance(processor.tokenizer			,					UpperCamelCase_		)
  
  		self.assertEqual(processor.feature_extractor.to_json_string()			,					feature_extractor.to_json_string()		)
  		self.assertIsInstance(processor.feature_extractor			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      WhisperProcessor(tokenizer=self.get_tokenizer()			,					feature_extractor=self.get_feature_extractor()		)
  		processor.save_pretrained(self.tmpdirname		)
  
  		snake_case__  					=      self.get_tokenizer(bos_token='(BOS)'			,					eos_token='(EOS)'		)
  		snake_case__  					=      self.get_feature_extractor(do_normalize=UpperCamelCase_			,					padding_value=1.0		)
  
  		snake_case__  					=      WhisperProcessor.from_pretrained(
  		    self.tmpdirname			,					bos_token='(BOS)'			,					eos_token='(EOS)'			,					do_normalize=UpperCamelCase_			,					padding_value=1.0		)
  
  		self.assertEqual(processor.tokenizer.get_vocab()			,					tokenizer_add_kwargs.get_vocab()		)
  		self.assertIsInstance(processor.tokenizer			,					UpperCamelCase_		)
  
  		self.assertEqual(processor.feature_extractor.to_json_string()			,					feature_extractor_add_kwargs.to_json_string()		)
  		self.assertIsInstance(processor.feature_extractor			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      self.get_feature_extractor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      WhisperProcessor(tokenizer=UpperCamelCase_			,					feature_extractor=UpperCamelCase_		)
  
  		snake_case__  					=      floats_list((3, 1000)		)
  
  		snake_case__  					=      feature_extractor(UpperCamelCase_			,					return_tensors='np'		)
  		snake_case__  					=      processor(UpperCamelCase_			,					return_tensors='np'		)
  
  		for key in input_feat_extract.keys():
  			self.assertAlmostEqual(input_feat_extract[key].sum()			,					input_processor[key].sum()			,					delta=1E-2		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Union[str, Any]:
  		snake_case__  					=      self.get_feature_extractor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      WhisperProcessor(tokenizer=UpperCamelCase_			,					feature_extractor=UpperCamelCase_		)
  
  		snake_case__  					=      'This is a test string'
  
  		snake_case__  					=      processor(text=UpperCamelCase_		)
  
  		snake_case__  					=      tokenizer(UpperCamelCase_		)
  
  		for key in encoded_tok.keys():
  			self.assertListEqual(encoded_tok[key]			,					encoded_processor[key]		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Tuple:
  		snake_case__  					=      self.get_feature_extractor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      WhisperProcessor(tokenizer=UpperCamelCase_			,					feature_extractor=UpperCamelCase_		)
  
  		snake_case__  					=      [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]
  
  		snake_case__  					=      processor.batch_decode(UpperCamelCase_		)
  		snake_case__  					=      tokenizer.batch_decode(UpperCamelCase_		)
  
  		self.assertListEqual(UpperCamelCase_			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      self.get_feature_extractor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      WhisperProcessor(tokenizer=UpperCamelCase_			,					feature_extractor=UpperCamelCase_		)
  
  		self.assertListEqual(
  		    processor.model_input_names			,					feature_extractor.model_input_names			,					msg='`processor` and `feature_extractor` model input names do not match'			,					)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      self.get_feature_extractor()
  		snake_case__  					=      self.get_tokenizer()
  
  		snake_case__  					=      WhisperProcessor(tokenizer=UpperCamelCase_			,					feature_extractor=UpperCamelCase_		)
  		snake_case__  					=      processor.get_decoder_prompt_ids(task='transcribe'			,					no_timestamps=UpperCamelCase_		)
  
  		self.assertIsInstance(UpperCamelCase_			,					UpperCamelCase_		)
  		for ids in forced_decoder_ids:
  			self.assertIsInstance(UpperCamelCase_			,					(list, tuple)		)
  
  		snake_case__  					=      [TRANSCRIBE, NOTIMESTAMPS]
  		self.assertListEqual([ids[-1] for ids in forced_decoder_ids]			,					UpperCamelCase_		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		List[str]:
  		snake_case__  					=      WhisperProcessor(tokenizer=self.get_tokenizer()			,					feature_extractor=self.get_feature_extractor()		)
  		snake_case__  					=      processor.get_prompt_ids('Mr. Quilter'		)
  		snake_case__  					=      processor.tokenizer.decode(UpperCamelCase_		)
  
  		self.assertListEqual(prompt_ids.tolist()			,					[5_0360, 1770, 13, 2264, 346, 353]		)
  		self.assertEqual(UpperCamelCase_			,					'<|startofprev|> Mr. Quilter'		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		Dict:
  		snake_case__  					=      WhisperProcessor(tokenizer=self.get_tokenizer()			,					feature_extractor=self.get_feature_extractor()		)
  		snake_case__  					=      processor.get_prompt_ids(''		)
  		snake_case__  					=      processor.tokenizer.decode(UpperCamelCase_		)
  
  		self.assertListEqual(prompt_ids.tolist()			,					[5_0360, 220]		)
  		self.assertEqual(UpperCamelCase_			,					'<|startofprev|> '		)
  
  
  
  
  
  	def      _snake_case      (		self		)			->		int:
  		snake_case__  					=      WhisperProcessor(tokenizer=self.get_tokenizer()			,					feature_extractor=self.get_feature_extractor()		)
  
  		def _test_prompt_error_raised_helper(UpperCamelCase_			,					UpperCamelCase_		):
  			with pytest.raises(UpperCamelCase_		) as excinfo:
  				processor.get_prompt_ids(UpperCamelCase_		)
  			snake_case__  					=      F'''Encountered text in the prompt corresponding to disallowed special token: {special_token}.'''
  			self.assertEqual(UpperCamelCase_			,					str(excinfo.value		)		)
  
  		_test_prompt_error_raised_helper('<|startofprev|> test'			,					'<|startofprev|>'		)
  		_test_prompt_error_raised_helper('test <|notimestamps|>'			,					'<|notimestamps|>'		)
  		_test_prompt_error_raised_helper('test <|zh|> test <|transcribe|>'			,					'<|zh|>'		)
  
  
  
  