code: 
  
  
  
  """simple docstring"""
  from __future__ import annotations
  
  import unittest
  import warnings
  
  from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
  from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
  from transformers.utils import cached_property
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
  		import tensorflow as tf
  
  		from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel
  @require_tf
  class 							_UpperCAmelCase							:
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					a__             =MarianConfig
  					a__             ={}
  					a__             ='gelu'
  
  
  
  
  
  					def __init__(      self ,			A ,			A=1_3 ,			A=7 ,			A=True ,			A=False ,			A=9_9 ,			A=3_2 ,			A=2 ,			A=4 ,			A=3_7 ,			A=0.1 ,			A=0.1 ,			A=2_0 ,			A=2 ,			A=1 ,			A=0 ,			)							->      Dict:
  									_UpperCAmelCase							:						int       							=					parent
  									_UpperCAmelCase							:						int       							=					batch_size
  									_UpperCAmelCase							:						str       							=					seq_length
  									_UpperCAmelCase							:						Tuple       							=					is_training
  									_UpperCAmelCase							:						List[Any]       							=					use_labels
  									_UpperCAmelCase							:						Tuple       							=					vocab_size
  									_UpperCAmelCase							:						int       							=					hidden_size
  									_UpperCAmelCase							:						List[str]       							=					num_hidden_layers
  									_UpperCAmelCase							:						Tuple       							=					num_attention_heads
  									_UpperCAmelCase							:						Any       							=					intermediate_size
  
  									_UpperCAmelCase							:						List[str]       							=					hidden_dropout_prob
  									_UpperCAmelCase							:						Optional[int]       							=					attention_probs_dropout_prob
  									_UpperCAmelCase							:						int       							=					max_position_embeddings
  									_UpperCAmelCase							:						List[str]       							=					eos_token_id
  									_UpperCAmelCase							:						Optional[int]       							=					pad_token_id
  									_UpperCAmelCase							:						Any       							=					bos_token_id
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self	)							->      Any:
  									_UpperCAmelCase							:						Optional[int]       							=					ids_tensor([self.batch_size, self.seq_length - 1] ,			self.vocab_size	)
  									_UpperCAmelCase							:						List[Any]       							=					tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size	) ,			1	)
  									_UpperCAmelCase							:						Any       							=					tf.concat([input_ids, eos_tensor] ,			axis=1	)
  
  									_UpperCAmelCase							:						Dict       							=					ids_tensor([self.batch_size, self.seq_length] ,			self.vocab_size	)
  
  									_UpperCAmelCase							:						List[Any]       							=					self.config_cls(
  									    vocab_size=self.vocab_size ,			d_model=self.hidden_size ,			encoder_layers=self.num_hidden_layers ,			decoder_layers=self.num_hidden_layers ,			encoder_attention_heads=self.num_attention_heads ,			decoder_attention_heads=self.num_attention_heads ,			encoder_ffn_dim=self.intermediate_size ,			decoder_ffn_dim=self.intermediate_size ,			dropout=self.hidden_dropout_prob ,			attention_dropout=self.attention_probs_dropout_prob ,			max_position_embeddings=self.max_position_embeddings ,			eos_token_ids=[2] ,			bos_token_id=self.bos_token_id ,			pad_token_id=self.pad_token_id ,			decoder_start_token_id=self.pad_token_id ,			**self.config_updates ,			)
  									_UpperCAmelCase							:						Optional[Any]       							=					prepare_marian_inputs_dict(A ,			A ,			A	)
  									return config, inputs_dict
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self ,			A ,			A	)							->      Union[str, Any]:
  									_UpperCAmelCase							:						Optional[Any]       							=					TFMarianModel(config=A	).get_decoder()
  									_UpperCAmelCase							:						Any       							=					inputs_dict['input_ids']
  
  									_UpperCAmelCase							:						Tuple       							=					input_ids[:1, :]
  									_UpperCAmelCase							:						Any       							=					inputs_dict['attention_mask'][:1, :]
  									_UpperCAmelCase							:						Optional[Any]       							=					inputs_dict['head_mask']
  									_UpperCAmelCase							:						List[Any]       							=					1
  
  									# first forward pass
  									_UpperCAmelCase							:						Optional[int]       							=					model(A ,			attention_mask=A ,			head_mask=A ,			use_cache=A	)
  
  									_UpperCAmelCase							:						int       							=					outputs.to_tuple()
  
  									# create hypothetical next token and extent to next_input_ids
  									_UpperCAmelCase							:						List[Any]       							=					ids_tensor((self.batch_size, 3) ,			config.vocab_size	)
  									_UpperCAmelCase							:						Any       							=					tf.cast(ids_tensor((self.batch_size, 3) ,			2	) ,			tf.inta	)
  
  									# append to next input_ids and
  									_UpperCAmelCase							:						str       							=					tf.concat([input_ids, next_tokens] ,			axis=-1	)
  									_UpperCAmelCase							:						Union[str, Any]       							=					tf.concat([attention_mask, next_attn_mask] ,			axis=-1	)
  
  									_UpperCAmelCase							:						Optional[Any]       							=					model(A ,			attention_mask=A	)[0]
  									_UpperCAmelCase							:						Union[str, Any]       							=					model(A ,			attention_mask=A ,			past_key_values=A	)[0]
  
  									self.parent.assertEqual(next_tokens.shape[1] ,			output_from_past.shape[1]	)
  
  									# select random slice
  									_UpperCAmelCase							:						List[str]       							=					int(ids_tensor((1,) ,			output_from_past.shape[-1]	)	)
  									_UpperCAmelCase							:						Any       							=					output_from_no_past[:, -3:, random_slice_idx]
  									_UpperCAmelCase							:						Any       							=					output_from_past[:, :, random_slice_idx]
  
  									# test that outputs are equal for slice
  									tf.debugging.assert_near(A ,			A ,			rtol=1E-3	)
  
  
  
  
  
  def 				lowerCamelCase_		(UpperCamelCase__	:	str  , UpperCamelCase__	:	List[Any]  , UpperCamelCase__	:	Tuple  , UpperCamelCase__	:	List[str]=None  , UpperCamelCase__	:	int=None  , UpperCamelCase__	:	Union[str, Any]=None  , UpperCamelCase__	:	Union[str, Any]=None  , UpperCamelCase__	:	List[Any]=None  , ):
  				if attention_mask is None:
  								_UpperCAmelCase							:						Any       							=					tf.cast(tf.math.not_equal(lowerCAmelCase__  , config.pad_token_id							)  , tf.inta							)
  				if decoder_attention_mask is None:
  								_UpperCAmelCase							:						Union[str, Any]       							=					tf.concat(
  								    [
  								        tf.ones(decoder_input_ids[:, :1].shape  , dtype=tf.inta							),
  								        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]  , config.pad_token_id							)  , tf.inta							),
  								    ]  , axis=-1  , )
  				if head_mask is None:
  								_UpperCAmelCase							:						Optional[int]       							=					tf.ones((config.encoder_layers, config.encoder_attention_heads)							)
  				if decoder_head_mask is None:
  								_UpperCAmelCase							:						List[Any]       							=					tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
  				if cross_attn_head_mask is None:
  								_UpperCAmelCase							:						List[str]       							=					tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
  				return {
  				    "input_ids": input_ids,
  				    "decoder_input_ids": decoder_input_ids,
  				    "attention_mask": attention_mask,
  				    "decoder_attention_mask": decoder_attention_mask,
  				    "head_mask": head_mask,
  				    "decoder_head_mask": decoder_head_mask,
  				    "cross_attn_head_mask": cross_attn_head_mask,
  				}
  @require_tf
  class 							_UpperCAmelCase							(				snake_case__	,snake_case__	,unittest.TestCase   ):
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					a__             =(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
  					a__             =(TFMarianMTModel,) if is_tf_available() else ()
  					a__             =(
  					    {
  					        'conversational': TFMarianMTModel,
  					        'feature-extraction': TFMarianModel,
  					        'summarization': TFMarianMTModel,
  					        'text2text-generation': TFMarianMTModel,
  					        'translation': TFMarianMTModel,
  					    }
  					    if is_tf_available()
  					    else {}
  					)
  					a__             =True
  					a__             =False
  					a__             =False
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self	)							->      Union[str, Any]:
  									_UpperCAmelCase							:						str       							=					TFMarianModelTester(self	)
  									_UpperCAmelCase							:						Tuple       							=					ConfigTester(self ,			config_class=A	)
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self	)							->      List[Any]:
  									self.config_tester.run_common_tests()
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self	)							->      str:
  									_UpperCAmelCase							:						int       							=					self.model_tester.prepare_config_and_inputs_for_common()
  									self.model_tester.check_decoder_model_past_large_inputs(*A	)
  
  
  
  @require_tf
  class 							_UpperCAmelCase							(				unittest.TestCase   ):
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					a__             =1_0_0_0  # show more chars for failing integration tests
  
  
  
  
  
  					@classmethod
  					def 				__lowerCAmelCase			(      cls	)							->      List[str]:
  									_UpperCAmelCase							:						Optional[Any]       							=					f'Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}'
  									return cls
  
  
  
  
  
  					@cached_property
  					def 				__lowerCAmelCase			(      self	)							->      Union[str, Any]:
  									return AutoTokenizer.from_pretrained(self.model_name	)
  
  
  
  
  
  					@property
  					def 				__lowerCAmelCase			(      self	)							->      Optional[int]:
  									return self.tokenizer.eos_token_id
  
  
  
  
  
  					@cached_property
  					def 				__lowerCAmelCase			(      self	)							->      str:
  									warnings.simplefilter('''error'''	)
  									_UpperCAmelCase							:						TFMarianMTModel       							=					TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name	)
  									assert isinstance(A ,			A	)
  									_UpperCAmelCase							:						List[Any]       							=					model.config
  									self.assertListEqual(c.bad_words_ids ,			[[c.pad_token_id]]	)
  									self.assertEqual(c.max_length ,			5_1_2	)
  									self.assertEqual(c.decoder_start_token_id ,			c.pad_token_id	)
  									return model
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self ,			**A	)							->      str:
  									_UpperCAmelCase							:						List[str]       							=					self.translate_src_text(**A	)
  									self.assertListEqual(self.expected_text ,			A	)
  
  
  
  
  
  					def 				__lowerCAmelCase			(      self ,			**A	)							->      List[Any]:
  									_UpperCAmelCase							:						str       							=					self.tokenizer(self.src_text ,			**A ,			padding=A ,			return_tensors='''tf'''	)
  									_UpperCAmelCase							:						Dict       							=					self.model.generate(
  									    model_inputs.input_ids ,			attention_mask=model_inputs.attention_mask ,			num_beams=2 ,			max_length=1_2_8	)
  									_UpperCAmelCase							:						Dict       							=					self.tokenizer.batch_decode(generated_ids.numpy() ,			skip_special_tokens=A	)
  									return generated_words
  
  
  
  @require_sentencepiece
  @require_tokenizers
  @require_tf
  class 							_UpperCAmelCase							(				snake_case__   ):
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					a__             ='mt'
  					a__             ='en'
  					a__             =['Billi messu b\'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem.']
  					a__             =['Touching gently, Jesus healed a man who was affected by the sad disease of leprosy.']
  
  
  
  
  
  					@unittest.skip('''Skipping until #12647 is resolved.'''	)
  					@slow
  					def 				__lowerCAmelCase			(      self	)							->      Union[str, Any]:
  									self._assert_generated_batch_equal_expected()
  
  
  
  @require_sentencepiece
  @require_tokenizers
  @require_tf
  class 							_UpperCAmelCase							(				snake_case__   ):
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					a__             ='en'
  					a__             ='zh'
  					a__             =['My name is Wolfgang and I live in Berlin']
  					a__             =['我叫沃尔夫冈 我住在柏林']
  
  
  
  
  
  					@unittest.skip('''Skipping until #12647 is resolved.'''	)
  					@slow
  					def 				__lowerCAmelCase			(      self	)							->      Union[str, Any]:
  									self._assert_generated_batch_equal_expected()
  
  
  
  @require_sentencepiece
  @require_tokenizers
  @require_tf
  class 							_UpperCAmelCase							(				snake_case__   ):
  
  
  
  
  
  					'''simple docstring'''
  
  
  
  					a__             ='en'
  					a__             ='ROMANCE'
  					a__             =[
  					    '>>fr<< Don\'t spend so much time watching TV.',
  					    '>>pt<< Your message has been sent.',
  					    '>>es<< He\'s two years older than me.',
  					]
  					a__             =[
  					    'Ne passez pas autant de temps à regarder la télé.',
  					    'A sua mensagem foi enviada.',
  					    'Es dos años más viejo que yo.',
  					]
  
  
  
  
  
  					@unittest.skip('''Skipping until #12647 is resolved.'''	)
  					@slow
  					def 				__lowerCAmelCase			(      self	)							->      Tuple:
  									self._assert_generated_batch_equal_expected()
  
  
  
  
  
  					@unittest.skip('''Skipping until #12647 is resolved.'''	)
  					@slow
  					def 				__lowerCAmelCase			(      self	)							->      Union[str, Any]:
  									_UpperCAmelCase							:						Union[str, Any]       							=					TranslationPipeline(self.model ,			self.tokenizer ,			framework='''tf'''	)
  									_UpperCAmelCase							:						List[Any]       							=					pipeline(self.src_text	)
  									self.assertEqual(self.expected_text ,			[x['''translation_text'''] for x in output]	)
  
  
  
style_context: 
  
  
  
  """simple docstring"""
  import os
  from typing import List, Union
  
  import tensorflow as tf
  from tensorflow_text import BertTokenizer as BertTokenizerLayer
  from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs
  
  from .tokenization_bert import BertTokenizer
  class 							_UpperCAmelCase							(				tf.keras.layers.Layer   ):
  
  
  
  
  
       '''simple docstring'''
  
  
  
       def __init__(      self ,			A ,			A ,			A = None ,			A = None ,			A = None ,			A = "longest" ,			A = True ,			A = 5_1_2 ,			A = None ,			A = True ,			A = True ,			A = True ,			**A ,			)							->      Tuple:
           super().__init__()
           if use_fast_bert_tokenizer:
               _UpperCAmelCase							:						Union[str, Any]       							=					FastBertTokenizer(
                   A ,			token_out_type=tf.intaa ,			lower_case_nfd_strip_accents=A ,			**A	)
           else:
               _UpperCAmelCase							:						List[Any]       							=					tf.lookup.StaticVocabularyTable(
                   tf.lookup.KeyValueTensorInitializer(
                       keys=A ,			key_dtype=tf.string ,			values=tf.range(tf.size(A ,			out_type=tf.intaa	) ,			dtype=tf.intaa	) ,			value_dtype=tf.intaa ,			) ,			num_oov_buckets=1 ,			)
               _UpperCAmelCase							:						str       							=					BertTokenizerLayer(
                   A ,			token_out_type=tf.intaa ,			lower_case=A ,			**A	)
  
           _UpperCAmelCase							:						int       							=					vocab_list
           _UpperCAmelCase							:						Dict       							=					do_lower_case
           _UpperCAmelCase							:						Optional[int]       							=					cls_token_id or vocab_list.index('''[CLS]'''	)
           _UpperCAmelCase							:						Any       							=					sep_token_id or vocab_list.index('''[SEP]'''	)
           _UpperCAmelCase							:						List[Any]       							=					pad_token_id or vocab_list.index('''[PAD]'''	)
           _UpperCAmelCase							:						List[str]       							=					ShrinkLongestTrimmer(max_length - 3 ,			axis=1	)  # Allow room for special tokens
           _UpperCAmelCase							:						Tuple       							=					max_length
           _UpperCAmelCase							:						List[Any]       							=					padding
           _UpperCAmelCase							:						List[Any]       							=					truncation
           _UpperCAmelCase							:						Union[str, Any]       							=					pad_to_multiple_of
           _UpperCAmelCase							:						Union[str, Any]       							=					return_token_type_ids
           _UpperCAmelCase							:						Union[str, Any]       							=					return_attention_mask
  
  
  
  
  
       @classmethod
       def 				__lowerCAmelCase			(      cls ,			A ,			**A	)							->      Dict:  # noqa: F821
           _UpperCAmelCase							:						Optional[int]       							=					kwargs.pop('''do_lower_case''' ,			A	)
           _UpperCAmelCase							:						int       							=					tokenizer.do_lower_case if do_lower_case is None else do_lower_case
           _UpperCAmelCase							:						Optional[Any]       							=					kwargs.pop('''cls_token_id''' ,			A	)
           _UpperCAmelCase							:						str       							=					tokenizer.cls_token_id if cls_token_id is None else cls_token_id
           _UpperCAmelCase							:						str       							=					kwargs.pop('''sep_token_id''' ,			A	)
           _UpperCAmelCase							:						int       							=					tokenizer.sep_token_id if sep_token_id is None else sep_token_id
           _UpperCAmelCase							:						List[Any]       							=					kwargs.pop('''pad_token_id''' ,			A	)
           _UpperCAmelCase							:						Optional[Any]       							=					tokenizer.pad_token_id if pad_token_id is None else pad_token_id
  
           _UpperCAmelCase							:						int       							=					tokenizer.get_vocab()
           _UpperCAmelCase							:						List[str]       							=					sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()] ,			key=lambda A	: x[1]	)
           _UpperCAmelCase							:						Dict       							=					[entry[0] for entry in vocab]
           return cls(
               vocab_list=A ,			do_lower_case=A ,			cls_token_id=A ,			sep_token_id=A ,			pad_token_id=A ,			**A ,			)
  
  
  
  
  
       @classmethod
       def 				__lowerCAmelCase			(      cls ,			A ,			*A ,			**A	)							->      int:
           try:
               _UpperCAmelCase							:						int       							=					BertTokenizer.from_pretrained(A ,			*A ,			**A	)
           except:  # noqa: E722
               from .tokenization_bert_fast import BertTokenizerFast
  
               _UpperCAmelCase							:						Dict       							=					BertTokenizerFast.from_pretrained(A ,			*A ,			**A	)
           return cls.from_tokenizer(A ,			**A	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A	)							->      Any:
           if self.do_lower_case:
               _UpperCAmelCase							:						int       							=					case_fold_utfa(A	)
           _UpperCAmelCase							:						str       							=					self.tf_tokenizer.tokenize(A	)
           return tokens.merge_dims(1 ,			-1	)
  
  
  
  
  
       def 				__lowerCAmelCase			(      self ,			A ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			A=None ,			)							->      Optional[Any]:
           if padding is None:
               _UpperCAmelCase							:						str       							=					self.padding
           if padding not in ("longest", "max_length"):
               raise ValueError('''Padding must be either \'longest\' or \'max_length\'!'''	)
           if max_length is not None and text_pair is not None:
               # Because we have to instantiate a Trimmer to do it properly
               raise ValueError('''max_length cannot be overridden at call time when truncating paired texts!'''	)
           if max_length is None:
               _UpperCAmelCase							:						Union[str, Any]       							=					self.max_length
           if truncation is None:
               _UpperCAmelCase							:						List[Any]       							=					self.truncation
           if pad_to_multiple_of is None:
               _UpperCAmelCase							:						List[Any]       							=					self.pad_to_multiple_of
           if return_token_type_ids is None:
               _UpperCAmelCase							:						List[str]       							=					self.return_token_type_ids
           if return_attention_mask is None:
               _UpperCAmelCase							:						Any       							=					self.return_attention_mask
           if not isinstance(A ,			tf.Tensor	):
               _UpperCAmelCase							:						Dict       							=					tf.convert_to_tensor(A	)
           if text_pair is not None and not isinstance(A ,			tf.Tensor	):
               _UpperCAmelCase							:						str       							=					tf.convert_to_tensor(A	)
           if text_pair is not None:
               if text.shape.rank > 1:
                   raise ValueError('''text argument should not be multidimensional when a text pair is supplied!'''	)
               if text_pair.shape.rank > 1:
                   raise ValueError('''text_pair should not be multidimensional!'''	)
           if text.shape.rank == 2:
               _UpperCAmelCase							,       _UpperCAmelCase							:						str       							=					text[:, 0], text[:, 1]
           _UpperCAmelCase							:						Optional[Any]       							=					self.unpaired_tokenize(A	)
           if text_pair is None:  # Unpaired text
               if truncation:
                   _UpperCAmelCase							:						Optional[int]       							=					text[:, : max_length - 2]  # Allow room for special tokens
               _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					combine_segments(
                   (text,) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
           else:  # Paired text
               _UpperCAmelCase							:						Tuple       							=					self.unpaired_tokenize(A	)
               if truncation:
                   _UpperCAmelCase							,       _UpperCAmelCase							:						Optional[int]       							=					self.paired_trimmer.trim([text, text_pair]	)
               _UpperCAmelCase							,       _UpperCAmelCase							:						Union[str, Any]       							=					combine_segments(
                   (text, text_pair) ,			start_of_sequence_id=self.cls_token_id ,			end_of_segment_id=self.sep_token_id	)
           if padding == "longest":
               _UpperCAmelCase							:						List[Any]       							=					input_ids.bounding_shape(axis=1	)
               if pad_to_multiple_of is not None:
                   # No ceiling division in tensorflow, so we negate floordiv instead
                   _UpperCAmelCase							:						Tuple       							=					pad_to_multiple_of * (-tf.math.floordiv(-pad_length ,			A	))
           else:
               _UpperCAmelCase							:						int       							=					max_length
  
           _UpperCAmelCase							,       _UpperCAmelCase							:						Any       							=					pad_model_inputs(A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
           _UpperCAmelCase							:						Dict       							=					{'''input_ids''': input_ids}
           if return_attention_mask:
               _UpperCAmelCase							:						Optional[int]       							=					attention_mask
           if return_token_type_ids:
               _UpperCAmelCase							,       _UpperCAmelCase							:						Tuple       							=					pad_model_inputs(
                   A ,			max_seq_length=A ,			pad_value=self.pad_token_id	)
               _UpperCAmelCase							:						int       							=					token_type_ids
           return output
  
  
  
  
  
       def 				__lowerCAmelCase			(      self	)							->      List[str]:
           return {
               "vocab_list": self.vocab_list,
               "do_lower_case": self.do_lower_case,
               "cls_token_id": self.cls_token_id,
               "sep_token_id": self.sep_token_id,
               "pad_token_id": self.pad_token_id,
           }
  
  
  