



"""simple docstring"""







from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy






class   UpperCAmelCase__		(   UpperCamelCase  ):
		lowerCAmelCase_							:						Optional[int]							=  """MarkupLMFeatureExtractor"""
		lowerCAmelCase_							:						str							=  ("""MarkupLMTokenizer""", """MarkupLMTokenizerFast""")
		lowerCAmelCase_							:						List[str]							=  True



		def __call__(			self	:     Dict   ,				snake_case	:     Tuple=None   ,				snake_case	:     List[str]=None   ,				snake_case	:     List[str]=None   ,				snake_case	:     Dict=None   ,				snake_case	:     str=None   ,				snake_case	:     bool = True   ,				snake_case	:     Union[bool, str, PaddingStrategy] = False   ,				snake_case	:     Union[bool, str, TruncationStrategy] = None   ,				snake_case	:     Optional[int] = None   ,				snake_case	:     int = 0   ,				snake_case	:     Optional[int] = None   ,				snake_case	:     Optional[bool] = None   ,				snake_case	:     Optional[bool] = None   ,				snake_case	:     bool = False   ,				snake_case	:     bool = False   ,				snake_case	:     bool = False   ,				snake_case	:     bool = False   ,				snake_case	:     bool = True   ,				snake_case	:     Optional[Union[str, TensorType]] = None   ,				**snake_case	:     Tuple   ,				)      ->    BatchEncoding:






									'''simple docstring'''

									if self.parse_html:
																if html_strings is None:
																							raise ValueError('Make sure to pass HTML strings in case `parse_html` is set to `True`'					)

																if nodes is not None or xpaths is not None or node_labels is not None:
																							raise ValueError(
																							    'Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'					)

																A    =       self.feature_extractor(snake_case					)
																A    =       features['nodes']
																A    =       features['xpaths']
									else:
																if html_strings is not None:
																							raise ValueError('You have passed HTML strings but `parse_html` is set to `False`.'					)
																if nodes is None or xpaths is None:
																							raise ValueError('Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'					)

        # # second, apply the tokenizer
									if questions is not None and self.parse_html:
																if isinstance(snake_case   ,				snake_case					):
																							A    =       [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

									A    =       self.tokenizer(
									    text=questions if questions is not None else nodes   ,				text_pair=nodes if questions is not None else None   ,				xpaths=snake_case   ,				node_labels=snake_case   ,				add_special_tokens=snake_case   ,				padding=snake_case   ,				truncation=snake_case   ,				max_length=snake_case   ,				stride=snake_case   ,				pad_to_multiple_of=snake_case   ,				return_token_type_ids=snake_case   ,				return_attention_mask=snake_case   ,				return_overflowing_tokens=snake_case   ,				return_special_tokens_mask=snake_case   ,				return_offsets_mapping=snake_case   ,				return_length=snake_case   ,				verbose=snake_case   ,				return_tensors=snake_case   ,				**snake_case   ,				)

									return encoded_inputs



		def        A_		(			self	:     Tuple   ,				*snake_case	:     Optional[Any]   ,				**snake_case	:     str					)      ->    Dict:






									'''simple docstring'''

									return self.tokenizer.batch_decode(*snake_case   ,				**snake_case					)



		def        A_		(			self	:     str   ,				*snake_case	:     List[str]   ,				**snake_case	:     Dict					)      ->    Union[str, Any]:






									'''simple docstring'''

									return self.tokenizer.decode(*snake_case   ,				**snake_case					)



		@property
		def        A_		(			self	:     int					)      ->    Optional[Any]:






									'''simple docstring'''

									A    =       self.tokenizer.model_input_names
									return tokenizer_input_names








"""simple docstring"""







from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
							import tensorflow as tf

							from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel






@require_tf
class   UpperCAmelCase__		:
		lowerCAmelCase_							:						Optional[Any]							=  MarianConfig
		lowerCAmelCase_							:						Dict							=  {}
		lowerCAmelCase_							:						List[str]							=  """gelu"""



		def __init__(			self	:     Optional[Any]   ,				snake_case	:     Any   ,				snake_case	:     int=13   ,				snake_case	:     Dict=7   ,				snake_case	:     Dict=True   ,				snake_case	:     List[str]=False   ,				snake_case	:     int=99   ,				snake_case	:     Optional[Any]=32   ,				snake_case	:     Optional[Any]=2   ,				snake_case	:     int=4   ,				snake_case	:     Optional[int]=37   ,				snake_case	:     Optional[int]=0.1   ,				snake_case	:     str=0.1   ,				snake_case	:     Any=20   ,				snake_case	:     Dict=2   ,				snake_case	:     List[Any]=1   ,				snake_case	:     str=0   ,				)      ->    str:






									'''simple docstring'''

									A    =       parent
									A    =       batch_size
									A    =       seq_length
									A    =       is_training
									A    =       use_labels
									A    =       vocab_size
									A    =       hidden_size
									A    =       num_hidden_layers
									A    =       num_attention_heads
									A    =       intermediate_size

									A    =       hidden_dropout_prob
									A    =       attention_probs_dropout_prob
									A    =       max_position_embeddings
									A    =       eos_token_id
									A    =       pad_token_id
									A    =       bos_token_id



		def        A_		(			self	:     List[Any]					)      ->    Tuple:






									'''simple docstring'''

									A    =       ids_tensor([self.batch_size, self.seq_length - 1]   ,				self.vocab_size					)
									A    =       tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size					)   ,				1					)
									A    =       tf.concat([input_ids, eos_tensor]   ,				axis=1					)

									A    =       ids_tensor([self.batch_size, self.seq_length]   ,				self.vocab_size					)

									A    =       self.config_cls(
									    vocab_size=self.vocab_size   ,				d_model=self.hidden_size   ,				encoder_layers=self.num_hidden_layers   ,				decoder_layers=self.num_hidden_layers   ,				encoder_attention_heads=self.num_attention_heads   ,				decoder_attention_heads=self.num_attention_heads   ,				encoder_ffn_dim=self.intermediate_size   ,				decoder_ffn_dim=self.intermediate_size   ,				dropout=self.hidden_dropout_prob   ,				attention_dropout=self.attention_probs_dropout_prob   ,				max_position_embeddings=self.max_position_embeddings   ,				eos_token_ids=[2]   ,				bos_token_id=self.bos_token_id   ,				pad_token_id=self.pad_token_id   ,				decoder_start_token_id=self.pad_token_id   ,				**self.config_updates   ,				)
									A    =       prepare_marian_inputs_dict(snake_case   ,				snake_case   ,				snake_case					)
									return config, inputs_dict



		def        A_		(			self	:     Tuple   ,				snake_case	:     Optional[int]   ,				snake_case	:     Optional[int]					)      ->    Optional[int]:






									'''simple docstring'''

									A    =       TFMarianModel(config=snake_case					).get_decoder()
									A    =       inputs_dict['input_ids']

									A    =       input_ids[:1, :]
									A    =       inputs_dict['attention_mask'][:1, :]
									A    =       inputs_dict['head_mask']
									A    =       1

									# first forward pass
									A    =       model(snake_case   ,				attention_mask=snake_case   ,				head_mask=snake_case   ,				use_cache=snake_case					)

									A   ,	A    =       outputs.to_tuple()

									# create hypothetical next token and extent to next_input_ids
									A    =       ids_tensor((self.batch_size, 3)   ,				config.vocab_size					)
									A    =       tf.cast(ids_tensor((self.batch_size, 3)   ,				2					)   ,				tf.inta					)

									# append to next input_ids and
									A    =       tf.concat([input_ids, next_tokens]   ,				axis=-1					)
									A    =       tf.concat([attention_mask, next_attn_mask]   ,				axis=-1					)

									A    =       model(snake_case   ,				attention_mask=snake_case					)[0]
									A    =       model(snake_case   ,				attention_mask=snake_case   ,				past_key_values=snake_case					)[0]

									self.parent.assertEqual(next_tokens.shape[1]   ,				output_from_past.shape[1]					)

									# select random slice
									A    =       int(ids_tensor((1,)   ,				output_from_past.shape[-1]					)					)
									A    =       output_from_no_past[:, -3:, random_slice_idx]
									A    =       output_from_past[:, :, random_slice_idx]

									# test that outputs are equal for slice
									tf.debugging.assert_near(snake_case   ,				snake_case   ,				rtol=1E-3					)



def 	lowerCAmelCase__      (  lowerCamelCase__					,   lowerCamelCase__					,   lowerCamelCase__					,   lowerCamelCase__=None					,   lowerCamelCase__=None					,   lowerCamelCase__=None					,   lowerCamelCase__=None					,   lowerCamelCase__=None					,   )			->				Tuple:
							if attention_mask is None:
														A    =       tf.cast(tf.math.not_equal(lowerCamelCase__					,   config.pad_token_id      )					,   tf.inta      )
							if decoder_attention_mask is None:
														A    =       tf.concat(
														    [
														        tf.ones(decoder_input_ids[:, :1].shape					,   dtype=tf.inta      ),
														        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]					,   config.pad_token_id      )					,   tf.inta      ),
														    ]					,   axis=-1					,   )
							if head_mask is None:
														A    =       tf.ones((config.encoder_layers, config.encoder_attention_heads)      )
							if decoder_head_mask is None:
														A    =       tf.ones((config.decoder_layers, config.decoder_attention_heads)      )
							if cross_attn_head_mask is None:
														A    =       tf.ones((config.decoder_layers, config.decoder_attention_heads)      )
							return {
							    "input_ids": input_ids,
							    "decoder_input_ids": decoder_input_ids,
							    "attention_mask": attention_mask,
							    "decoder_attention_mask": decoder_attention_mask,
							    "head_mask": head_mask,
							    "decoder_head_mask": decoder_head_mask,
							    "cross_attn_head_mask": cross_attn_head_mask,
							}






@require_tf
class   UpperCAmelCase__		(   UpperCamelCase ,UpperCamelCase ,unittest.TestCase  ):
		lowerCAmelCase_							:						List[str]							=  (TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
		lowerCAmelCase_							:						List[Any]							=  (TFMarianMTModel,) if is_tf_available() else ()
		lowerCAmelCase_							:						str							=  (
		    {
		        """conversational""": TFMarianMTModel,
		        """feature-extraction""": TFMarianModel,
		        """summarization""": TFMarianMTModel,
		        """text2text-generation""": TFMarianMTModel,
		        """translation""": TFMarianMTModel,
		    }
		    if is_tf_available()
		    else {}
		)
		lowerCAmelCase_							:						Dict							=  True
		lowerCAmelCase_							:						Any							=  False
		lowerCAmelCase_							:						int							=  False



		def        A_		(			self	:     str					)      ->    Optional[int]:






									'''simple docstring'''

									A    =       TFMarianModelTester(self					)
									A    =       ConfigTester(self   ,				config_class=snake_case					)



		def        A_		(			self	:     int					)      ->    Optional[int]:






									'''simple docstring'''

									self.config_tester.run_common_tests()



		def        A_		(			self	:     Dict					)      ->    int:






									'''simple docstring'''

									A    =       self.model_tester.prepare_config_and_inputs_for_common()
									self.model_tester.check_decoder_model_past_large_inputs(*snake_case					)






@require_tf
class   UpperCAmelCase__		(   unittest.TestCase  ):
		lowerCAmelCase_							:						str							=  10_00  # show more chars for failing integration tests



		@classmethod
		def        A_		(			cls	:     str					)      ->    None:






									'''simple docstring'''

									A    =       f"""Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"""
									return cls



		@cached_property
		def        A_		(			self	:     Tuple					)      ->    MarianTokenizer:






									'''simple docstring'''

									return AutoTokenizer.from_pretrained(self.model_name					)



		@property
		def        A_		(			self	:     str					)      ->    int:






									'''simple docstring'''

									return self.tokenizer.eos_token_id



		@cached_property
		def        A_		(			self	:     List[Any]					)      ->    Optional[int]:






									'''simple docstring'''

									warnings.simplefilter('error'					)
									A    =       TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name					)
									assert isinstance(snake_case   ,				snake_case					)
									A    =       model.config
									self.assertListEqual(c.bad_words_ids   ,				[[c.pad_token_id]]					)
									self.assertEqual(c.max_length   ,				512					)
									self.assertEqual(c.decoder_start_token_id   ,				c.pad_token_id					)
									return model



		def        A_		(			self	:     List[Any]   ,				**snake_case	:     Optional[int]					)      ->    Any:






									'''simple docstring'''

									A    =       self.translate_src_text(**snake_case					)
									self.assertListEqual(self.expected_text   ,				snake_case					)



		def        A_		(			self	:     Optional[Any]   ,				**snake_case	:     int					)      ->    str:






									'''simple docstring'''

									A    =       self.tokenizer(self.src_text   ,				**snake_case   ,				padding=snake_case   ,				return_tensors='tf'					)
									A    =       self.model.generate(
									    model_inputs.input_ids   ,				attention_mask=model_inputs.attention_mask   ,				num_beams=2   ,				max_length=128					)
									A    =       self.tokenizer.batch_decode(generated_ids.numpy()   ,				skip_special_tokens=snake_case					)
									return generated_words






@require_sentencepiece
@require_tokenizers
@require_tf
class   UpperCAmelCase__		(   UpperCamelCase  ):
		lowerCAmelCase_							:						Dict							=  """mt"""
		lowerCAmelCase_							:						int							=  """en"""
		lowerCAmelCase_							:						Optional[int]							=  ["""Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."""]
		lowerCAmelCase_							:						List[Any]							=  ["""Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."""]



		@unittest.skip('Skipping until #12647 is resolved.'					)
		@slow
		def        A_		(			self	:     List[str]					)      ->    List[Any]:






									'''simple docstring'''

									self._assert_generated_batch_equal_expected()






@require_sentencepiece
@require_tokenizers
@require_tf
class   UpperCAmelCase__		(   UpperCamelCase  ):
		lowerCAmelCase_							:						List[str]							=  """en"""
		lowerCAmelCase_							:						Dict							=  """zh"""
		lowerCAmelCase_							:						Any							=  ["""My name is Wolfgang and I live in Berlin"""]
		lowerCAmelCase_							:						int							=  ["""我叫沃尔夫冈 我住在柏林"""]



		@unittest.skip('Skipping until #12647 is resolved.'					)
		@slow
		def        A_		(			self	:     Tuple					)      ->    List[str]:






									'''simple docstring'''

									self._assert_generated_batch_equal_expected()






@require_sentencepiece
@require_tokenizers
@require_tf
class   UpperCAmelCase__		(   UpperCamelCase  ):
		lowerCAmelCase_							:						str							=  """en"""
		lowerCAmelCase_							:						Dict							=  """ROMANCE"""
		lowerCAmelCase_							:						Optional[int]							=  [
		    """>>fr<< Don't spend so much time watching TV.""",
		    """>>pt<< Your message has been sent.""",
		    """>>es<< He's two years older than me.""",
		]
		lowerCAmelCase_							:						Union[str, Any]							=  [
		    """Ne passez pas autant de temps à regarder la télé.""",
		    """A sua mensagem foi enviada.""",
		    """Es dos años más viejo que yo.""",
		]



		@unittest.skip('Skipping until #12647 is resolved.'					)
		@slow
		def        A_		(			self	:     Optional[int]					)      ->    List[Any]:






									'''simple docstring'''

									self._assert_generated_batch_equal_expected()



		@unittest.skip('Skipping until #12647 is resolved.'					)
		@slow
		def        A_		(			self	:     Tuple					)      ->    List[str]:






									'''simple docstring'''

									A    =       TranslationPipeline(self.model   ,				self.tokenizer   ,				framework='tf'					)
									A    =       pipeline(self.src_text					)
									self.assertEqual(self.expected_text   ,				[x['translation_text'] for x in output]					)



