




from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel





@require_tf
class 						SCREAMING_SNAKE_CASE__		:
					lowercase_   :			List[Any]   =   MarianConfig
					lowercase_   :			Dict   =   {}
					lowercase_   :			Optional[int]   =   "gelu"
					def __init__(	self  :		int					,	__lowerCamelCase  :		List[str]					,	__lowerCamelCase  :		Optional[int]=13					,	__lowerCamelCase  :		List[str]=7					,	__lowerCamelCase  :		int=True					,	__lowerCamelCase  :		Optional[Any]=False					,	__lowerCamelCase  :		List[Any]=99					,	__lowerCamelCase  :		Optional[Any]=32					,	__lowerCamelCase  :		List[Any]=2					,	__lowerCamelCase  :		str=4					,	__lowerCamelCase  :		Dict=37					,	__lowerCamelCase  :		int=0.1					,	__lowerCamelCase  :		str=0.1					,	__lowerCamelCase  :		List[str]=20					,	__lowerCamelCase  :		str=2					,	__lowerCamelCase  :		Any=1					,	__lowerCamelCase  :		List[str]=0					,	):



							"""simple docstring"""
							lowerCAmelCase__								=			parent
							lowerCAmelCase__								=			batch_size
							lowerCAmelCase__								=			seq_length
							lowerCAmelCase__								=			is_training
							lowerCAmelCase__								=			use_labels
							lowerCAmelCase__								=			vocab_size
							lowerCAmelCase__								=			hidden_size
							lowerCAmelCase__								=			num_hidden_layers
							lowerCAmelCase__								=			num_attention_heads
							lowerCAmelCase__								=			intermediate_size

							lowerCAmelCase__								=			hidden_dropout_prob
							lowerCAmelCase__								=			attention_probs_dropout_prob
							lowerCAmelCase__								=			max_position_embeddings
							lowerCAmelCase__								=			eos_token_id
							lowerCAmelCase__								=			pad_token_id
							lowerCAmelCase__								=			bos_token_id
					def 					A__			(	self  :		Any	):



							"""simple docstring"""
							lowerCAmelCase__								=			ids_tensor([self.batch_size, self.seq_length - 1]					,	self.vocab_size	)
							lowerCAmelCase__								=			tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size	)					,	1	)
							lowerCAmelCase__								=			tf.concat([input_ids, eos_tensor]					,	axis=1	)

							lowerCAmelCase__								=			ids_tensor([self.batch_size, self.seq_length]					,	self.vocab_size	)

							lowerCAmelCase__								=			self.config_cls(
							    vocab_size=self.vocab_size					,	d_model=self.hidden_size					,	encoder_layers=self.num_hidden_layers					,	decoder_layers=self.num_hidden_layers					,	encoder_attention_heads=self.num_attention_heads					,	decoder_attention_heads=self.num_attention_heads					,	encoder_ffn_dim=self.intermediate_size					,	decoder_ffn_dim=self.intermediate_size					,	dropout=self.hidden_dropout_prob					,	attention_dropout=self.attention_probs_dropout_prob					,	max_position_embeddings=self.max_position_embeddings					,	eos_token_ids=[2]					,	bos_token_id=self.bos_token_id					,	pad_token_id=self.pad_token_id					,	decoder_start_token_id=self.pad_token_id					,	**self.config_updates					,	)
							lowerCAmelCase__								=			prepare_marian_inputs_dict(UpperCAmelCase_					,	UpperCAmelCase_					,	UpperCAmelCase_	)
							return config, inputs_dict



					def 					A__			(	self  :		Optional[int]					,	__lowerCamelCase  :		str					,	__lowerCamelCase  :		Optional[Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			TFMarianModel(config=UpperCAmelCase_	).get_decoder()
							lowerCAmelCase__								=			inputs_dict['input_ids']

							lowerCAmelCase__								=			input_ids[:1, :]
							lowerCAmelCase__								=			inputs_dict['attention_mask'][:1, :]
							lowerCAmelCase__								=			inputs_dict['head_mask']
							lowerCAmelCase__								=			1

							# first forward pass
							lowerCAmelCase__								=			model(UpperCAmelCase_					,	attention_mask=UpperCAmelCase_					,	head_mask=UpperCAmelCase_					,	use_cache=UpperCAmelCase_	)

							lowerCAmelCase__								=			outputs.to_tuple()

							# create hypothetical next token and extent to next_input_ids
							lowerCAmelCase__								=			ids_tensor((self.batch_size, 3)					,	config.vocab_size	)
							lowerCAmelCase__								=			tf.cast(ids_tensor((self.batch_size, 3)					,	2	)					,	tf.inta	)

							# append to next input_ids and
							lowerCAmelCase__								=			tf.concat([input_ids, next_tokens]					,	axis=-1	)
							lowerCAmelCase__								=			tf.concat([attention_mask, next_attn_mask]					,	axis=-1	)

							lowerCAmelCase__								=			model(UpperCAmelCase_					,	attention_mask=UpperCAmelCase_	)[0]
							lowerCAmelCase__								=			model(UpperCAmelCase_					,	attention_mask=UpperCAmelCase_					,	past_key_values=UpperCAmelCase_	)[0]

							self.parent.assertEqual(next_tokens.shape[1]					,	output_from_past.shape[1]	)

							# select random slice
							lowerCAmelCase__								=			int(ids_tensor((1,)					,	output_from_past.shape[-1]	)	)
							lowerCAmelCase__								=			output_from_no_past[:, -3:, random_slice_idx]
							lowerCAmelCase__								=			output_from_past[:, :, random_slice_idx]

							# test that outputs are equal for slice
							tf.debugging.assert_near(UpperCAmelCase_					,	UpperCAmelCase_					,	rtol=1e-3	)



def 		a_       (   __lowerCAmelCase      ,   __lowerCAmelCase      ,   __lowerCAmelCase      ,   __lowerCAmelCase=None      ,   __lowerCAmelCase=None      ,   __lowerCAmelCase=None      ,   __lowerCAmelCase=None      ,   __lowerCAmelCase=None      ,   ):






		if attention_mask is None:
				lowerCAmelCase__								=			tf.cast(tf.math.not_equal(_UpperCAmelCase      ,   config.pad_token_id							)      ,   tf.inta							)
		if decoder_attention_mask is None:
				lowerCAmelCase__								=			tf.concat(
				    [
				        tf.ones(decoder_input_ids[:, :1].shape      ,   dtype=tf.inta							),
				        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]      ,   config.pad_token_id							)      ,   tf.inta							),
				    ]      ,   axis=-1      ,   )
		if head_mask is None:
				lowerCAmelCase__								=			tf.ones((config.encoder_layers, config.encoder_attention_heads)							)
		if decoder_head_mask is None:
				lowerCAmelCase__								=			tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
		if cross_attn_head_mask is None:
				lowerCAmelCase__								=			tf.ones((config.decoder_layers, config.decoder_attention_heads)							)
		return {
		    "input_ids": input_ids,
		    "decoder_input_ids": decoder_input_ids,
		    "attention_mask": attention_mask,
		    "decoder_attention_mask": decoder_attention_mask,
		    "head_mask": head_mask,
		    "decoder_head_mask": decoder_head_mask,
		    "cross_attn_head_mask": cross_attn_head_mask,
		}





@require_tf
class 						SCREAMING_SNAKE_CASE__		(_a						,				_a						,				unittest.TestCase ):
					lowercase_   :			Any   =   (TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
					lowercase_   :			Optional[Any]   =   (TFMarianMTModel,) if is_tf_available() else ()
					lowercase_   :			Union[str, Any]   =   (
					    {
					        "conversational": TFMarianMTModel,
					        "feature-extraction": TFMarianModel,
					        "summarization": TFMarianMTModel,
					        "text2text-generation": TFMarianMTModel,
					        "translation": TFMarianMTModel,
					    }
					    if is_tf_available()
					    else {}
					)
					lowercase_   :			int   =   True
					lowercase_   :			Union[str, Any]   =   False
					lowercase_   :			List[Any]   =   False
					def 					A__			(	self  :		str	):



							"""simple docstring"""
							lowerCAmelCase__								=			TFMarianModelTester(self	)
							lowerCAmelCase__								=			ConfigTester(self					,	config_class=UpperCAmelCase_	)
					def 					A__			(	self  :		int	):



							"""simple docstring"""
							self.config_tester.run_common_tests()



					def 					A__			(	self  :		List[str]	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.model_tester.prepare_config_and_inputs_for_common()
							self.model_tester.check_decoder_model_past_large_inputs(*UpperCAmelCase_	)





@require_tf
class 						SCREAMING_SNAKE_CASE__		(unittest.TestCase ):
					lowercase_   :			Optional[int]   =   1000  # show more chars for failing integration tests
					@classmethod
					def 					A__			(	cls  :		Dict	):



							"""simple docstring"""
							lowerCAmelCase__								=			F"""Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"""
							return cls
					@cached_property
					def 					A__			(	self  :		List[str]	):



							"""simple docstring"""
							return AutoTokenizer.from_pretrained(self.model_name	)
					@property
					def 					A__			(	self  :		int	):



							"""simple docstring"""
							return self.tokenizer.eos_token_id
					@cached_property
					def 					A__			(	self  :		Optional[int]	):



							"""simple docstring"""
							warnings.simplefilter('''error'''	)
							lowerCAmelCase__								=			TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name	)
							assert isinstance(UpperCAmelCase_					,	UpperCAmelCase_	)
							lowerCAmelCase__								=			model.config
							self.assertListEqual(c.bad_words_ids					,	[[c.pad_token_id]]	)
							self.assertEqual(c.max_length					,	5_12	)
							self.assertEqual(c.decoder_start_token_id					,	c.pad_token_id	)
							return model
					def 					A__			(	self  :		int					,	**__lowerCamelCase  :		Optional[Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.translate_src_text(**UpperCAmelCase_	)
							self.assertListEqual(self.expected_text					,	UpperCAmelCase_	)



					def 					A__			(	self  :		List[str]					,	**__lowerCamelCase  :		Optional[Any]	):



							"""simple docstring"""
							lowerCAmelCase__								=			self.tokenizer(self.src_text					,	**UpperCAmelCase_					,	padding=UpperCAmelCase_					,	return_tensors='''tf'''	)
							lowerCAmelCase__								=			self.model.generate(
							    model_inputs.input_ids					,	attention_mask=model_inputs.attention_mask					,	num_beams=2					,	max_length=1_28	)
							lowerCAmelCase__								=			self.tokenizer.batch_decode(generated_ids.numpy()					,	skip_special_tokens=UpperCAmelCase_	)
							return generated_words





@require_sentencepiece
@require_tokenizers
@require_tf
class 						SCREAMING_SNAKE_CASE__		(_a ):
					lowercase_   :			Optional[Any]   =   "mt"
					lowercase_   :			Dict   =   "en"
					lowercase_   :			Union[str, Any]   =   ["Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."]
					lowercase_   :			Dict   =   ["Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."]
					@unittest.skip('''Skipping until #12647 is resolved.'''	)
					@slow
					def 					A__			(	self  :		Tuple	):



							"""simple docstring"""
							self._assert_generated_batch_equal_expected()





@require_sentencepiece
@require_tokenizers
@require_tf
class 						SCREAMING_SNAKE_CASE__		(_a ):
					lowercase_   :			Optional[Any]   =   "en"
					lowercase_   :			int   =   "zh"
					lowercase_   :			int   =   ["My name is Wolfgang and I live in Berlin"]
					lowercase_   :			str   =   ["我叫沃尔夫冈 我住在柏林"]
					@unittest.skip('''Skipping until #12647 is resolved.'''	)
					@slow
					def 					A__			(	self  :		List[Any]	):



							"""simple docstring"""
							self._assert_generated_batch_equal_expected()





@require_sentencepiece
@require_tokenizers
@require_tf
class 						SCREAMING_SNAKE_CASE__		(_a ):
					lowercase_   :			str   =   "en"
					lowercase_   :			Tuple   =   "ROMANCE"
					lowercase_   :			Tuple   =   [
					    ">>fr<< Don't spend so much time watching TV.",
					    ">>pt<< Your message has been sent.",
					    ">>es<< He's two years older than me.",
					]
					lowercase_   :			Union[str, Any]   =   [
					    "Ne passez pas autant de temps à regarder la télé.",
					    "A sua mensagem foi enviada.",
					    "Es dos años más viejo que yo.",
					]
					@unittest.skip('''Skipping until #12647 is resolved.'''	)
					@slow
					def 					A__			(	self  :		Union[str, Any]	):



							"""simple docstring"""
							self._assert_generated_batch_equal_expected()



					@unittest.skip('''Skipping until #12647 is resolved.'''	)
					@slow
					def 					A__			(	self  :		Dict	):



							"""simple docstring"""
							lowerCAmelCase__								=			TranslationPipeline(self.model					,	self.tokenizer					,	framework='''tf'''	)
							lowerCAmelCase__								=			pipeline(self.src_text	)
							self.assertEqual(self.expected_text					,	[x['''translation_text'''] for x in output]	)







import importlib
import inspect
from functools import wraps
from typing import TYPE_CHECKING, Optional, Union

from .download.streaming_download_manager import (
    xbasename,
    xdirname,
    xet_parse,
    xexists,
    xgetsize,
    xglob,
    xgzip_open,
    xisdir,
    xisfile,
    xjoin,
    xlistdir,
    xnumpy_load,
    xopen,
    xpandas_read_csv,
    xpandas_read_excel,
    xPath,
    xrelpath,
    xsio_loadmat,
    xsplit,
    xsplitext,
    xwalk,
    xxml_dom_minidom_parse,
)
from .utils.logging import get_logger
from .utils.patching import patch_submodule
from .utils.py_utils import get_imports


__magic_name__     :     List[Any]  							=					get_logger(__name__)


if TYPE_CHECKING:
      from .builder import DatasetBuilder
def 		a_       (   __lowerCAmelCase      ,   __lowerCAmelCase = None							):
  lowerCAmelCase__								=			importlib.import_module(__lowerCAmelCase							)

  # TODO(QL): always update the module to add subsequent new authentication
  if hasattr(__lowerCAmelCase      ,   '''_patched_for_streaming'''							) and module._patched_for_streaming:
    return

  def wrap_auth(__lowerCAmelCase							):
    @wraps(__lowerCAmelCase							)
    def wrapper(*__lowerCAmelCase      ,   **__lowerCAmelCase							):
      return function(*__lowerCAmelCase      ,   use_auth_token=__lowerCAmelCase      ,   **__lowerCAmelCase							)

    lowerCAmelCase__								=			'''wrap_auth'''
    return wrapper

  # open files in a streaming fashion
  patch_submodule(__lowerCAmelCase      ,   '''open'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.listdir'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.walk'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''glob.glob'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  # allow to navigate in remote zip files
  patch_submodule(__lowerCAmelCase      ,   '''os.path.join'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.dirname'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.basename'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.relpath'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.split'''      ,   __lowerCAmelCase							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.splitext'''      ,   __lowerCAmelCase							).start()
  # allow checks on paths
  patch_submodule(__lowerCAmelCase      ,   '''os.path.exists'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.isdir'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.isfile'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''os.path.getsize'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''pathlib.Path'''      ,   __lowerCAmelCase							).start()
  # file readers
  patch_submodule(__lowerCAmelCase      ,   '''gzip.open'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''numpy.load'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''pandas.read_csv'''      ,   wrap_auth(__lowerCAmelCase							)      ,   attrs=['''__version__''']							).start()
  patch_submodule(__lowerCAmelCase      ,   '''pandas.read_excel'''      ,   wrap_auth(__lowerCAmelCase							)      ,   attrs=['''__version__''']							).start()
  patch_submodule(__lowerCAmelCase      ,   '''scipy.io.loadmat'''      ,   wrap_auth(__lowerCAmelCase							)      ,   attrs=['''__version__''']							).start()
  patch_submodule(__lowerCAmelCase      ,   '''xml.etree.ElementTree.parse'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  patch_submodule(__lowerCAmelCase      ,   '''xml.dom.minidom.parse'''      ,   wrap_auth(__lowerCAmelCase							)							).start()
  lowerCAmelCase__								=			True




def 		a_       (   __lowerCAmelCase							):
  extend_module_for_streaming(builder.__module__      ,   use_auth_token=builder.use_auth_token							)
  # if needed, we also have to extend additional internal imports (like wmt14 -> wmt_utils)
  if not builder.__module__.startswith('''datasets.'''							):  # check that it's not a packaged builder like csv
    for imports in get_imports(inspect.getfile(builder.__class__							)							):
      if imports[0] == "internal":
        lowerCAmelCase__								=			imports[1]
        lowerCAmelCase__								=			'''.'''.join(builder.__module__.split('''.'''							)[:-1] + [internal_import_name]							)
        extend_module_for_streaming(__lowerCAmelCase      ,   use_auth_token=builder.use_auth_token							)

    # builders can inherit from other builders that might use streaming functionality
    # (for example, ImageFolder and AudioFolder inherit from FolderBuilder which implements examples generation)
    # but these parents builders are not patched automatically as they are not instantiated, so we patch them here
  from .builder import DatasetBuilder

  lowerCAmelCase__								=			[
      cls.__module__
      for cls in type(__lowerCAmelCase							).__mro__[1:]  # make sure it's not the same module we've already patched
      if issubclass(cls      ,   __lowerCAmelCase							) and cls.__module__ != DatasetBuilder.__module__
  ]  # check it's not a standard builder from datasets.builder
  for module in parent_builder_modules:
    extend_module_for_streaming(__lowerCAmelCase      ,   use_auth_token=builder.use_auth_token							)

