
"""simple docstring"""






import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)





class 							snake_case       :







   def __init__( self       ,							__UpperCAmelCase)   ->Optional[int]:
      a_						   =     foo







   def __call__( self)   ->Dict:
      return self.foo






class 							snake_case       (     datasets.Dataset		):







   @fingerprint_transform(inplace=_lowerCAmelCase)
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase)   ->Dict:
      return DatasetChild(self.data       ,							fingerprint=_lowerCAmelCase)







   @fingerprint_transform(inplace=_lowerCAmelCase)
   def    UpperCAmelCase__	( self       ,							__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase)   ->Optional[int]:
      return DatasetChild(self.data       ,							fingerprint=_lowerCAmelCase)






class 							snake_case       :







   def __init__( self       ,							__UpperCAmelCase)   ->Tuple:
      a_						   =     callable







   def __call__( self       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase)   ->int:
      if self.callable is not None:
         return self.callable(*_lowerCAmelCase       ,							**_lowerCAmelCase)







   def __getstate__( self)   ->List[Any]:
      raise pickle.PicklingError()






class 							snake_case       (     A__		):







   @require_transformers
   @pytest.mark.integration
   def    UpperCAmelCase__	( self)   ->Tuple:
      from transformers import AutoTokenizer

      def encode(__UpperCAmelCase):
         return tokenizer(_lowerCAmelCase)

      # TODO: add hash consistency tests across sessions
      a_						   =     AutoTokenizer.from_pretrained("bert-base-uncased")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase: tokenizer(_lowerCAmelCase))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     AutoTokenizer.from_pretrained("bert-base-cased")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase: tokenizer(_lowerCAmelCase))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     AutoTokenizer.from_pretrained("bert-base-uncased")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(lambda __UpperCAmelCase: tokenizer(_lowerCAmelCase))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   @require_transformers
   @pytest.mark.integration
   def    UpperCAmelCase__	( self)   ->Tuple:
      from transformers import AutoTokenizer

      a_						   =     AutoTokenizer.from_pretrained("gpt2")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      tokenizer("Hello world !")  # call once to change the tokenizer's cache
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   @require_regex
   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      import regex

      a_						   =     regex.Regex("foo")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     regex.Regex("bar")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     regex.Regex("foo")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)






class 							snake_case       (     A__		):







   def    UpperCAmelCase__	( self)   ->Any:
      def func():
         return foo

      a_						   =     [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     [1]
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->Optional[int]:
      def func():
         pass

      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()

      def func():
         pass

      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->Optional[Any]:
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->List[Any]:
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
      a_						   =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->Tuple:
      a_						   =     (
          "co_argcount",
          "co_kwonlyargcount",
          "co_nlocals",
          "co_stacksize",
          "co_flags",
          "co_code",
          "co_consts",
          "co_names",
          "co_varnames",
          "co_filename",
          "co_name",
          "co_firstlineno",
          "co_lnotab",
          "co_freevars",
          "co_cellvars",
      )

      def _create_code_pyaa(*__UpperCAmelCase):
         if hasattr(args[-3]       ,							"encode"):
            a_						   =     list(_lowerCAmelCase)
            a_						   =     args[-3].encode()  # co_lnotab
            a_						   =     args[-10].encode()  # co_code
         if hasattr(_lowerCAmelCase       ,							"co_posonlyargcount"):
            if len(_lowerCAmelCase) == 16:
               return CodeType(*_lowerCAmelCase)
            elif len(_lowerCAmelCase) == 15:
               return CodeType(args[0]       ,							0       ,							*args[1:])
            return CodeType(args[0]       ,							0       ,							0       ,							*args[1:])
         elif hasattr(_lowerCAmelCase       ,							"co_kwonlyargcount"):
            if len(_lowerCAmelCase) == 16:
               return CodeType(args[0]       ,							*args[2:])
            elif len(_lowerCAmelCase) == 15:
               return CodeType(*_lowerCAmelCase)
            return CodeType(args[0]       ,							0       ,							*args[1:])
         if len(_lowerCAmelCase) == 16:
            return CodeType(args[0]       ,							*args[3:])
         elif len(_lowerCAmelCase) == 15:
            return CodeType(args[0]       ,							*args[2:])
         return CodeType(*_lowerCAmelCase)

      def create_ipython_func(__UpperCAmelCase       ,							__UpperCAmelCase):
         def func():
            return returned_obj

         a_						   =     func.__code__
         # Use _create_code from dill in order to make it work for different python versions
         if datasets.config.PY_VERSION < version.parse("3.8"):
            a_						   =     _create_code_pyaa(
                *[getattr(_lowerCAmelCase       ,							_lowerCAmelCase) if k != "co_filename" else co_filename for k in code_args_pyaa])
         else:
            a_						   =     code.replace(co_filename=_lowerCAmelCase)
         return FunctionType(_lowerCAmelCase       ,							func.__globals__       ,							func.__name__       ,							func.__defaults__       ,							func.__closure__)

      a_   ,			a_						   =     "<ipython-input-2-e0383a102aae>", [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase       ,							_lowerCAmelCase))).hexdigest()
      a_   ,			a_						   =     "<ipython-input-2-e0383a102aae>", [1]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase       ,							_lowerCAmelCase))).hexdigest()
      a_   ,			a_						   =     "<ipython-input-5-713f6613acf3>", [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase       ,							_lowerCAmelCase))).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)

      a_   ,			a_						   =     os.path.join(gettempdir()       ,							"ipykernel_12345"       ,							"321456789.py"), [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase       ,							_lowerCAmelCase))).hexdigest()
      a_   ,			a_						   =     os.path.join(gettempdir()       ,							"ipykernel_12345"       ,							"321456789.py"), [1]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase       ,							_lowerCAmelCase))).hexdigest()
      a_   ,			a_						   =     os.path.join(gettempdir()       ,							"ipykernel_12345"       ,							"654123987.py"), [0]
      a_						   =     mda(datasets.utils.py_utils.dumps(create_ipython_func(_lowerCAmelCase       ,							_lowerCAmelCase))).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->Any:
      a_   ,			a_						   =     [0], [1]

      def func():
         return foo, bar

      a_						   =     "__main__"

      def globalvars_mocka_side_effect(__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase):
         return {"foo": foo, "bar": bar}

      def globalvars_mocka_side_effect(__UpperCAmelCase       ,							*__UpperCAmelCase       ,							**__UpperCAmelCase):
         return {"bar": bar, "foo": foo}

      with patch("dill.detect.globalvars"       ,							side_effect=_lowerCAmelCase) as globalvars_mocka:
         a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count       ,							0)
      with patch("dill.detect.globalvars"       ,							side_effect=_lowerCAmelCase) as globalvars_mocka:
         a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
         self.assertGreater(globalvars_mocka.call_count       ,							0)
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)






class 							snake_case       (     A__		):







   def    UpperCAmelCase__	( self)   ->List[str]:
      a_						   =     Hasher.hash("hello")
      a_						   =     Hasher.hash("hello")
      a_						   =     Hasher.hash("there")
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      a_						   =     Hasher.hash(Foo("hello"))
      a_						   =     Hasher.hash(Foo("hello"))
      a_						   =     Hasher.hash(Foo("there"))
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->Optional[int]:
      a_						   =     Hasher()
      for x in ["hello", Foo("hello")]:
         hasher.update(_lowerCAmelCase)
      a_						   =     hasher.hexdigest()
      a_						   =     Hasher()
      for x in ["hello", Foo("hello")]:
         hasher.update(_lowerCAmelCase)
      a_						   =     hasher.hexdigest()
      a_						   =     Hasher()
      for x in ["there", Foo("there")]:
         hasher.update(_lowerCAmelCase)
      a_						   =     hasher.hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   def    UpperCAmelCase__	( self)   ->List[Any]:
      with self.assertRaises(pickle.PicklingError):
         Hasher.hash(UnpicklableCallable(Foo("hello")))







   def    UpperCAmelCase__	( self)   ->List[Any]:
      a_						   =     "abc"
      a_						   =     [string, string]  # two strings have the same ids
      a_						   =     [string, string]
      a_						   =     json.loads(F'''[\"{string}\", \"{string}\"]''')  # two strings have different ids
      self.assertIs(obja[0]       ,							_lowerCAmelCase)
      self.assertIs(obja[0]       ,							obja[1])
      self.assertIs(obja[0]       ,							_lowerCAmelCase)
      self.assertIs(obja[0]       ,							obja[1])
      self.assertIsNot(obja[0]       ,							_lowerCAmelCase)
      self.assertIsNot(obja[0]       ,							obja[1])
      a_						   =     Hasher.hash(_lowerCAmelCase)
      a_						   =     Hasher.hash(_lowerCAmelCase)
      a_						   =     Hasher.hash(_lowerCAmelCase)
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   @require_tiktoken
   def    UpperCAmelCase__	( self)   ->Optional[int]:
      import tiktoken

      a_						   =     tiktoken.get_encoding("gpt2")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     tiktoken.get_encoding("r50k_base")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     tiktoken.get_encoding("gpt2")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   @require_torch
   def    UpperCAmelCase__	( self)   ->Union[str, Any]:
      import torch

      a_						   =     torch.tensor([1.0])
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     torch.tensor([2.0])
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     torch.tensor([1.0])
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)







   @require_spacy
   @require_spacy_model("en_core_web_sm")
   @require_spacy_model("fr_core_news_sm")
   @pytest.mark.integration
   def    UpperCAmelCase__	( self)   ->List[Any]:
      import spacy

      a_						   =     spacy.load("en_core_web_sm")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     spacy.load("fr_core_news_sm")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      a_						   =     spacy.load("en_core_web_sm")
      a_						   =     mda(datasets.utils.py_utils.dumps(_lowerCAmelCase)).hexdigest()
      self.assertEqual(_lowerCAmelCase       ,							_lowerCAmelCase)
      self.assertNotEqual(_lowerCAmelCase       ,							_lowerCAmelCase)






@pytest.mark.integration
def    UpperCamelCase						(					UpperCAmelCase     )						->Optional[int]:

   """simple docstring"""



   a_						   =     tmp_path / "dir1"
   a_						   =     tmp_path / "dir2"
   dira.mkdir()
   dira.mkdir()
   a_						   =     "script.py"
   a_						   =     dedent(
       "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "     )
   a_						   =     dira / script_filename
   a_						   =     dira / script_filename
   with script_patha.open("w"     ) as f:
      f.write(UpperCAmelCase     )
   with script_patha.open("w"     ) as f:
      f.write(UpperCAmelCase     )
   a_						   =     subprocess.check_output(["python", str(UpperCAmelCase     )]     )
   a_						   =     subprocess.check_output(["python", str(UpperCAmelCase     )]     )
   assert fingerprinta == fingerprinta
def    UpperCamelCase						(					)						->Optional[Any]:

   """simple docstring"""



   a_						   =     {"a": [0, 1, 2]}
   a_						   =     DatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     )
   a_						   =     dataset.funca()._fingerprint
   assert expected_fingerprint == dataset.funca()._fingerprint
   assert expected_fingerprint != dataset.funca()._fingerprint

   with Pool(2     ) as p:
      assert expected_fingerprint == p.apply_async(dataset.funca     ).get()._fingerprint
      assert expected_fingerprint != p.apply_async(dataset.funca     ).get()._fingerprint
def    UpperCamelCase						(					)						->str:

   """simple docstring"""



   a_						   =     {"a": [0, 1, 2]}





   class 							snake_case       (     datasets.Dataset		):







      @fingerprint_transform(inplace=_lowerCAmelCase)
      def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Optional[int]:
         return DummyDatasetChild(self.data       ,							fingerprint=_lowerCAmelCase)






   a_						   =     DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     ).func()





   class 							snake_case       (     datasets.Dataset		):







      @fingerprint_transform(inplace=_lowerCAmelCase       ,							version="1.0.0")
      def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Optional[int]:
         return DummyDatasetChild(self.data       ,							fingerprint=_lowerCAmelCase)






   a_						   =     DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     ).func()





   class 							snake_case       (     datasets.Dataset		):







      @fingerprint_transform(inplace=_lowerCAmelCase       ,							version="2.0.0")
      def    UpperCAmelCase__	( self       ,							__UpperCAmelCase)   ->Union[str, Any]:
         return DummyDatasetChild(self.data       ,							fingerprint=_lowerCAmelCase)






   a_						   =     DummyDatasetChild(InMemoryTable.from_pydict(UpperCAmelCase     )     ).func()

   assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}     ) == 3



def    UpperCamelCase						(					)						->Dict:

   """simple docstring"""



   a_						   =     Hasher()
   hasher.update(lambda UpperCAmelCase     : x     )






'''simple docstring'''


from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
				import tensorflow as tf

				from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel



@require_tf
class 				__UpperCAmelCase	:







		'''simple docstring'''







		__lowerCAmelCase								=     MarianConfig
		__lowerCAmelCase								=     {}
		__lowerCAmelCase								=     '''gelu'''







		def __init__(self    :   Tuple   ,   _lowerCAmelCase    :   Union[str, Any]   ,   _lowerCAmelCase    :   Dict=13   ,   _lowerCAmelCase    :   Optional[int]=7   ,   _lowerCAmelCase    :   str=True   ,   _lowerCAmelCase    :   Optional[int]=False   ,   _lowerCAmelCase    :   Tuple=99   ,   _lowerCAmelCase    :   str=32   ,   _lowerCAmelCase    :   str=2   ,   _lowerCAmelCase    :   Union[str, Any]=4   ,   _lowerCAmelCase    :   Dict=37   ,   _lowerCAmelCase    :   int=0.1   ,   _lowerCAmelCase    :   List[Any]=0.1   ,   _lowerCAmelCase    :   Tuple=20   ,   _lowerCAmelCase    :   List[str]=2   ,   _lowerCAmelCase    :   Any=1   ,   _lowerCAmelCase    :   int=0   ,   ):
			A					=   parent
			A					=   batch_size
			A					=   seq_length
			A					=   is_training
			A					=   use_labels
			A					=   vocab_size
			A					=   hidden_size
			A					=   num_hidden_layers
			A					=   num_attention_heads
			A					=   intermediate_size

			A					=   hidden_dropout_prob
			A					=   attention_probs_dropout_prob
			A					=   max_position_embeddings
			A					=   eos_token_id
			A					=   pad_token_id
			A					=   bos_token_id







		def 							A  (self    :   int   ):
			A					=   ids_tensor([self.batch_size, self.seq_length - 1]   ,   self.vocab_size   )
			A					=   tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size   )   ,   1   )
			A					=   tf.concat([input_ids, eos_tensor]   ,   axis=1   )

			A					=   ids_tensor([self.batch_size, self.seq_length]   ,   self.vocab_size   )

			A					=   self.config_cls(
			    vocab_size=self.vocab_size   ,   d_model=self.hidden_size   ,   encoder_layers=self.num_hidden_layers   ,   decoder_layers=self.num_hidden_layers   ,   encoder_attention_heads=self.num_attention_heads   ,   decoder_attention_heads=self.num_attention_heads   ,   encoder_ffn_dim=self.intermediate_size   ,   decoder_ffn_dim=self.intermediate_size   ,   dropout=self.hidden_dropout_prob   ,   attention_dropout=self.attention_probs_dropout_prob   ,   max_position_embeddings=self.max_position_embeddings   ,   eos_token_ids=[2]   ,   bos_token_id=self.bos_token_id   ,   pad_token_id=self.pad_token_id   ,   decoder_start_token_id=self.pad_token_id   ,   **self.config_updates   ,   )
			A					=   prepare_marian_inputs_dict(_lowerCAmelCase   ,   _lowerCAmelCase   ,   _lowerCAmelCase   )
			return config, inputs_dict







		def 							A  (self    :   List[Any]   ,   _lowerCAmelCase    :   int   ,   _lowerCAmelCase    :   Any   ):
			A					=   TFMarianModel(config=_lowerCAmelCase   ).get_decoder()
			A					=   inputs_dict["""input_ids"""]

			A					=   input_ids[:1, :]
			A					=   inputs_dict["""attention_mask"""][:1, :]
			A					=   inputs_dict["""head_mask"""]
			A					=   1

			# first forward pass
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   head_mask=_lowerCAmelCase   ,   use_cache=_lowerCAmelCase   )

			A	,     A					=   outputs.to_tuple()

			# create hypothetical next token and extent to next_input_ids
			A					=   ids_tensor((self.batch_size, 3)   ,   config.vocab_size   )
			A					=   tf.cast(ids_tensor((self.batch_size, 3)   ,   2   )   ,   tf.inta   )

			# append to next input_ids and
			A					=   tf.concat([input_ids, next_tokens]   ,   axis=-1   )
			A					=   tf.concat([attention_mask, next_attn_mask]   ,   axis=-1   )

			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   )[0]
			A					=   model(_lowerCAmelCase   ,   attention_mask=_lowerCAmelCase   ,   past_key_values=_lowerCAmelCase   )[0]

			self.parent.assertEqual(next_tokens.shape[1]   ,   output_from_past.shape[1]   )

			# select random slice
			A					=   int(ids_tensor((1,)   ,   output_from_past.shape[-1]   )   )
			A					=   output_from_no_past[:, -3:, random_slice_idx]
			A					=   output_from_past[:, :, random_slice_idx]

			# test that outputs are equal for slice
			tf.debugging.assert_near(_lowerCAmelCase   ,   _lowerCAmelCase   ,   rtol=1e-3   )






def  __a			(  UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase							,      UpperCAmelCase=None							,      UpperCAmelCase=None							,      UpperCAmelCase=None							,      UpperCAmelCase=None							,      UpperCAmelCase=None							,      )   ->List[str]:



	"""simple docstring"""







	if attention_mask is None:
		A					=   tf.cast(tf.math.not_equal(UpperCAmelCase							,      config.pad_token_id     )							,      tf.inta     )
	if decoder_attention_mask is None:
		A					=   tf.concat(
		    [
		        tf.ones(decoder_input_ids[:, :1].shape							,      dtype=tf.inta     ),
		        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]							,      config.pad_token_id     )							,      tf.inta     ),
		    ]							,      axis=-1							,      )
	if head_mask is None:
		A					=   tf.ones((config.encoder_layers, config.encoder_attention_heads)     )
	if decoder_head_mask is None:
		A					=   tf.ones((config.decoder_layers, config.decoder_attention_heads)     )
	if cross_attn_head_mask is None:
		A					=   tf.ones((config.decoder_layers, config.decoder_attention_heads)     )
	return {
	    "input_ids": input_ids,
	    "decoder_input_ids": decoder_input_ids,
	    "attention_mask": attention_mask,
	    "decoder_attention_mask": decoder_attention_mask,
	    "head_mask": head_mask,
	    "decoder_head_mask": decoder_head_mask,
	    "cross_attn_head_mask": cross_attn_head_mask,
	}



@require_tf
class 				__UpperCAmelCase	(							A__		,       A__		,       unittest.TestCase    ):







		'''simple docstring'''







		__lowerCAmelCase								=     (TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
		__lowerCAmelCase								=     (TFMarianMTModel,) if is_tf_available() else ()
		__lowerCAmelCase								=     (
		    {
		        '''conversational''': TFMarianMTModel,
		        '''feature-extraction''': TFMarianModel,
		        '''summarization''': TFMarianMTModel,
		        '''text2text-generation''': TFMarianMTModel,
		        '''translation''': TFMarianMTModel,
		    }
		    if is_tf_available()
		    else {}
		)
		__lowerCAmelCase								=     True
		__lowerCAmelCase								=     False
		__lowerCAmelCase								=     False







		def 							A  (self    :   Tuple   ):
			A					=   TFMarianModelTester(self   )
			A					=   ConfigTester(self   ,   config_class=_lowerCAmelCase   )







		def 							A  (self    :   List[str]   ):
			self.config_tester.run_common_tests()







		def 							A  (self    :   Any   ):
			A					=   self.model_tester.prepare_config_and_inputs_for_common()
			self.model_tester.check_decoder_model_past_large_inputs(*_lowerCAmelCase   )






@require_tf
class 				__UpperCAmelCase	(							unittest.TestCase    ):







		'''simple docstring'''







		__lowerCAmelCase								=     10_00  # show more chars for failing integration tests







		@classmethod
		def 							A  (cls    :   Union[str, Any]   ):
			A					=   F"""Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"""
			return cls







		@cached_property
		def 							A  (self    :   str   ):
			return AutoTokenizer.from_pretrained(self.model_name   )







		@property
		def 							A  (self    :   Optional[int]   ):
			return self.tokenizer.eos_token_id







		@cached_property
		def 							A  (self    :   Optional[Any]   ):
			warnings.simplefilter("""error"""   )
			A					=   TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name   )
			assert isinstance(_lowerCAmelCase   ,   _lowerCAmelCase   )
			A					=   model.config
			self.assertListEqual(c.bad_words_ids   ,   [[c.pad_token_id]]   )
			self.assertEqual(c.max_length   ,   512   )
			self.assertEqual(c.decoder_start_token_id   ,   c.pad_token_id   )
			return model







		def 							A  (self    :   Optional[int]   ,   **_lowerCAmelCase    :   Any   ):
			A					=   self.translate_src_text(**_lowerCAmelCase   )
			self.assertListEqual(self.expected_text   ,   _lowerCAmelCase   )







		def 							A  (self    :   Union[str, Any]   ,   **_lowerCAmelCase    :   Union[str, Any]   ):
			A					=   self.tokenizer(self.src_text   ,   **_lowerCAmelCase   ,   padding=_lowerCAmelCase   ,   return_tensors="""tf"""   )
			A					=   self.model.generate(
			    model_inputs.input_ids   ,   attention_mask=model_inputs.attention_mask   ,   num_beams=2   ,   max_length=128   )
			A					=   self.tokenizer.batch_decode(generated_ids.numpy()   ,   skip_special_tokens=_lowerCAmelCase   )
			return generated_words






@require_sentencepiece
@require_tokenizers
@require_tf
class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		__lowerCAmelCase								=     '''mt'''
		__lowerCAmelCase								=     '''en'''
		__lowerCAmelCase								=     ['''Billi messu b\'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem.''']
		__lowerCAmelCase								=     ['''Touching gently, Jesus healed a man who was affected by the sad disease of leprosy.''']







		@unittest.skip("""Skipping until #12647 is resolved."""   )
		@slow
		def 							A  (self    :   str   ):
			self._assert_generated_batch_equal_expected()






@require_sentencepiece
@require_tokenizers
@require_tf
class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		__lowerCAmelCase								=     '''en'''
		__lowerCAmelCase								=     '''zh'''
		__lowerCAmelCase								=     ['''My name is Wolfgang and I live in Berlin''']
		__lowerCAmelCase								=     ['''我叫沃尔夫冈 我住在柏林''']







		@unittest.skip("""Skipping until #12647 is resolved."""   )
		@slow
		def 							A  (self    :   Any   ):
			self._assert_generated_batch_equal_expected()






@require_sentencepiece
@require_tokenizers
@require_tf
class 				__UpperCAmelCase	(							A__    ):







		'''simple docstring'''







		__lowerCAmelCase								=     '''en'''
		__lowerCAmelCase								=     '''ROMANCE'''
		__lowerCAmelCase								=     [
		    '''>>fr<< Don\'t spend so much time watching TV.''',
		    '''>>pt<< Your message has been sent.''',
		    '''>>es<< He\'s two years older than me.''',
		]
		__lowerCAmelCase								=     [
		    '''Ne passez pas autant de temps à regarder la télé.''',
		    '''A sua mensagem foi enviada.''',
		    '''Es dos años más viejo que yo.''',
		]







		@unittest.skip("""Skipping until #12647 is resolved."""   )
		@slow
		def 							A  (self    :   Union[str, Any]   ):
			self._assert_generated_batch_equal_expected()







		@unittest.skip("""Skipping until #12647 is resolved."""   )
		@slow
		def 							A  (self    :   Union[str, Any]   ):
			A					=   TranslationPipeline(self.model   ,   self.tokenizer   ,   framework="""tf"""   )
			A					=   pipeline(self.src_text   )
			self.assertEqual(self.expected_text   ,   [x["""translation_text"""] for x in output]   )

