






"""simple docstring"""





from __future__ import annotations

import unittest
import warnings

from transformers import AutoTokenizer, MarianConfig, MarianTokenizer, TranslationPipeline, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
  import tensorflow as tf

  from transformers import TFAutoModelForSeqaSeqLM, TFMarianModel, TFMarianMTModel







@require_tf
class 						_a  :





   '''simple docstring'''






   UpperCamelCase__     	=							MarianConfig
   UpperCamelCase__     	=							{}
   UpperCamelCase__     	=							"""gelu"""





   def __init__(       self					, UpperCAmelCase_					, UpperCAmelCase_=13					, UpperCAmelCase_=7					, UpperCAmelCase_=True					, UpperCAmelCase_=False					, UpperCAmelCase_=99					, UpperCAmelCase_=32					, UpperCAmelCase_=2					, UpperCAmelCase_=4					, UpperCAmelCase_=37					, UpperCAmelCase_=0.1					, UpperCAmelCase_=0.1					, UpperCAmelCase_=20					, UpperCAmelCase_=2					, UpperCAmelCase_=1					, UpperCAmelCase_=0					, )			->   Optional[int]:




          '''simple docstring'''




          lowercase__:							Union[str, Any]         =     parent
          lowercase__:							List[Any]         =     batch_size
          lowercase__:							Tuple         =     seq_length
          lowercase__:							str         =     is_training
          lowercase__:							Any         =     use_labels
          lowercase__:							str         =     vocab_size
          lowercase__:							Dict         =     hidden_size
          lowercase__:							Union[str, Any]         =     num_hidden_layers
          lowercase__:							Any         =     num_attention_heads
          lowercase__:							Optional[Any]         =     intermediate_size

          lowercase__:							Optional[Any]         =     hidden_dropout_prob
          lowercase__:							List[Any]         =     attention_probs_dropout_prob
          lowercase__:							Dict         =     max_position_embeddings
          lowercase__:							Union[str, Any]         =     eos_token_id
          lowercase__:							Optional[Any]         =     pad_token_id
          lowercase__:							List[Any]         =     bos_token_id





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							List[str]         =     ids_tensor([self.batch_size, self.seq_length - 1]					, self.vocab_size)
          lowercase__:							List[str]         =     tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size)					, 1)
          lowercase__:							int         =     tf.concat([input_ids, eos_tensor]					, axis=1)

          lowercase__:							int         =     ids_tensor([self.batch_size, self.seq_length]					, self.vocab_size)

          lowercase__:							List[Any]         =     self.config_cls(
              vocab_size=self.vocab_size					, d_model=self.hidden_size					, encoder_layers=self.num_hidden_layers					, decoder_layers=self.num_hidden_layers					, encoder_attention_heads=self.num_attention_heads					, decoder_attention_heads=self.num_attention_heads					, encoder_ffn_dim=self.intermediate_size					, decoder_ffn_dim=self.intermediate_size					, dropout=self.hidden_dropout_prob					, attention_dropout=self.attention_probs_dropout_prob					, max_position_embeddings=self.max_position_embeddings					, eos_token_ids=[2]					, bos_token_id=self.bos_token_id					, pad_token_id=self.pad_token_id					, decoder_start_token_id=self.pad_token_id					, **self.config_updates					, )
          lowercase__:							str         =     prepare_marian_inputs_dict(UpperCAmelCase_					, UpperCAmelCase_					, UpperCAmelCase_)
          return config, inputs_dict





   def __lowercase							(       self					, UpperCAmelCase_					, UpperCAmelCase_)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							List[Any]         =     TFMarianModel(config=UpperCAmelCase_).get_decoder()
          lowercase__:							List[str]         =     inputs_dict["input_ids"]

          lowercase__:							Dict         =     input_ids[:1, :]
          lowercase__:							int         =     inputs_dict["attention_mask"][:1, :]
          lowercase__:							int         =     inputs_dict["head_mask"]
          lowercase__:							Tuple         =     1

          # first forward pass
          lowercase__:							Optional[Any]         =     model(UpperCAmelCase_					, attention_mask=UpperCAmelCase_					, head_mask=UpperCAmelCase_					, use_cache=UpperCAmelCase_)

          lowercase__    ,      lowercase__:							int         =     outputs.to_tuple()

          # create hypothetical next token and extent to next_input_ids
          lowercase__:							str         =     ids_tensor((self.batch_size, 3)					, config.vocab_size)
          lowercase__:							str         =     tf.cast(ids_tensor((self.batch_size, 3)					, 2)					, tf.inta)

          # append to next input_ids and
          lowercase__:							List[Any]         =     tf.concat([input_ids, next_tokens]					, axis=-1)
          lowercase__:							str         =     tf.concat([attention_mask, next_attn_mask]					, axis=-1)

          lowercase__:							Any         =     model(UpperCAmelCase_					, attention_mask=UpperCAmelCase_)[0]
          lowercase__:							Tuple         =     model(UpperCAmelCase_					, attention_mask=UpperCAmelCase_					, past_key_values=UpperCAmelCase_)[0]

          self.parent.assertEqual(next_tokens.shape[1]					, output_from_past.shape[1])

          # select random slice
          lowercase__:							Optional[Any]         =     int(ids_tensor((1,)					, output_from_past.shape[-1]))
          lowercase__:							List[str]         =     output_from_no_past[:, -3:, random_slice_idx]
          lowercase__:							Optional[int]         =     output_from_past[:, :, random_slice_idx]

          # test that outputs are equal for slice
          tf.debugging.assert_near(UpperCAmelCase_					, UpperCAmelCase_					, rtol=1E-3)







def 							A(    snake_case_		,  snake_case_		,  snake_case_		,  snake_case_=None		,  snake_case_=None		,  snake_case_=None		,  snake_case_=None		,  snake_case_=None		,  ):
       """simple docstring"""





       if attention_mask is None:
              lowercase__:							Optional[int]         =     tf.cast(tf.math.not_equal(snake_case_		,  config.pad_token_id						)		,  tf.inta						)
       if decoder_attention_mask is None:
              lowercase__:							Optional[Any]         =     tf.concat(
                  [
                      tf.ones(decoder_input_ids[:, :1].shape		,  dtype=tf.inta						),
                      tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]		,  config.pad_token_id						)		,  tf.inta						),
                  ]		,  axis=-1		,  )
       if head_mask is None:
              lowercase__:							str         =     tf.ones((config.encoder_layers, config.encoder_attention_heads)						)
       if decoder_head_mask is None:
              lowercase__:							Union[str, Any]         =     tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
       if cross_attn_head_mask is None:
              lowercase__:							str         =     tf.ones((config.decoder_layers, config.decoder_attention_heads)						)
       return {
           "input_ids": input_ids,
           "decoder_input_ids": decoder_input_ids,
           "attention_mask": attention_mask,
           "decoder_attention_mask": decoder_attention_mask,
           "head_mask": head_mask,
           "decoder_head_mask": decoder_head_mask,
           "cross_attn_head_mask": cross_attn_head_mask,
       }







@require_tf
class 						_a  (		lowercase_							,	lowercase_							,	unittest.TestCase       ):





   '''simple docstring'''






   UpperCamelCase__     	=							(TFMarianMTModel, TFMarianModel) if is_tf_available() else ()
   UpperCamelCase__     	=							(TFMarianMTModel,) if is_tf_available() else ()
   UpperCamelCase__     	=							(
       {
           """conversational""": TFMarianMTModel,
           """feature-extraction""": TFMarianModel,
           """summarization""": TFMarianMTModel,
           """text2text-generation""": TFMarianMTModel,
           """translation""": TFMarianMTModel,
       }
       if is_tf_available()
       else {}
   )
   UpperCamelCase__     	=							True
   UpperCamelCase__     	=							False
   UpperCamelCase__     	=							False





   def __lowercase							(       self)			->   Optional[int]:




          '''simple docstring'''




          lowercase__:							Tuple         =     TFMarianModelTester(self)
          lowercase__:							Dict         =     ConfigTester(self					, config_class=UpperCAmelCase_)





   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          self.config_tester.run_common_tests()





   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          lowercase__:							List[Any]         =     self.model_tester.prepare_config_and_inputs_for_common()
          self.model_tester.check_decoder_model_past_large_inputs(*UpperCAmelCase_)







@require_tf
class 						_a  (		unittest.TestCase       ):





   '''simple docstring'''






   UpperCamelCase__     	=							1_000  # show more chars for failing integration tests





   @classmethod
   def __lowercase							(       cls)			->   None:




          '''simple docstring'''




          lowercase__:							Union[str, Any]         =     F"""Helsinki-NLP/opus-mt-{cls.src}-{cls.tgt}"""
          return cls





   @cached_property
   def __lowercase							(       self)			->   MarianTokenizer:




          '''simple docstring'''




          return AutoTokenizer.from_pretrained(self.model_name)





   @property
   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          return self.tokenizer.eos_token_id





   @cached_property
   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          warnings.simplefilter("error")
          lowercase__:							TFMarianMTModel         =     TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name)
          assert isinstance(UpperCAmelCase_					, UpperCAmelCase_)
          lowercase__:							Optional[Any]         =     model.config
          self.assertListEqual(c.bad_words_ids					, [[c.pad_token_id]])
          self.assertEqual(c.max_length					, 512)
          self.assertEqual(c.decoder_start_token_id					, c.pad_token_id)
          return model





   def __lowercase							(       self					, **UpperCAmelCase_)			->   Tuple:




          '''simple docstring'''




          lowercase__:							List[str]         =     self.translate_src_text(**UpperCAmelCase_)
          self.assertListEqual(self.expected_text					, UpperCAmelCase_)





   def __lowercase							(       self					, **UpperCAmelCase_)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							int         =     self.tokenizer(self.src_text					, **UpperCAmelCase_					, padding=UpperCAmelCase_					, return_tensors="tf")
          lowercase__:							Optional[Any]         =     self.model.generate(
              model_inputs.input_ids					, attention_mask=model_inputs.attention_mask					, num_beams=2					, max_length=128)
          lowercase__:							Optional[Any]         =     self.tokenizer.batch_decode(generated_ids.numpy()					, skip_special_tokens=UpperCAmelCase_)
          return generated_words







@require_sentencepiece
@require_tokenizers
@require_tf
class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							"""mt"""
   UpperCamelCase__     	=							"""en"""
   UpperCamelCase__     	=							["""Billi messu b'mod ġentili, Ġesù fejjaq raġel li kien milqut bil - marda kerha tal - ġdiem."""]
   UpperCamelCase__     	=							["""Touching gently, Jesus healed a man who was affected by the sad disease of leprosy."""]





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							"""en"""
   UpperCamelCase__     	=							"""zh"""
   UpperCamelCase__     	=							["""My name is Wolfgang and I live in Berlin"""]
   UpperCamelCase__     	=							["""我叫沃尔夫冈 我住在柏林"""]





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   List[str]:




          '''simple docstring'''




          self._assert_generated_batch_equal_expected()







@require_sentencepiece
@require_tokenizers
@require_tf
class 						_a  (		lowercase_       ):





   '''simple docstring'''






   UpperCamelCase__     	=							"""en"""
   UpperCamelCase__     	=							"""ROMANCE"""
   UpperCamelCase__     	=							[
       """>>fr<< Don't spend so much time watching TV.""",
       """>>pt<< Your message has been sent.""",
       """>>es<< He's two years older than me.""",
   ]
   UpperCamelCase__     	=							[
       """Ne passez pas autant de temps à regarder la télé.""",
       """A sua mensagem foi enviada.""",
       """Es dos años más viejo que yo.""",
   ]





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          self._assert_generated_batch_equal_expected()





   @unittest.skip("Skipping until #12647 is resolved.")
   @slow
   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							Any         =     TranslationPipeline(self.model					, self.tokenizer					, framework="tf")
          lowercase__:							Optional[int]         =     pipeline(self.src_text)
          self.assertEqual(self.expected_text					, [x["translation_text"] for x in output])














"""simple docstring"""





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)







class 						_a  :





   '''simple docstring'''






   def __init__(       self					, UpperCAmelCase_)			->   List[Any]:




          '''simple docstring'''




          lowercase__:							List[str]         =     foo





   def __call__(       self)			->   Union[str, Any]:




          '''simple docstring'''




          return self.foo







class 						_a  (		datasets.Dataset       ):





   '''simple docstring'''






   @fingerprint_transform(inplace=UpperCAmelCase_)
   def __lowercase							(       self					, UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   int:




          '''simple docstring'''




          return DatasetChild(self.data					, fingerprint=UpperCAmelCase_)





   @fingerprint_transform(inplace=UpperCAmelCase_)
   def __lowercase							(       self					, UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   int:




          '''simple docstring'''




          return DatasetChild(self.data					, fingerprint=UpperCAmelCase_)







class 						_a  :





   '''simple docstring'''






   def __init__(       self					, UpperCAmelCase_)			->   Optional[int]:




          '''simple docstring'''




          lowercase__:							Tuple         =     callable





   def __call__(       self					, *UpperCAmelCase_					, **UpperCAmelCase_)			->   Optional[int]:




          '''simple docstring'''




          if self.callable is not None:
                 return self.callable(*UpperCAmelCase_					, **UpperCAmelCase_)





   def __getstate__(       self)			->   int:




          '''simple docstring'''




          raise pickle.PicklingError()







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   @require_transformers
   @pytest.mark.integration
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          from transformers import AutoTokenizer

          def encode(UpperCAmelCase_):
                 return tokenizer(UpperCAmelCase_)

          # TODO: add hash consistency tests across sessions
          lowercase__:							Tuple         =     AutoTokenizer.from_pretrained("bert-base-uncased")
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     AutoTokenizer.from_pretrained("bert-base-cased")
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Optional[int]         =     AutoTokenizer.from_pretrained("bert-base-uncased")
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase_: tokenizer(UpperCAmelCase_))).hexdigest()
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_transformers
   @pytest.mark.integration
   def __lowercase							(       self)			->   Optional[int]:




          '''simple docstring'''




          from transformers import AutoTokenizer

          lowercase__:							Optional[int]         =     AutoTokenizer.from_pretrained("gpt2")
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          tokenizer("Hello world !")  # call once to change the tokenizer's cache
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_regex
   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          import regex

          lowercase__:							List[Any]         =     regex.Regex("foo")
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							List[Any]         =     regex.Regex("bar")
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Union[str, Any]         =     regex.Regex("foo")
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          def func():
                 return foo

          lowercase__:							int         =     [0]
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     [1]
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Union[str, Any]         =     [0]
          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          def func():
                 pass

          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()

          def func():
                 pass

          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
          lowercase__:							int         =     mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
          lowercase__:							List[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
          lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Union[str, Any]:




          '''simple docstring'''




          lowercase__:							List[Any]         =     (
              "co_argcount",
              "co_kwonlyargcount",
              "co_nlocals",
              "co_stacksize",
              "co_flags",
              "co_code",
              "co_consts",
              "co_names",
              "co_varnames",
              "co_filename",
              "co_name",
              "co_firstlineno",
              "co_lnotab",
              "co_freevars",
              "co_cellvars",
          )

          def _create_code_pyaa(*UpperCAmelCase_):
                 if hasattr(args[-3]					, "encode"):
                        lowercase__:							Tuple         =     list(UpperCAmelCase_)
                        lowercase__:							str         =     args[-3].encode()  # co_lnotab
                        lowercase__:							Any         =     args[-10].encode()  # co_code
                 if hasattr(UpperCAmelCase_					, "co_posonlyargcount"):
                        if len(UpperCAmelCase_) == 16:
                               return CodeType(*UpperCAmelCase_)
                        elif len(UpperCAmelCase_) == 15:
                               return CodeType(args[0]					, 0					, *args[1:])
                        return CodeType(args[0]					, 0					, 0					, *args[1:])
                 elif hasattr(UpperCAmelCase_					, "co_kwonlyargcount"):
                        if len(UpperCAmelCase_) == 16:
                               return CodeType(args[0]					, *args[2:])
                        elif len(UpperCAmelCase_) == 15:
                               return CodeType(*UpperCAmelCase_)
                        return CodeType(args[0]					, 0					, *args[1:])
                 if len(UpperCAmelCase_) == 16:
                        return CodeType(args[0]					, *args[3:])
                 elif len(UpperCAmelCase_) == 15:
                        return CodeType(args[0]					, *args[2:])
                 return CodeType(*UpperCAmelCase_)

          def create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_):
                 def func():
                        return returned_obj

                 lowercase__:							Dict         =     func.__code__
                 # Use _create_code from dill in order to make it work for different python versions
                 if datasets.config.PY_VERSION < version.parse("3.8"):
                        lowercase__:							Union[str, Any]         =     _create_code_pyaa(
                            *[getattr(UpperCAmelCase_					, UpperCAmelCase_) if k != "co_filename" else co_filename for k in code_args_pyaa])
                 else:
                        lowercase__:							Tuple         =     code.replace(co_filename=UpperCAmelCase_)
                 return FunctionType(UpperCAmelCase_					, func.__globals__					, func.__name__					, func.__defaults__					, func.__closure__)

          lowercase__    ,      lowercase__:							Any         =     "<ipython-input-2-e0383a102aae>", [0]
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Optional[Any]         =     "<ipython-input-2-e0383a102aae>", [1]
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Tuple         =     "<ipython-input-5-713f6613acf3>", [0]
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)

          lowercase__    ,      lowercase__:							Dict         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "321456789.py"), [0]
          lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Dict         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "321456789.py"), [1]
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          lowercase__    ,      lowercase__:							Any         =     os.path.join(gettempdir()					, "ipykernel_12345"					, "654123987.py"), [0]
          lowercase__:							Dict         =     mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase_					, UpperCAmelCase_))).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Dict:




          '''simple docstring'''




          lowercase__    ,      lowercase__:							str         =     [0], [1]

          def func():
                 return foo, bar

          lowercase__:							List[str]         =     "__main__"

          def globalvars_mocka_side_effect(UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_):
                 return {"foo": foo, "bar": bar}

          def globalvars_mocka_side_effect(UpperCAmelCase_					, *UpperCAmelCase_					, **UpperCAmelCase_):
                 return {"bar": bar, "foo": foo}

          with patch("dill.detect.globalvars"					, side_effect=UpperCAmelCase_) as globalvars_mocka:
                 lowercase__:							Optional[Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count					, 0)
          with patch("dill.detect.globalvars"					, side_effect=UpperCAmelCase_) as globalvars_mocka:
                 lowercase__:							Tuple         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
                 self.assertGreater(globalvars_mocka.call_count					, 0)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)







class 						_a  (		lowercase_       ):





   '''simple docstring'''






   def __lowercase							(       self)			->   int:




          '''simple docstring'''




          lowercase__:							List[Any]         =     Hasher.hash("hello")
          lowercase__:							Optional[int]         =     Hasher.hash("hello")
          lowercase__:							int         =     Hasher.hash("there")
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          lowercase__:							str         =     Hasher.hash(Foo("hello"))
          lowercase__:							Tuple         =     Hasher.hash(Foo("hello"))
          lowercase__:							Union[str, Any]         =     Hasher.hash(Foo("there"))
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          lowercase__:							str         =     Hasher()
          for x in ["hello", Foo("hello")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							Dict         =     hasher.hexdigest()
          lowercase__:							Any         =     Hasher()
          for x in ["hello", Foo("hello")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							List[Any]         =     hasher.hexdigest()
          lowercase__:							Optional[int]         =     Hasher()
          for x in ["there", Foo("there")]:
                 hasher.update(UpperCAmelCase_)
          lowercase__:							Tuple         =     hasher.hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   def __lowercase							(       self)			->   List[str]:




          '''simple docstring'''




          with self.assertRaises(pickle.PicklingError):
                 Hasher.hash(UnpicklableCallable(Foo("hello")))





   def __lowercase							(       self)			->   Optional[Any]:




          '''simple docstring'''




          lowercase__:							Optional[Any]         =     "abc"
          lowercase__:							Any         =     [string, string]  # two strings have the same ids
          lowercase__:							str         =     [string, string]
          lowercase__:							Optional[int]         =     json.loads(F"""[\"{string}\", \"{string}\"]""")  # two strings have different ids
          self.assertIs(obja[0]					, UpperCAmelCase_)
          self.assertIs(obja[0]					, obja[1])
          self.assertIs(obja[0]					, UpperCAmelCase_)
          self.assertIs(obja[0]					, obja[1])
          self.assertIsNot(obja[0]					, UpperCAmelCase_)
          self.assertIsNot(obja[0]					, obja[1])
          lowercase__:							List[Any]         =     Hasher.hash(UpperCAmelCase_)
          lowercase__:							str         =     Hasher.hash(UpperCAmelCase_)
          lowercase__:							Tuple         =     Hasher.hash(UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_tiktoken
   def __lowercase							(       self)			->   str:




          '''simple docstring'''




          import tiktoken

          lowercase__:							Union[str, Any]         =     tiktoken.get_encoding("gpt2")
          lowercase__:							List[str]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     tiktoken.get_encoding("r50k_base")
          lowercase__:							Dict         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							List[Any]         =     tiktoken.get_encoding("gpt2")
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_torch
   def __lowercase							(       self)			->   List[Any]:




          '''simple docstring'''




          import torch

          lowercase__:							Tuple         =     torch.tensor([1.0])
          lowercase__:							Optional[int]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Dict         =     torch.tensor([2.0])
          lowercase__:							Union[str, Any]         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Tuple         =     torch.tensor([1.0])
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)





   @require_spacy
   @require_spacy_model("en_core_web_sm")
   @require_spacy_model("fr_core_news_sm")
   @pytest.mark.integration
   def __lowercase							(       self)			->   Any:




          '''simple docstring'''




          import spacy

          lowercase__:							Tuple         =     spacy.load("en_core_web_sm")
          lowercase__:							Any         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							Any         =     spacy.load("fr_core_news_sm")
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          lowercase__:							int         =     spacy.load("en_core_web_sm")
          lowercase__:							str         =     mda(datasets.utils.py_utils.dumps(UpperCAmelCase_)).hexdigest()
          self.assertEqual(UpperCAmelCase_					, UpperCAmelCase_)
          self.assertNotEqual(UpperCAmelCase_					, UpperCAmelCase_)







@pytest.mark.integration
def 							A(    snake_case_						):
       """simple docstring"""





       lowercase__:							Optional[Any]         =     tmp_path / "dir1"
       lowercase__:							Tuple         =     tmp_path / "dir2"
       dira.mkdir()
       dira.mkdir()
       lowercase__:							Union[str, Any]         =     "script.py"
       lowercase__:							Any         =     dedent(
           "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
       lowercase__:							List[str]         =     dira / script_filename
       lowercase__:							int         =     dira / script_filename
       with script_patha.open("w"						) as f:
              f.write(snake_case_						)
       with script_patha.open("w"						) as f:
              f.write(snake_case_						)
       lowercase__:							Any         =     subprocess.check_output(["python", str(snake_case_						)]						)
       lowercase__:							Optional[Any]         =     subprocess.check_output(["python", str(snake_case_						)]						)
       assert fingerprinta == fingerprinta






def 							A(    ):
       """simple docstring"""





       lowercase__:							Optional[int]         =     {"a": [0, 1, 2]}
       lowercase__:							Dict         =     DatasetChild(InMemoryTable.from_pydict(snake_case_						)						)
       lowercase__:							List[Any]         =     dataset.funca()._fingerprint
       assert expected_fingerprint == dataset.funca()._fingerprint
       assert expected_fingerprint != dataset.funca()._fingerprint

       with Pool(2						) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint






def 							A(    ):
       """simple docstring"""





       lowercase__:							List[str]         =     {"a": [0, 1, 2]}







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_)
          def __lowercase							(       self					, UpperCAmelCase_)			->   List[str]:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Tuple         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_					, version="1.0.0")
          def __lowercase							(       self					, UpperCAmelCase_)			->   Optional[int]:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Optional[int]         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()







       class 						_a  (		datasets.Dataset       ):





          '''simple docstring'''






          @fingerprint_transform(inplace=UpperCAmelCase_					, version="2.0.0")
          def __lowercase							(       self					, UpperCAmelCase_)			->   Any:




                 '''simple docstring'''




                 return DummyDatasetChild(self.data					, fingerprint=UpperCAmelCase_)







       lowercase__:							Optional[int]         =     DummyDatasetChild(InMemoryTable.from_pydict(snake_case_						)						).func()

       assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3







def 							A(    ):
       """simple docstring"""





       lowercase__:							int         =     Hasher()
       hasher.update(lambda snake_case_						: x						)






