code: 
  
  
  
  
  
  import json
  import os
  import pickle
  import subprocess
  from hashlib import mda
  from pathlib import Path
  from tempfile import gettempdir
  from textwrap import dedent
  from types import CodeType, FunctionType
  from unittest import TestCase
  from unittest.mock import patch
  
  import pytest
  from multiprocess import Pool
  from packaging import version
  
  import datasets
  from datasets.fingerprint import Hasher, fingerprint_transform
  from datasets.table import InMemoryTable
  
  from .utils import (
      require_regex,
      require_spacy,
      require_spacy_model,
      require_tiktoken,
      require_torch,
      require_transformers,
  )
  
  
  
  
  class 				_lowerCamelCase :
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def __init__( self				,						UpperCAmelCase   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Any        =       foo
  
  
  
  
  
  
  
        def __call__( self   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return self.foo
  
  
  
  
  class 				_lowerCamelCase (    datasets.Dataset    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @fingerprint_transform(inplace=UpperCAmelCase   )
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						*UpperCAmelCase				,						**UpperCAmelCase   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return DatasetChild(self.data				,						fingerprint=UpperCAmelCase   )
  
  
  
  
  
  
  
        @fingerprint_transform(inplace=UpperCAmelCase   )
        def 						UpperCAmelCase		( self				,						UpperCAmelCase				,						*UpperCAmelCase				,						**UpperCAmelCase   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return DatasetChild(self.data				,						fingerprint=UpperCAmelCase   )
  
  
  
  
  class 				_lowerCamelCase :
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def __init__( self				,						UpperCAmelCase   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Union[str, Any]        =       callable
  
  
  
  
  
  
  
        def __call__( self				,						*UpperCAmelCase				,						**UpperCAmelCase   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          if self.callable is not None:
            return self.callable(*UpperCAmelCase				,						**UpperCAmelCase   )
  
  
  
  
  
  
  
        def __getstate__( self   )       ->   List[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          raise pickle.PicklingError()
  
  
  
  
  class 				_lowerCamelCase (    a    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @require_transformers
        @pytest.mark.integration
        def 						UpperCAmelCase		( self   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          from transformers import AutoTokenizer
  
          def encode(UpperCAmelCase   ):
            return tokenizer(UpperCAmelCase   )
  
          # TODO: add hash consistency tests across sessions
          __snake_case   :				Optional[int]        =       AutoTokenizer.from_pretrained("bert-base-uncased"   )
          __snake_case   :				List[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				str        =       mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase   : tokenizer(UpperCAmelCase   )   )   ).hexdigest()
          __snake_case   :				Any        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				int        =       AutoTokenizer.from_pretrained("bert-base-cased"   )
          __snake_case   :				str        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Dict        =       mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase   : tokenizer(UpperCAmelCase   )   )   ).hexdigest()
          __snake_case   :				Dict        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Optional[int]        =       AutoTokenizer.from_pretrained("bert-base-uncased"   )
          __snake_case   :				List[str]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				str        =       mda(datasets.utils.py_utils.dumps(lambda UpperCAmelCase   : tokenizer(UpperCAmelCase   )   )   ).hexdigest()
          __snake_case   :				List[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        @require_transformers
        @pytest.mark.integration
        def 						UpperCAmelCase		( self   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          from transformers import AutoTokenizer
  
          __snake_case   :				Union[str, Any]        =       AutoTokenizer.from_pretrained("gpt2"   )
          __snake_case   :				int        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          tokenizer("Hello world !"   )  # call once to change the tokenizer's cache
          __snake_case   :				List[str]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        @require_regex
        def 						UpperCAmelCase		( self   )       ->   Optional[Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          import regex
  
          __snake_case   :				int        =       regex.Regex("foo"   )
          __snake_case   :				int        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Tuple        =       regex.Regex("bar"   )
          __snake_case   :				Optional[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				str        =       regex.Regex("foo"   )
          __snake_case   :				Any        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  class 				_lowerCamelCase (    a    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          def func():
            return foo
  
          __snake_case   :				Tuple        =       [0]
          __snake_case   :				str        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Optional[Any]        =       [1]
          __snake_case   :				List[str]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Dict        =       [0]
          __snake_case   :				Optional[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          def func():
            pass
  
          __snake_case   :				List[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
  
          def func():
            pass
  
          __snake_case   :				Tuple        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Dict        =       mda(datasets.utils.py_utils.dumps(Foo([0]   )   )   ).hexdigest()
          __snake_case   :				Optional[int]        =       mda(datasets.utils.py_utils.dumps(Foo([1]   )   )   ).hexdigest()
          __snake_case   :				List[Any]        =       mda(datasets.utils.py_utils.dumps(Foo([0]   )   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       mda(datasets.utils.py_utils.dumps(Foo([0]   ).__call__   )   ).hexdigest()
          __snake_case   :				List[Any]        =       mda(datasets.utils.py_utils.dumps(Foo([1]   ).__call__   )   ).hexdigest()
          __snake_case   :				Optional[int]        =       mda(datasets.utils.py_utils.dumps(Foo([0]   ).__call__   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Union[str, Any]        =       (
              "co_argcount",
              "co_kwonlyargcount",
              "co_nlocals",
              "co_stacksize",
              "co_flags",
              "co_code",
              "co_consts",
              "co_names",
              "co_varnames",
              "co_filename",
              "co_name",
              "co_firstlineno",
              "co_lnotab",
              "co_freevars",
              "co_cellvars",
          )
  
          def _create_code_pyaa(*UpperCAmelCase   ):
            if hasattr(args[-3]				,						"encode"   ):
              __snake_case   :				Optional[Any]        =       list(UpperCAmelCase   )
              __snake_case   :				Dict        =       args[-3].encode()  # co_lnotab
              __snake_case   :				Optional[int]        =       args[-10].encode()  # co_code
            if hasattr(UpperCAmelCase				,						"co_posonlyargcount"   ):
              if len(UpperCAmelCase   ) == 16:
                return CodeType(*UpperCAmelCase   )
              elif len(UpperCAmelCase   ) == 15:
                return CodeType(args[0]				,						0				,						*args[1:]   )
              return CodeType(args[0]				,						0				,						0				,						*args[1:]   )
            elif hasattr(UpperCAmelCase				,						"co_kwonlyargcount"   ):
              if len(UpperCAmelCase   ) == 16:
                return CodeType(args[0]				,						*args[2:]   )
              elif len(UpperCAmelCase   ) == 15:
                return CodeType(*UpperCAmelCase   )
              return CodeType(args[0]				,						0				,						*args[1:]   )
            if len(UpperCAmelCase   ) == 16:
              return CodeType(args[0]				,						*args[3:]   )
            elif len(UpperCAmelCase   ) == 15:
              return CodeType(args[0]				,						*args[2:]   )
            return CodeType(*UpperCAmelCase   )
  
          def create_ipython_func(UpperCAmelCase				,						UpperCAmelCase   ):
            def func():
              return returned_obj
  
            __snake_case   :				str        =       func.__code__
            # Use _create_code from dill in order to make it work for different python versions
            if datasets.config.PY_VERSION < version.parse("3.8"   ):
              __snake_case   :				List[Any]        =       _create_code_pyaa(
                  *[getattr(UpperCAmelCase				,						UpperCAmelCase   ) if k != "co_filename" else co_filename for k in code_args_pyaa]   )
            else:
              __snake_case   :				Union[str, Any]        =       code.replace(co_filename=UpperCAmelCase   )
            return FunctionType(UpperCAmelCase				,						func.__globals__				,						func.__name__				,						func.__defaults__				,						func.__closure__   )
  
          __snake_case    ,   __snake_case   :				Tuple        =       "<ipython-input-2-e0383a102aae>", [0]
          __snake_case   :				Tuple        =       mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase				,						UpperCAmelCase   )   )   ).hexdigest()
          __snake_case    ,   __snake_case   :				Tuple        =       "<ipython-input-2-e0383a102aae>", [1]
          __snake_case   :				int        =       mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase				,						UpperCAmelCase   )   )   ).hexdigest()
          __snake_case    ,   __snake_case   :				Tuple        =       "<ipython-input-5-713f6613acf3>", [0]
          __snake_case   :				Any        =       mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase				,						UpperCAmelCase   )   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
          __snake_case    ,   __snake_case   :				Dict        =       os.path.join(gettempdir()				,						"ipykernel_12345"				,						"321456789.py"   ), [0]
          __snake_case   :				Optional[Any]        =       mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase				,						UpperCAmelCase   )   )   ).hexdigest()
          __snake_case    ,   __snake_case   :				Union[str, Any]        =       os.path.join(gettempdir()				,						"ipykernel_12345"				,						"321456789.py"   ), [1]
          __snake_case   :				List[Any]        =       mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase				,						UpperCAmelCase   )   )   ).hexdigest()
          __snake_case    ,   __snake_case   :				List[str]        =       os.path.join(gettempdir()				,						"ipykernel_12345"				,						"654123987.py"   ), [0]
          __snake_case   :				List[str]        =       mda(datasets.utils.py_utils.dumps(create_ipython_func(UpperCAmelCase				,						UpperCAmelCase   )   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case    ,   __snake_case   :				str        =       [0], [1]
  
          def func():
            return foo, bar
  
          __snake_case   :				Optional[Any]        =       "__main__"
  
          def globalvars_mocka_side_effect(UpperCAmelCase				,						*UpperCAmelCase				,						**UpperCAmelCase   ):
            return {"foo": foo, "bar": bar}
  
          def globalvars_mocka_side_effect(UpperCAmelCase				,						*UpperCAmelCase				,						**UpperCAmelCase   ):
            return {"bar": bar, "foo": foo}
  
          with patch("dill.detect.globalvars"				,						side_effect=UpperCAmelCase   ) as globalvars_mocka:
            __snake_case   :				Tuple        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
            self.assertGreater(globalvars_mocka.call_count				,						0   )
          with patch("dill.detect.globalvars"				,						side_effect=UpperCAmelCase   ) as globalvars_mocka:
            __snake_case   :				List[str]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
            self.assertGreater(globalvars_mocka.call_count				,						0   )
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  class 				_lowerCamelCase (    a    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   int:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Dict        =       Hasher.hash("hello"   )
          __snake_case   :				Any        =       Hasher.hash("hello"   )
          __snake_case   :				Dict        =       Hasher.hash("there"   )
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Union[str, Any]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Tuple        =       Hasher.hash(Foo("hello"   )   )
          __snake_case   :				Tuple        =       Hasher.hash(Foo("hello"   )   )
          __snake_case   :				int        =       Hasher.hash(Foo("there"   )   )
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[Any]        =       Hasher()
          for x in ["hello", Foo("hello"   )]:
            hasher.update(UpperCAmelCase   )
          __snake_case   :				Tuple        =       hasher.hexdigest()
          __snake_case   :				str        =       Hasher()
          for x in ["hello", Foo("hello"   )]:
            hasher.update(UpperCAmelCase   )
          __snake_case   :				int        =       hasher.hexdigest()
          __snake_case   :				List[Any]        =       Hasher()
          for x in ["there", Foo("there"   )]:
            hasher.update(UpperCAmelCase   )
          __snake_case   :				Tuple        =       hasher.hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          with self.assertRaises(pickle.PicklingError   ):
            Hasher.hash(UnpicklableCallable(Foo("hello"   )   )   )
  
  
  
  
  
  
  
        def 						UpperCAmelCase		( self   )       ->   Tuple:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Any        =       "abc"
          __snake_case   :				str        =       [string, string]  # two strings have the same ids
          __snake_case   :				Union[str, Any]        =       [string, string]
          __snake_case   :				Optional[int]        =       json.loads(F"""[\"{string}\", \"{string}\"]"""   )  # two strings have different ids
          self.assertIs(obja[0]				,						UpperCAmelCase   )
          self.assertIs(obja[0]				,						obja[1]   )
          self.assertIs(obja[0]				,						UpperCAmelCase   )
          self.assertIs(obja[0]				,						obja[1]   )
          self.assertIsNot(obja[0]				,						UpperCAmelCase   )
          self.assertIsNot(obja[0]				,						obja[1]   )
          __snake_case   :				Dict        =       Hasher.hash(UpperCAmelCase   )
          __snake_case   :				Union[str, Any]        =       Hasher.hash(UpperCAmelCase   )
          __snake_case   :				Optional[Any]        =       Hasher.hash(UpperCAmelCase   )
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        @require_tiktoken
        def 						UpperCAmelCase		( self   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          import tiktoken
  
          __snake_case   :				Any        =       tiktoken.get_encoding("gpt2"   )
          __snake_case   :				Optional[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Dict        =       tiktoken.get_encoding("r50k_base"   )
          __snake_case   :				List[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Optional[int]        =       tiktoken.get_encoding("gpt2"   )
          __snake_case   :				Tuple        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        @require_torch
        def 						UpperCAmelCase		( self   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          import torch
  
          __snake_case   :				Tuple        =       torch.tensor([1.0]   )
          __snake_case   :				Optional[Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				List[Any]        =       torch.tensor([2.0]   )
          __snake_case   :				Tuple        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Optional[Any]        =       torch.tensor([1.0]   )
          __snake_case   :				Union[str, Any]        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
        @require_spacy
        @require_spacy_model("en_core_web_sm"   )
        @require_spacy_model("fr_core_news_sm"   )
        @pytest.mark.integration
        def 						UpperCAmelCase		( self   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          import spacy
  
          __snake_case   :				Any        =       spacy.load("en_core_web_sm"   )
          __snake_case   :				Any        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				List[Any]        =       spacy.load("fr_core_news_sm"   )
          __snake_case   :				Any        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          __snake_case   :				Any        =       spacy.load("en_core_web_sm"   )
          __snake_case   :				str        =       mda(datasets.utils.py_utils.dumps(UpperCAmelCase   )   ).hexdigest()
          self.assertEqual(UpperCAmelCase				,						UpperCAmelCase   )
          self.assertNotEqual(UpperCAmelCase				,						UpperCAmelCase   )
  
  
  
  
  
  
  
  @pytest.mark.integration
  def       lowerCAmelCase__(	lowercase       :    Path				) ->    Dict:
    __snake_case   :				Tuple        =       tmp_path / "dir1"
    __snake_case   :				Optional[int]        =       tmp_path / "dir2"
    dira.mkdir()
    dira.mkdir()
    __snake_case   :				Any        =       "script.py"
    __snake_case   :				int        =       dedent(
        "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "				)
    __snake_case   :				str        =       dira / script_filename
    __snake_case   :				Optional[Any]        =       dira / script_filename
    with script_patha.open("w"				) as f:
      f.write(lowercase				)
    with script_patha.open("w"				) as f:
      f.write(lowercase				)
    __snake_case   :				Any        =       subprocess.check_output(["python", str(lowercase				)]				)
    __snake_case   :				str        =       subprocess.check_output(["python", str(lowercase				)]				)
    assert fingerprinta == fingerprinta
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	) ->    Optional[int]:
    __snake_case   :				Optional[Any]        =       {"a": [0, 1, 2]}
    __snake_case   :				List[Any]        =       DatasetChild(InMemoryTable.from_pydict(lowercase				)				)
    __snake_case   :				Dict        =       dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint
  
    with Pool(2				) as p:
      assert expected_fingerprint == p.apply_async(dataset.funca				).get()._fingerprint
      assert expected_fingerprint != p.apply_async(dataset.funca				).get()._fingerprint
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	) ->    Tuple:
    __snake_case   :				int        =       {"a": [0, 1, 2]}
  
  
  
  
    class 				_lowerCamelCase (    datasets.Dataset    ):
  
  
  
          """simple docstring"""
  
  
  
  
  
  
          @fingerprint_transform(inplace=UpperCAmelCase   )
          def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Union[str, Any]:
  
  
  
  
  
  
            '''simple docstring'''
  
  
  
  
  
  
            return DummyDatasetChild(self.data				,						fingerprint=UpperCAmelCase   )
  
  
  
  
    __snake_case   :				Tuple        =       DummyDatasetChild(InMemoryTable.from_pydict(lowercase				)				).func()
  
  
  
  
    class 				_lowerCamelCase (    datasets.Dataset    ):
  
  
  
          """simple docstring"""
  
  
  
  
  
  
          @fingerprint_transform(inplace=UpperCAmelCase				,						version="1.0.0"   )
          def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   Dict:
  
  
  
  
  
  
            '''simple docstring'''
  
  
  
  
  
  
            return DummyDatasetChild(self.data				,						fingerprint=UpperCAmelCase   )
  
  
  
  
    __snake_case   :				Optional[int]        =       DummyDatasetChild(InMemoryTable.from_pydict(lowercase				)				).func()
  
  
  
  
    class 				_lowerCamelCase (    datasets.Dataset    ):
  
  
  
          """simple docstring"""
  
  
  
  
  
  
          @fingerprint_transform(inplace=UpperCAmelCase				,						version="2.0.0"   )
          def 						UpperCAmelCase		( self				,						UpperCAmelCase   )       ->   List[str]:
  
  
  
  
  
  
            '''simple docstring'''
  
  
  
  
  
  
            return DummyDatasetChild(self.data				,						fingerprint=UpperCAmelCase   )
  
  
  
  
    __snake_case   :				Union[str, Any]        =       DummyDatasetChild(InMemoryTable.from_pydict(lowercase				)				).func()
  
    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}				) == 3
  
  
  
  
  
  
  
  def       lowerCAmelCase__(	) ->    List[str]:
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    __snake_case   :				Optional[int]        =       Hasher()
    hasher.update(lambda lowercase				: x				)
  
  
style_context: 
  
  
  
  
  
  import torch
  from torch import autograd
  
  
  
  
  class 				_lowerCamelCase (    autograd.Function    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase   )       ->   str:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Dict        =       inputs.numel()
          __snake_case   :				Optional[int]        =       int(0.005 * nb_elems   ) + 1
          if sigmoid:
            __snake_case   :				Any        =       (torch.sigmoid(UpperCAmelCase   ) > threshold).type(inputs.type()   )
          else:
            __snake_case   :				Optional[int]        =       (inputs > threshold).type(inputs.type()   )
          if mask.sum() < nb_min:
            # We limit the pruning so that at least 0.5% (half a percent) of the weights are remaining
            __snake_case   :				List[str]        =       inputs.flatten().kthvalue(max(nb_elems - nb_min				,						1   )   ).values
            __snake_case   :				Union[str, Any]        =       (inputs > k_threshold).type(inputs.type()   )
          return mask
  
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return gradOutput, None, None
  
  
  
  
  class 				_lowerCamelCase (    autograd.Function    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase				,						UpperCAmelCase   )       ->   Optional[int]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Optional[int]        =       inputs.clone()
          __snake_case    ,   __snake_case   :				int        =       inputs.flatten().sort(descending=UpperCAmelCase   )
          __snake_case   :				Tuple        =       int(threshold * inputs.numel()   )
  
          # flat_out and mask access the same memory.
          __snake_case   :				Optional[int]        =       mask.flatten()
          __snake_case   :				Union[str, Any]        =       0
          __snake_case   :				Any        =       1
          return mask
  
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase   )       ->   List[str]:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          return gradOutput, None
  
  
  
  
  class 				_lowerCamelCase (    a    ):
  
  
  
        """simple docstring"""
  
  
  
  
  
  
        @staticmethod
        def 						UpperCAmelCase		( UpperCAmelCase				,						UpperCAmelCase   )       ->   Any:
  
  
  
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          __snake_case   :				Union[str, Any]        =       inputs.clone()
          __snake_case    ,   __snake_case   :				Optional[Any]        =       inputs.abs().flatten().sort(descending=UpperCAmelCase   )
          __snake_case   :				List[Any]        =       int(threshold * inputs.numel()   )
  
          # flat_out and mask access the same memory.
          __snake_case   :				Optional[Any]        =       mask.flatten()
          __snake_case   :				Tuple        =       0
          __snake_case   :				Any        =       1
          return mask
  
  