
'''simple docstring'''





import os
from argparse import ArgumentParser
from pathlib import Path
from shutil import copyfile
from typing import List

from datasets import config
from datasets.builder import DatasetBuilder
from datasets.commands import BaseDatasetsCLICommand
from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadMode
from datasets.load import dataset_module_factory, import_main_class
from datasets.utils.info_utils import VerificationMode
def   snake_case_       (							_a     :							Dict ,					**_a     :							List[str]			):
						return RunBeamCommand(
						    args.dataset ,					args.name ,					args.cache_dir ,					args.beam_pipeline_options ,					args.data_dir ,					args.all_configs ,					args.save_info or args.save_infos ,					args.ignore_verifications ,					args.force_redownload ,					**_a ,					)




class  lowerCAmelCase__   (       _a       ):



		'''simple docstring'''






		@staticmethod
		def 		_lowerCamelCase	(lowerCAmelCase_  : Tuple    )   ->       Optional[int]:
								snake_case_						       =							parser.add_parser("run_beam"	, help="Run a Beam dataset processing pipeline"    )
								run_beam_parser.add_argument("dataset"	, type=lowerCamelCase__	, help="Name of the dataset to download"    )
								run_beam_parser.add_argument("--name"	, type=lowerCamelCase__	, default=lowerCamelCase__	, help="Dataset config name"    )
								run_beam_parser.add_argument(
								    "--cache_dir"	, type=lowerCamelCase__	, default=lowerCamelCase__	, help="Cache directory where the datasets are stored"	, )
								run_beam_parser.add_argument(
								    "--beam_pipeline_options"	, type=lowerCamelCase__	, default=""	, help="Beam pipeline options, separated by commas. Example:: `--beam_pipeline_options=job_name=my-job,project=my-project`"	, )
								run_beam_parser.add_argument(
								    "--data_dir"	, type=lowerCamelCase__	, default=lowerCamelCase__	, help="Can be used to specify a manual directory to get the files from"	, )
								run_beam_parser.add_argument("--all_configs"	, action="store_true"	, help="Test all dataset configurations"    )
								run_beam_parser.add_argument("--save_info"	, action="store_true"	, help="Save the dataset infos file"    )
								run_beam_parser.add_argument(
								    "--ignore_verifications"	, action="store_true"	, help="Run the test without checksums and splits checks"    )
								run_beam_parser.add_argument("--force_redownload"	, action="store_true"	, help="Force dataset redownload"    )
								# aliases
								run_beam_parser.add_argument("--save_infos"	, action="store_true"	, help="alias for save_info"    )
								run_beam_parser.set_defaults(func=lowerCamelCase__    )

		def __init__(self  : Any	, lowerCAmelCase_  : Dict	, lowerCAmelCase_  : Union[str, Any]	, lowerCAmelCase_  : Optional[int]	, lowerCAmelCase_  : Any	, lowerCAmelCase_  : Tuple	, lowerCAmelCase_  : List[Any]	, lowerCAmelCase_  : str	, lowerCAmelCase_  : List[Any]	, lowerCAmelCase_  : List[Any]	, **lowerCAmelCase_  : Optional[Any]	, )   ->       Optional[int]:
								snake_case_						       =							dataset
								snake_case_						       =							name
								snake_case_						       =							cache_dir
								snake_case_						       =							beam_pipeline_options
								snake_case_						       =							data_dir
								snake_case_						       =							all_configs
								snake_case_						       =							save_infos
								snake_case_						       =							ignore_verifications
								snake_case_						       =							force_redownload
								snake_case_						       =							config_kwargs


		def 		_lowerCamelCase	(self  : str    )   ->       int:
								import apache_beam as beam

								if self._name is not None and self._all_configs:
														print("Both parameters `name` and `all_configs` can't be used at once."    )
														exit(1    )
								snake_case_						       =							self._dataset, self._name
								snake_case_						       =							dataset_module_factory(lowerCamelCase__    )
								snake_case_						       =							import_main_class(dataset_module.module_path    )
								snake_case_						       =							[]
								if self._beam_pipeline_options:
														snake_case_						       =							beam.options.pipeline_options.PipelineOptions(
														    flags=[f'''--{opt.strip()}''' for opt in self._beam_pipeline_options.split(","    ) if opt]    )
								else:
														snake_case_						       =							None
								if self._all_configs and len(builder_cls.BUILDER_CONFIGS    ) > 0:
														for builder_config in builder_cls.BUILDER_CONFIGS:
																				builders.append(
																				    builder_cls(
																				        config_name=builder_config.name	, data_dir=self._data_dir	, hash=dataset_module.hash	, beam_options=lowerCamelCase__	, cache_dir=self._cache_dir	, base_path=dataset_module.builder_kwargs.get("base_path"    )	, )    )
								else:
														builders.append(
														    builder_cls(
														        config_name=lowerCamelCase__	, data_dir=self._data_dir	, beam_options=lowerCamelCase__	, cache_dir=self._cache_dir	, base_path=dataset_module.builder_kwargs.get("base_path"    )	, **self._config_kwargs	, )    )

								for builder in builders:
														builder.download_and_prepare(
														    download_mode=DownloadMode.REUSE_CACHE_IF_EXISTS
														    if not self._force_redownload
														    else DownloadMode.FORCE_REDOWNLOAD	, download_config=DownloadConfig(cache_dir=config.DOWNLOADED_DATASETS_PATH    )	, verification_mode=VerificationMode.NO_CHECKS
														    if self._ignore_verifications
														    else VerificationMode.ALL_CHECKS	, try_from_hf_gcs=lowerCamelCase__	, )
														if self._save_infos:
																				builder._save_infos()

								print("Apache beam run successful."    )

								# If save_infos=True, the dataset infos file is created next to the loaded module file.
								# Let's move it to the original directory of the dataset script, to allow the user to
								# upload them on S3 at the same time afterwards.
								if self._save_infos:
														snake_case_						       =							os.path.join(builder_cls.get_imported_module_dir()	, config.DATASETDICT_INFOS_FILENAME    )

														snake_case_						       =							Path(lowerCamelCase__    ).name + ".py"

														snake_case_						       =							os.path.join(lowerCamelCase__	, lowerCamelCase__    )
														if os.path.isfile(lowerCamelCase__    ):
																				snake_case_						       =							os.path.dirname(lowerCamelCase__    )
														elif os.path.isfile(lowerCamelCase__    ):
																				snake_case_						       =							path
														else:  # in case of a remote dataset
																				print(f'''Dataset Infos file saved at {dataset_infos_path}'''    )
																				exit(1    )

														# Move datasetinfo back to the user
														snake_case_						       =							os.path.join(lowerCamelCase__	, config.DATASETDICT_INFOS_FILENAME    )
														copyfile(lowerCamelCase__	, lowerCamelCase__    )
														print(f'''Dataset Infos file saved at {user_dataset_infos_path}'''    )



'''simple docstring'''





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)



class  lowerCAmelCase__   :



  '''simple docstring'''






  def __init__(self  : int	, lowerCAmelCase_  : Optional[Any]    )   ->       Tuple:
        snake_case_						       =							foo


  def __call__(self  : List[Any]    )   ->       Optional[int]:
        return self.foo



class  lowerCAmelCase__   (       datasets.Dataset       ):



  '''simple docstring'''






  @fingerprint_transform(inplace=lowerCAmelCase_    )
  def 		_lowerCamelCase	(self  : Tuple	, lowerCAmelCase_  : Any	, *lowerCAmelCase_  : Union[str, Any]	, **lowerCAmelCase_  : Any    )   ->       str:
        return DatasetChild(self.data	, fingerprint=lowerCAmelCase_    )


  @fingerprint_transform(inplace=lowerCAmelCase_    )
  def 		_lowerCamelCase	(self  : str	, lowerCAmelCase_  : Optional[Any]	, *lowerCAmelCase_  : List[Any]	, **lowerCAmelCase_  : Any    )   ->       List[str]:
        return DatasetChild(self.data	, fingerprint=lowerCAmelCase_    )



class  lowerCAmelCase__   :



  '''simple docstring'''






  def __init__(self  : int	, lowerCAmelCase_  : List[str]    )   ->       Dict:
        snake_case_						       =							callable

  def __call__(self  : List[str]	, *lowerCAmelCase_  : Any	, **lowerCAmelCase_  : Tuple    )   ->       Dict:
        if self.callable is not None:
              return self.callable(*lowerCAmelCase_	, **lowerCAmelCase_    )


  def __getstate__(self  : List[str]    )   ->       Tuple:
        raise pickle.PicklingError()



class  lowerCAmelCase__   (       UpperCamelCase       ):



  '''simple docstring'''






  @require_transformers
  @pytest.mark.integration
  def 		_lowerCamelCase	(self  : str    )   ->       str:
        from transformers import AutoTokenizer

        def encode(lowerCAmelCase_  : str    ):
              return tokenizer(lowerCAmelCase_    )

        # TODO: add hash consistency tests across sessions
        snake_case_						       =							AutoTokenizer.from_pretrained("bert-base-uncased"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lambda lowerCAmelCase_    : tokenizer(lowerCAmelCase_    )    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							AutoTokenizer.from_pretrained("bert-base-cased"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lambda lowerCAmelCase_    : tokenizer(lowerCAmelCase_    )    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							AutoTokenizer.from_pretrained("bert-base-uncased"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lambda lowerCAmelCase_    : tokenizer(lowerCAmelCase_    )    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  @require_transformers
  @pytest.mark.integration
  def 		_lowerCamelCase	(self  : Optional[Any]    )   ->       Optional[int]:
        from transformers import AutoTokenizer

        snake_case_						       =							AutoTokenizer.from_pretrained("gpt2"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        tokenizer("Hello world !"    )  # call once to change the tokenizer's cache
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )


  @require_regex
  def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       Optional[int]:
        import regex

        snake_case_						       =							regex.Regex("foo"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							regex.Regex("bar"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							regex.Regex("foo"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )



class  lowerCAmelCase__   (       UpperCamelCase       ):



  '''simple docstring'''






  def 		_lowerCamelCase	(self  : Dict    )   ->       Optional[Any]:
        def func():
              return foo

        snake_case_						       =							[0]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							[1]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							[0]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  def 		_lowerCamelCase	(self  : str    )   ->       Tuple:
        def func():
              pass

        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()

        def func():
              pass

        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  def 		_lowerCamelCase	(self  : List[Any]    )   ->       Union[str, Any]:
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    )    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([1]    )    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    )    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       int:
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    ).__call__    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([1]    ).__call__    )    ).hexdigest()
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    ).__call__    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  def 		_lowerCamelCase	(self  : int    )   ->       Optional[Any]:
        snake_case_						       =							(
            "co_argcount",
            "co_kwonlyargcount",
            "co_nlocals",
            "co_stacksize",
            "co_flags",
            "co_code",
            "co_consts",
            "co_names",
            "co_varnames",
            "co_filename",
            "co_name",
            "co_firstlineno",
            "co_lnotab",
            "co_freevars",
            "co_cellvars",
        )

        def _create_code_pyaa(*lowerCAmelCase_  : Any    ):
              if hasattr(args[-3]	, "encode"    ):
                    snake_case_						       =							list(lowerCAmelCase_    )
                    snake_case_						       =							args[-3].encode()  # co_lnotab
                    snake_case_						       =							args[-10].encode()  # co_code
              if hasattr(lowerCAmelCase_	, "co_posonlyargcount"    ):
                    if len(lowerCAmelCase_    ) == 16:
                          return CodeType(*lowerCAmelCase_    )
                    elif len(lowerCAmelCase_    ) == 15:
                          return CodeType(args[0]	, 0	, *args[1:]    )
                    return CodeType(args[0]	, 0	, 0	, *args[1:]    )
              elif hasattr(lowerCAmelCase_	, "co_kwonlyargcount"    ):
                    if len(lowerCAmelCase_    ) == 16:
                          return CodeType(args[0]	, *args[2:]    )
                    elif len(lowerCAmelCase_    ) == 15:
                          return CodeType(*lowerCAmelCase_    )
                    return CodeType(args[0]	, 0	, *args[1:]    )
              if len(lowerCAmelCase_    ) == 16:
                    return CodeType(args[0]	, *args[3:]    )
              elif len(lowerCAmelCase_    ) == 15:
                    return CodeType(args[0]	, *args[2:]    )
              return CodeType(*lowerCAmelCase_    )

        def create_ipython_func(lowerCAmelCase_  : Tuple	, lowerCAmelCase_  : Any    ):
              def func():
                    return returned_obj

              snake_case_						       =							func.__code__
              # Use _create_code from dill in order to make it work for different python versions
              if datasets.config.PY_VERSION < version.parse("3.8"    ):
                    snake_case_						       =							_create_code_pyaa(
                        *[getattr(lowerCAmelCase_	, lowerCAmelCase_    ) if k != "co_filename" else co_filename for k in code_args_pyaa]    )
              else:
                    snake_case_						       =							code.replace(co_filename=lowerCAmelCase_    )
              return FunctionType(lowerCAmelCase_	, func.__globals__	, func.__name__	, func.__defaults__	, func.__closure__    )

        snake_case_       ,	snake_case_						       =							"<ipython-input-2-e0383a102aae>", [0]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
        snake_case_       ,	snake_case_						       =							"<ipython-input-2-e0383a102aae>", [1]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
        snake_case_       ,	snake_case_						       =							"<ipython-input-5-713f6613acf3>", [0]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

        snake_case_       ,	snake_case_						       =							os.path.join(gettempdir()	, "ipykernel_12345"	, "321456789.py"    ), [0]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
        snake_case_       ,	snake_case_						       =							os.path.join(gettempdir()	, "ipykernel_12345"	, "321456789.py"    ), [1]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
        snake_case_       ,	snake_case_						       =							os.path.join(gettempdir()	, "ipykernel_12345"	, "654123987.py"    ), [0]
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )


  def 		_lowerCamelCase	(self  : int    )   ->       List[Any]:
        snake_case_       ,	snake_case_						       =							[0], [1]

        def func():
              return foo, bar

        snake_case_						       =							"__main__"

        def globalvars_mocka_side_effect(lowerCAmelCase_  : str	, *lowerCAmelCase_  : Tuple	, **lowerCAmelCase_  : List[Any]    ):
              return {"foo": foo, "bar": bar}

        def globalvars_mocka_side_effect(lowerCAmelCase_  : List[str]	, *lowerCAmelCase_  : List[str]	, **lowerCAmelCase_  : Optional[Any]    ):
              return {"bar": bar, "foo": foo}

        with patch("dill.detect.globalvars"	, side_effect=lowerCAmelCase_    ) as globalvars_mocka:
              snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
              self.assertGreater(globalvars_mocka.call_count	, 0    )
        with patch("dill.detect.globalvars"	, side_effect=lowerCAmelCase_    ) as globalvars_mocka:
              snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
              self.assertGreater(globalvars_mocka.call_count	, 0    )
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )



class  lowerCAmelCase__   (       UpperCamelCase       ):



  '''simple docstring'''






  def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       Tuple:
        snake_case_						       =							Hasher.hash("hello"    )
        snake_case_						       =							Hasher.hash("hello"    )
        snake_case_						       =							Hasher.hash("there"    )
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  def 		_lowerCamelCase	(self  : Any    )   ->       int:
        snake_case_						       =							Hasher.hash(Foo("hello"    )    )
        snake_case_						       =							Hasher.hash(Foo("hello"    )    )
        snake_case_						       =							Hasher.hash(Foo("there"    )    )
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  def 		_lowerCamelCase	(self  : List[Any]    )   ->       List[Any]:
        snake_case_						       =							Hasher()
        for x in ["hello", Foo("hello"    )]:
              hasher.update(lowerCAmelCase_    )
        snake_case_						       =							hasher.hexdigest()
        snake_case_						       =							Hasher()
        for x in ["hello", Foo("hello"    )]:
              hasher.update(lowerCAmelCase_    )
        snake_case_						       =							hasher.hexdigest()
        snake_case_						       =							Hasher()
        for x in ["there", Foo("there"    )]:
              hasher.update(lowerCAmelCase_    )
        snake_case_						       =							hasher.hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  def 		_lowerCamelCase	(self  : str    )   ->       List[Any]:
        with self.assertRaises(pickle.PicklingError    ):
              Hasher.hash(UnpicklableCallable(Foo("hello"    )    )    )

  def 		_lowerCamelCase	(self  : Dict    )   ->       Optional[int]:
        snake_case_						       =							"abc"
        snake_case_						       =							[string, string]  # two strings have the same ids
        snake_case_						       =							[string, string]
        snake_case_						       =							json.loads(f'''["{string}", "{string}"]'''    )  # two strings have different ids
        self.assertIs(obja[0]	, lowerCAmelCase_    )
        self.assertIs(obja[0]	, obja[1]    )
        self.assertIs(obja[0]	, lowerCAmelCase_    )
        self.assertIs(obja[0]	, obja[1]    )
        self.assertIsNot(obja[0]	, lowerCAmelCase_    )
        self.assertIsNot(obja[0]	, obja[1]    )
        snake_case_						       =							Hasher.hash(lowerCAmelCase_    )
        snake_case_						       =							Hasher.hash(lowerCAmelCase_    )
        snake_case_						       =							Hasher.hash(lowerCAmelCase_    )
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  @require_tiktoken
  def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       Union[str, Any]:
        import tiktoken

        snake_case_						       =							tiktoken.get_encoding("gpt2"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							tiktoken.get_encoding("r50k_base"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							tiktoken.get_encoding("gpt2"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

  @require_torch
  def 		_lowerCamelCase	(self  : Optional[int]    )   ->       str:
        import torch

        snake_case_						       =							torch.tensor([1.0]    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							torch.tensor([2.0]    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							torch.tensor([1.0]    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )


  @require_spacy
  @require_spacy_model("en_core_web_sm"    )
  @require_spacy_model("fr_core_news_sm"    )
  @pytest.mark.integration
  def 		_lowerCamelCase	(self  : int    )   ->       Union[str, Any]:
        import spacy

        snake_case_						       =							spacy.load("en_core_web_sm"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							spacy.load("fr_core_news_sm"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        snake_case_						       =							spacy.load("en_core_web_sm"    )
        snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
        self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
        self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )

@pytest.mark.integration
def   snake_case_       (							_a     :							Path			):
      snake_case_						       =							tmp_path / "dir1"
      snake_case_						       =							tmp_path / "dir2"
      dira.mkdir()
      dira.mkdir()
      snake_case_						       =							"script.py"
      snake_case_						       =							dedent(
          "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "			)
      snake_case_						       =							dira / script_filename
      snake_case_						       =							dira / script_filename
      with script_patha.open("w"			) as f:
            f.write(_a			)
      with script_patha.open("w"			) as f:
            f.write(_a			)
      snake_case_						       =							subprocess.check_output(["python", str(_a			)]			)
      snake_case_						       =							subprocess.check_output(["python", str(_a			)]			)
      assert fingerprinta == fingerprinta
def   snake_case_       (							):
      snake_case_						       =							{"a": [0, 1, 2]}
      snake_case_						       =							DatasetChild(InMemoryTable.from_pydict(_a			)			)
      snake_case_						       =							dataset.funca()._fingerprint
      assert expected_fingerprint == dataset.funca()._fingerprint
      assert expected_fingerprint != dataset.funca()._fingerprint

      with Pool(2			) as p:
            assert expected_fingerprint == p.apply_async(dataset.funca			).get()._fingerprint
            assert expected_fingerprint != p.apply_async(dataset.funca			).get()._fingerprint
def   snake_case_       (							):
      snake_case_						       =							{"a": [0, 1, 2]}



      class  lowerCAmelCase__   (       datasets.Dataset       ):



        '''simple docstring'''






        @fingerprint_transform(inplace=lowerCAmelCase_    )
        def 		_lowerCamelCase	(self  : int	, lowerCAmelCase_  : str    )   ->       List[Any]:
              return DummyDatasetChild(self.data	, fingerprint=lowerCAmelCase_    )

      snake_case_						       =							DummyDatasetChild(InMemoryTable.from_pydict(_a			)			).func()



      class  lowerCAmelCase__   (       datasets.Dataset       ):



        '''simple docstring'''






        @fingerprint_transform(inplace=lowerCAmelCase_	, version="1.0.0"    )
        def 		_lowerCamelCase	(self  : Dict	, lowerCAmelCase_  : Any    )   ->       Any:
              return DummyDatasetChild(self.data	, fingerprint=lowerCAmelCase_    )

      snake_case_						       =							DummyDatasetChild(InMemoryTable.from_pydict(_a			)			).func()



      class  lowerCAmelCase__   (       datasets.Dataset       ):



        '''simple docstring'''






        @fingerprint_transform(inplace=lowerCAmelCase_	, version="2.0.0"    )
        def 		_lowerCamelCase	(self  : Union[str, Any]	, lowerCAmelCase_  : Any    )   ->       int:
              return DummyDatasetChild(self.data	, fingerprint=lowerCAmelCase_    )

      snake_case_						       =							DummyDatasetChild(InMemoryTable.from_pydict(_a			)			).func()

      assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}			) == 3




def   snake_case_       (							):
      # AttributeError: module 'dill._dill' has no attribute 'stack'
      snake_case_						       =							Hasher()
      hasher.update(lambda _a			: x			)

