






'''simple docstring'''





import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)





class 		SCREAMING_SNAKE_CASE__   :



		def __init__(    self:    Any				,   a:    List[Any])  ->Optional[Any]:






								'''simple docstring'''


								a_				     =   foo




		def __call__(    self:    Optional[Any])  ->str:






								'''simple docstring'''


								return self.foo





class 		SCREAMING_SNAKE_CASE__   (     datasets.Dataset    ):



		@fingerprint_transform(inplace=a)
		def _lowerCAmelCase     (    self:    List[str]				,   a:    Tuple				,   *a:    Optional[Any]				,   **a:    Optional[Any])  ->Union[str, Any]:






								'''simple docstring'''


								return DatasetChild(self.data				,   fingerprint=a)




		@fingerprint_transform(inplace=a)
		def _lowerCAmelCase     (    self:    Optional[Any]				,   a:    Tuple				,   *a:    Union[str, Any]				,   **a:    Optional[int])  ->Optional[Any]:






								'''simple docstring'''


								return DatasetChild(self.data				,   fingerprint=a)





class 		SCREAMING_SNAKE_CASE__   :



		def __init__(    self:    List[str]				,   a:    Optional[Any])  ->List[Any]:






								'''simple docstring'''


								a_				     =   callable



		def __call__(    self:    List[str]				,   *a:    Union[str, Any]				,   **a:    Tuple)  ->int:






								'''simple docstring'''


								if self.callable is not None:
														return self.callable(*a				,   **a)




		def __getstate__(    self:    str)  ->str:






								'''simple docstring'''


								raise pickle.PicklingError()





class 		SCREAMING_SNAKE_CASE__   (     lowercase_    ):



		@require_transformers
		@pytest.mark.integration
		def _lowerCAmelCase     (    self:    List[str])  ->Tuple:






								'''simple docstring'''


								from transformers import AutoTokenizer

								def encode(a:    Union[str, Any]):
														return tokenizer(a)

								# TODO: add hash consistency tests across sessions
								a_				     =   AutoTokenizer.from_pretrained("bert-base-uncased")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(lambda a: tokenizer(a))).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   AutoTokenizer.from_pretrained("bert-base-cased")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(lambda a: tokenizer(a))).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   AutoTokenizer.from_pretrained("bert-base-uncased")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(lambda a: tokenizer(a))).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		@require_transformers
		@pytest.mark.integration
		def _lowerCAmelCase     (    self:    str)  ->Dict:






								'''simple docstring'''


								from transformers import AutoTokenizer

								a_				     =   AutoTokenizer.from_pretrained("gpt2")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								tokenizer("Hello world !")  # call once to change the tokenizer's cache
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)




		@require_regex
		def _lowerCAmelCase     (    self:    int)  ->Tuple:






								'''simple docstring'''


								import regex

								a_				     =   regex.Regex("foo")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   regex.Regex("bar")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   regex.Regex("foo")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)





class 		SCREAMING_SNAKE_CASE__   (     lowercase_    ):



		def _lowerCAmelCase     (    self:    Optional[int])  ->int:






								'''simple docstring'''


								def func():
														return foo

								a_				     =   [0]
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   [1]
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   [0]
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		def _lowerCAmelCase     (    self:    Optional[Any])  ->Optional[int]:






								'''simple docstring'''


								def func():
														pass

								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()

								def func():
														pass

								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)



		def _lowerCAmelCase     (    self:    Optional[int])  ->str:






								'''simple docstring'''


								a_				     =   mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(Foo([1]))).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(Foo([0]))).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		def _lowerCAmelCase     (    self:    str)  ->Union[str, Any]:






								'''simple docstring'''


								a_				     =   mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(Foo([1]).__call__)).hexdigest()
								a_				     =   mda(datasets.utils.py_utils.dumps(Foo([0]).__call__)).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		def _lowerCAmelCase     (    self:    Optional[int])  ->Union[str, Any]:






								'''simple docstring'''


								a_				     =   (
								    "co_argcount",
								    "co_kwonlyargcount",
								    "co_nlocals",
								    "co_stacksize",
								    "co_flags",
								    "co_code",
								    "co_consts",
								    "co_names",
								    "co_varnames",
								    "co_filename",
								    "co_name",
								    "co_firstlineno",
								    "co_lnotab",
								    "co_freevars",
								    "co_cellvars",
								)

								def _create_code_pyaa(*a:    str):
														if hasattr(args[-3]				,   "encode"):
																				a_				     =   list(a)
																				a_				     =   args[-3].encode()  # co_lnotab
																				a_				     =   args[-10].encode()  # co_code
														if hasattr(a				,   "co_posonlyargcount"):
																				if len(a) == 16:
																										return CodeType(*a)
																				elif len(a) == 15:
																										return CodeType(args[0]				,   0				,   *args[1:])
																				return CodeType(args[0]				,   0				,   0				,   *args[1:])
														elif hasattr(a				,   "co_kwonlyargcount"):
																				if len(a) == 16:
																										return CodeType(args[0]				,   *args[2:])
																				elif len(a) == 15:
																										return CodeType(*a)
																				return CodeType(args[0]				,   0				,   *args[1:])
														if len(a) == 16:
																				return CodeType(args[0]				,   *args[3:])
														elif len(a) == 15:
																				return CodeType(args[0]				,   *args[2:])
														return CodeType(*a)

								def create_ipython_func(a:    List[Any]				,   a:    str):
														def func():
																				return returned_obj

														a_				     =   func.__code__
														# Use _create_code from dill in order to make it work for different python versions
														if datasets.config.PY_VERSION < version.parse("3.8"):
																				a_				     =   _create_code_pyaa(
																				    *[getattr(a				,   a) if k != "co_filename" else co_filename for k in code_args_pyaa])
														else:
																				a_				     =   code.replace(co_filename=a)
														return FunctionType(a				,   func.__globals__				,   func.__name__				,   func.__defaults__				,   func.__closure__)

								a_     ,       a_				     =   "<ipython-input-2-e0383a102aae>", [0]
								a_				     =   mda(datasets.utils.py_utils.dumps(create_ipython_func(a				,   a))).hexdigest()
								a_     ,       a_				     =   "<ipython-input-2-e0383a102aae>", [1]
								a_				     =   mda(datasets.utils.py_utils.dumps(create_ipython_func(a				,   a))).hexdigest()
								a_     ,       a_				     =   "<ipython-input-5-713f6613acf3>", [0]
								a_				     =   mda(datasets.utils.py_utils.dumps(create_ipython_func(a				,   a))).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)

								a_     ,       a_				     =   os.path.join(gettempdir()				,   "ipykernel_12345"				,   "321456789.py"), [0]
								a_				     =   mda(datasets.utils.py_utils.dumps(create_ipython_func(a				,   a))).hexdigest()
								a_     ,       a_				     =   os.path.join(gettempdir()				,   "ipykernel_12345"				,   "321456789.py"), [1]
								a_				     =   mda(datasets.utils.py_utils.dumps(create_ipython_func(a				,   a))).hexdigest()
								a_     ,       a_				     =   os.path.join(gettempdir()				,   "ipykernel_12345"				,   "654123987.py"), [0]
								a_				     =   mda(datasets.utils.py_utils.dumps(create_ipython_func(a				,   a))).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)




		def _lowerCAmelCase     (    self:    str)  ->Optional[Any]:






								'''simple docstring'''


								a_     ,       a_				     =   [0], [1]

								def func():
														return foo, bar

								a_				     =   "__main__"

								def globalvars_mocka_side_effect(a:    str				,   *a:    str				,   **a:    Optional[int]):
														return {"foo": foo, "bar": bar}

								def globalvars_mocka_side_effect(a:    Optional[Any]				,   *a:    Optional[int]				,   **a:    Optional[Any]):
														return {"bar": bar, "foo": foo}

								with patch("dill.detect.globalvars"				,   side_effect=a) as globalvars_mocka:
														a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
														self.assertGreater(globalvars_mocka.call_count				,   0)
								with patch("dill.detect.globalvars"				,   side_effect=a) as globalvars_mocka:
														a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
														self.assertGreater(globalvars_mocka.call_count				,   0)
								self.assertEqual(a				,   a)





class 		SCREAMING_SNAKE_CASE__   (     lowercase_    ):



		def _lowerCAmelCase     (    self:    List[str])  ->Optional[int]:






								'''simple docstring'''


								a_				     =   Hasher.hash("hello")
								a_				     =   Hasher.hash("hello")
								a_				     =   Hasher.hash("there")
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		def _lowerCAmelCase     (    self:    List[Any])  ->List[str]:






								'''simple docstring'''


								a_				     =   Hasher.hash(Foo("hello"))
								a_				     =   Hasher.hash(Foo("hello"))
								a_				     =   Hasher.hash(Foo("there"))
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		def _lowerCAmelCase     (    self:    Optional[int])  ->List[str]:






								'''simple docstring'''


								a_				     =   Hasher()
								for x in ["hello", Foo("hello")]:
														hasher.update(a)
								a_				     =   hasher.hexdigest()
								a_				     =   Hasher()
								for x in ["hello", Foo("hello")]:
														hasher.update(a)
								a_				     =   hasher.hexdigest()
								a_				     =   Hasher()
								for x in ["there", Foo("there")]:
														hasher.update(a)
								a_				     =   hasher.hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		def _lowerCAmelCase     (    self:    int)  ->Any:






								'''simple docstring'''


								with self.assertRaises(pickle.PicklingError):
														Hasher.hash(UnpicklableCallable(Foo("hello")))



		def _lowerCAmelCase     (    self:    int)  ->Optional[int]:






								'''simple docstring'''


								a_				     =   "abc"
								a_				     =   [string, string]  # two strings have the same ids
								a_				     =   [string, string]
								a_				     =   json.loads(f"""[\"{string}\", \"{string}\"]""")  # two strings have different ids
								self.assertIs(obja[0]				,   a)
								self.assertIs(obja[0]				,   obja[1])
								self.assertIs(obja[0]				,   a)
								self.assertIs(obja[0]				,   obja[1])
								self.assertIsNot(obja[0]				,   a)
								self.assertIsNot(obja[0]				,   obja[1])
								a_				     =   Hasher.hash(a)
								a_				     =   Hasher.hash(a)
								a_				     =   Hasher.hash(a)
								self.assertEqual(a				,   a)
								self.assertEqual(a				,   a)



		@require_tiktoken
		def _lowerCAmelCase     (    self:    Optional[int])  ->List[Any]:






								'''simple docstring'''


								import tiktoken

								a_				     =   tiktoken.get_encoding("gpt2")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   tiktoken.get_encoding("r50k_base")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   tiktoken.get_encoding("gpt2")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)



		@require_torch
		def _lowerCAmelCase     (    self:    str)  ->str:






								'''simple docstring'''


								import torch

								a_				     =   torch.tensor([1.0])
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   torch.tensor([2.0])
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   torch.tensor([1.0])
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)




		@require_spacy
		@require_spacy_model("en_core_web_sm")
		@require_spacy_model("fr_core_news_sm")
		@pytest.mark.integration
		def _lowerCAmelCase     (    self:    int)  ->List[str]:






								'''simple docstring'''


								import spacy

								a_				     =   spacy.load("en_core_web_sm")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   spacy.load("fr_core_news_sm")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								a_				     =   spacy.load("en_core_web_sm")
								a_				     =   mda(datasets.utils.py_utils.dumps(a)).hexdigest()
								self.assertEqual(a				,   a)
								self.assertNotEqual(a				,   a)





@pytest.mark.integration
def 			__UpperCAmelCase				(lowercase__				)		->				Any:

						'''simple docstring'''


						a_				     =   tmp_path / "dir1"
						a_				     =   tmp_path / "dir2"
						dira.mkdir()
						dira.mkdir()
						a_				     =   "script.py"
						a_				     =   dedent(
						    "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "				)
						a_				     =   dira / script_filename
						a_				     =   dira / script_filename
						with script_patha.open("w"				) as f:
												f.write(lowercase__				)
						with script_patha.open("w"				) as f:
												f.write(lowercase__				)
						a_				     =   subprocess.check_output(["python", str(lowercase__				)]				)
						a_				     =   subprocess.check_output(["python", str(lowercase__				)]				)
						assert fingerprinta == fingerprinta



def 			__UpperCAmelCase				()		->				List[str]:

						'''simple docstring'''


						a_				     =   {"a": [0, 1, 2]}
						a_				     =   DatasetChild(InMemoryTable.from_pydict(lowercase__				)				)
						a_				     =   dataset.funca()._fingerprint
						assert expected_fingerprint == dataset.funca()._fingerprint
						assert expected_fingerprint != dataset.funca()._fingerprint

						with Pool(2				) as p:
												assert expected_fingerprint == p.apply_async(dataset.funca				).get()._fingerprint
												assert expected_fingerprint != p.apply_async(dataset.funca				).get()._fingerprint



def 			__UpperCAmelCase				()		->				Dict:

						'''simple docstring'''


						a_				     =   {"a": [0, 1, 2]}





						class 		SCREAMING_SNAKE_CASE__   (     datasets.Dataset    ):



								@fingerprint_transform(inplace=a)
								def _lowerCAmelCase     (    self:    Optional[Any]				,   a:    Dict)  ->List[Any]:






														'''simple docstring'''


														return DummyDatasetChild(self.data				,   fingerprint=a)





						a_				     =   DummyDatasetChild(InMemoryTable.from_pydict(lowercase__				)				).func()





						class 		SCREAMING_SNAKE_CASE__   (     datasets.Dataset    ):



								@fingerprint_transform(inplace=a				,   version="1.0.0")
								def _lowerCAmelCase     (    self:    str				,   a:    Optional[int])  ->List[str]:






														'''simple docstring'''


														return DummyDatasetChild(self.data				,   fingerprint=a)





						a_				     =   DummyDatasetChild(InMemoryTable.from_pydict(lowercase__				)				).func()





						class 		SCREAMING_SNAKE_CASE__   (     datasets.Dataset    ):



								@fingerprint_transform(inplace=a				,   version="2.0.0")
								def _lowerCAmelCase     (    self:    Dict				,   a:    Dict)  ->Dict:






														'''simple docstring'''


														return DummyDatasetChild(self.data				,   fingerprint=a)





						a_				     =   DummyDatasetChild(InMemoryTable.from_pydict(lowercase__				)				).func()

						assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}				) == 3



def 			__UpperCAmelCase				()		->				List[str]:

						'''simple docstring'''


						a_				     =   Hasher()
						hasher.update(lambda lowercase__				: x				)











'''simple docstring'''





from __future__ import annotations

a_      = [2, 1, 4, 5, 6, 0, 8, 9, 1, 2, 0, 6, 4, 2, 0, 6, 5, 3, 2, 7]





class 		SCREAMING_SNAKE_CASE__   :



		def __init__(    self:    Optional[int]				,   a:    int)  ->None:






								'''simple docstring'''


								a_				     =   -1
								a_				     =   -1
								a_				     =   [-1] * length
								a_				     =   None
								a_				     =   None




		def __repr__(    self:    Union[str, Any])  ->str:






								'''simple docstring'''


								return f"""Node(min_value={self.minn} max_value={self.maxx})"""





def 			__UpperCAmelCase				(lowercase__				)		->				Node | None:

						'''simple docstring'''


						a_				     =   Node(len(lowercase__				)				)
						a_     ,       a_				     =   min(lowercase__				), max(lowercase__				)
						# Leaf node case where the node contains only one unique value
						if root.minn == root.maxx:
												return root
						a_				     =   (root.minn + root.maxx) // 2

						a_				     =   []
						a_				     =   []

						for index, num in enumerate(lowercase__				):
												if num <= pivot:
																		left_arr.append(lowercase__				)
												else:
																		right_arr.append(lowercase__				)
												a_				     =   len(lowercase__				)
						a_				     =   build_tree(lowercase__				)
						a_				     =   build_tree(lowercase__				)
						return root



def 			__UpperCAmelCase				(lowercase__	,lowercase__	,lowercase__				)		->				int:

						'''simple docstring'''


						if index < 0 or node is None:
												return 0
						# Leaf node cases
						if node.minn == node.maxx:
												return index + 1 if node.minn == num else 0
						a_				     =   (node.minn + node.maxx) // 2
						if num <= pivot:
												# go the left subtree and map index to the left subtree
												return rank_till_index(node.left	,lowercase__	,node.map_left[index] - 1				)
						else:
												# go to the right subtree and map index to the right subtree
												return rank_till_index(node.right	,lowercase__	,index - node.map_left[index]				)



def 			__UpperCAmelCase				(lowercase__	,lowercase__	,lowercase__	,lowercase__				)		->				int:

						'''simple docstring'''


						if start > end:
												return 0
						a_				     =   rank_till_index(lowercase__	,lowercase__	,lowercase__				)
						a_				     =   rank_till_index(lowercase__	,lowercase__	,start - 1				)
						return rank_till_end - rank_before_start



def 			__UpperCAmelCase				(lowercase__	,lowercase__	,lowercase__	,lowercase__				)		->				int:

						'''simple docstring'''


						if index > (end - start) or start > end or node is None:
												return -1
						# Leaf node case
						if node.minn == node.maxx:
												return node.minn
						# Number of elements in the left subtree in interval [start, end]
						a_				     =   node.map_left[end] - (
						    node.map_left[start - 1] if start else 0
						)
						if num_elements_in_left_tree > index:
												return quantile(
												    node.left	,lowercase__	,(node.map_left[start - 1] if start else 0)	,node.map_left[end] - 1	,)
						else:
												return quantile(
												    node.right	,index - num_elements_in_left_tree	,start - (node.map_left[start - 1] if start else 0)	,end - node.map_left[end]	,)



def 			__UpperCAmelCase				(lowercase__	,lowercase__	,lowercase__	,lowercase__	,lowercase__				)		->				int:

						'''simple docstring'''


						if (
						    start > end
						    or node is None
						    or start_num > end_num
						    or node.minn > end_num
						    or node.maxx < start_num
						):
												return 0
						if start_num <= node.minn and node.maxx <= end_num:
												return end - start + 1
						a_				     =   range_counting(
						    node.left	,(node.map_left[start - 1] if start else 0)	,node.map_left[end] - 1	,lowercase__	,lowercase__	,)
						a_				     =   range_counting(
						    node.right	,start - (node.map_left[start - 1] if start else 0)	,end - node.map_left[end]	,lowercase__	,lowercase__	,)
						return left + right


if __name__ == "__main__":
							import doctest

							doctest.testmod()



