




'''simple docstring'''







import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)





class   lowerCamelCase_							:




					'''simple docstring'''






					def __init__(					self,						__UpperCamelCase				):
							_snake_case			 = foo






					def __call__(					self				):
							return self.foo





class   lowerCamelCase_							( datasets.Dataset	):




					'''simple docstring'''






					@fingerprint_transform(inplace=__UpperCamelCase				)
					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						*__UpperCamelCase,						**__UpperCamelCase				):
							return DatasetChild(self.data,						fingerprint=__UpperCamelCase				)






					@fingerprint_transform(inplace=__UpperCamelCase				)
					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase,						*__UpperCamelCase,						**__UpperCamelCase				):
							return DatasetChild(self.data,						fingerprint=__UpperCamelCase				)





class   lowerCamelCase_							:




					'''simple docstring'''






					def __init__(					self,						__UpperCamelCase				):
							_snake_case			 = callable






					def __call__(					self,						*__UpperCamelCase,						**__UpperCamelCase				):






							if self.callable is not None:
									return self.callable(*__UpperCamelCase,						**__UpperCamelCase				)






					def __getstate__(					self				):
							raise pickle.PicklingError()





class   lowerCamelCase_							( UpperCAmelCase_	):




					'''simple docstring'''






					@require_transformers
					@pytest.mark.integration
					def        __SCREAMING_SNAKE_CASE			(					self				):
							from transformers import AutoTokenizer

							def encode(__UpperCamelCase				):
									return tokenizer(__UpperCamelCase				)

							# TODO: add hash consistency tests across sessions
							_snake_case			 = AutoTokenizer.from_pretrained("bert-base-uncased"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase				: tokenizer(__UpperCamelCase				)				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = AutoTokenizer.from_pretrained("bert-base-cased"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase				: tokenizer(__UpperCamelCase				)				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = AutoTokenizer.from_pretrained("bert-base-uncased"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(lambda __UpperCamelCase				: tokenizer(__UpperCamelCase				)				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					@require_transformers
					@pytest.mark.integration
					def        __SCREAMING_SNAKE_CASE			(					self				):
							from transformers import AutoTokenizer

							_snake_case			 = AutoTokenizer.from_pretrained("gpt2"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							tokenizer("Hello world !"				)  # call once to change the tokenizer's cache
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)






					@require_regex
					def        __SCREAMING_SNAKE_CASE			(					self				):
							import regex

							_snake_case			 = regex.Regex("foo"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = regex.Regex("bar"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = regex.Regex("foo"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)





class   lowerCamelCase_							( UpperCAmelCase_	):




					'''simple docstring'''






					def        __SCREAMING_SNAKE_CASE			(					self				):






							def func():
									return foo

							_snake_case			 = [0]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = [1]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = [0]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):






							def func():
									pass

							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()

							def func():
									pass

							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = mda(datasets.utils.py_utils.dumps(Foo([0]				)				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(Foo([1]				)				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(Foo([0]				)				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = mda(datasets.utils.py_utils.dumps(Foo([0]				).__call__				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(Foo([1]				).__call__				)				).hexdigest()
							_snake_case			 = mda(datasets.utils.py_utils.dumps(Foo([0]				).__call__				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = (
							    "co_argcount",
							    "co_kwonlyargcount",
							    "co_nlocals",
							    "co_stacksize",
							    "co_flags",
							    "co_code",
							    "co_consts",
							    "co_names",
							    "co_varnames",
							    "co_filename",
							    "co_name",
							    "co_firstlineno",
							    "co_lnotab",
							    "co_freevars",
							    "co_cellvars",
							)

							def _create_code_pyaa(*__UpperCamelCase				):
									if hasattr(args[-3],						"encode"				):
											_snake_case			 = list(__UpperCamelCase				)
											_snake_case			 = args[-3].encode()  # co_lnotab
											_snake_case			 = args[-10].encode()  # co_code
									if hasattr(__UpperCamelCase,						"co_posonlyargcount"				):
											if len(__UpperCamelCase				) == 16:
													return CodeType(*__UpperCamelCase				)
											elif len(__UpperCamelCase				) == 15:
													return CodeType(args[0],						0,						*args[1:]				)
											return CodeType(args[0],						0,						0,						*args[1:]				)
									elif hasattr(__UpperCamelCase,						"co_kwonlyargcount"				):
											if len(__UpperCamelCase				) == 16:
													return CodeType(args[0],						*args[2:]				)
											elif len(__UpperCamelCase				) == 15:
													return CodeType(*__UpperCamelCase				)
											return CodeType(args[0],						0,						*args[1:]				)
									if len(__UpperCamelCase				) == 16:
											return CodeType(args[0],						*args[3:]				)
									elif len(__UpperCamelCase				) == 15:
											return CodeType(args[0],						*args[2:]				)
									return CodeType(*__UpperCamelCase				)

							def create_ipython_func(__UpperCamelCase,						__UpperCamelCase				):
									def func():
											return returned_obj

									_snake_case			 = func.__code__
									# Use _create_code from dill in order to make it work for different python versions
									if datasets.config.PY_VERSION < version.parse("3.8"				):
											_snake_case			 = _create_code_pyaa(
											    *[getattr(__UpperCamelCase,						__UpperCamelCase				) if k != "co_filename" else co_filename for k in code_args_pyaa]				)
									else:
											_snake_case			 = code.replace(co_filename=__UpperCamelCase				)
									return FunctionType(__UpperCamelCase,						func.__globals__,						func.__name__,						func.__defaults__,						func.__closure__				)

							_snake_case			 = "<ipython-input-2-e0383a102aae>", [0]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase,						__UpperCamelCase				)				)				).hexdigest()
							_snake_case			 = "<ipython-input-2-e0383a102aae>", [1]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase,						__UpperCamelCase				)				)				).hexdigest()
							_snake_case			 = "<ipython-input-5-713f6613acf3>", [0]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase,						__UpperCamelCase				)				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)

							_snake_case			 = os.path.join(gettempdir(),						"ipykernel_12345",						"321456789.py"				), [0]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase,						__UpperCamelCase				)				)				).hexdigest()
							_snake_case			 = os.path.join(gettempdir(),						"ipykernel_12345",						"321456789.py"				), [1]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase,						__UpperCamelCase				)				)				).hexdigest()
							_snake_case			 = os.path.join(gettempdir(),						"ipykernel_12345",						"654123987.py"				), [0]
							_snake_case			 = mda(datasets.utils.py_utils.dumps(create_ipython_func(__UpperCamelCase,						__UpperCamelCase				)				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = [0], [1]

							def func():
									return foo, bar

							_snake_case			 = "__main__"

							def globalvars_mocka_side_effect(__UpperCamelCase,						*__UpperCamelCase,						**__UpperCamelCase				):
									return {"foo": foo, "bar": bar}

							def globalvars_mocka_side_effect(__UpperCamelCase,						*__UpperCamelCase,						**__UpperCamelCase				):
									return {"bar": bar, "foo": foo}

							with patch("dill.detect.globalvars",						side_effect=__UpperCamelCase				) as globalvars_mocka:
									_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
									self.assertGreater(globalvars_mocka.call_count,						0				)
							with patch("dill.detect.globalvars",						side_effect=__UpperCamelCase				) as globalvars_mocka:
									_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
									self.assertGreater(globalvars_mocka.call_count,						0				)
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)





class   lowerCamelCase_							( UpperCAmelCase_	):




					'''simple docstring'''






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = Hasher.hash("hello"				)
							_snake_case			 = Hasher.hash("hello"				)
							_snake_case			 = Hasher.hash("there"				)
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = Hasher.hash(Foo("hello"				)				)
							_snake_case			 = Hasher.hash(Foo("hello"				)				)
							_snake_case			 = Hasher.hash(Foo("there"				)				)
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = Hasher()
							for x in ["hello", Foo("hello"				)]:
									hasher.update(__UpperCamelCase				)
							_snake_case			 = hasher.hexdigest()
							_snake_case			 = Hasher()
							for x in ["hello", Foo("hello"				)]:
									hasher.update(__UpperCamelCase				)
							_snake_case			 = hasher.hexdigest()
							_snake_case			 = Hasher()
							for x in ["there", Foo("there"				)]:
									hasher.update(__UpperCamelCase				)
							_snake_case			 = hasher.hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					def        __SCREAMING_SNAKE_CASE			(					self				):






							with self.assertRaises(pickle.PicklingError				):
									Hasher.hash(UnpicklableCallable(Foo("hello"				)				)				)






					def        __SCREAMING_SNAKE_CASE			(					self				):
							_snake_case			 = "abc"
							_snake_case			 = [string, string]  # two strings have the same ids
							_snake_case			 = [string, string]
							_snake_case			 = json.loads(f'''[\"{string}\", \"{string}\"]'''				)  # two strings have different ids
							self.assertIs(obja[0],						__UpperCamelCase				)
							self.assertIs(obja[0],						obja[1]				)
							self.assertIs(obja[0],						__UpperCamelCase				)
							self.assertIs(obja[0],						obja[1]				)
							self.assertIsNot(obja[0],						__UpperCamelCase				)
							self.assertIsNot(obja[0],						obja[1]				)
							_snake_case			 = Hasher.hash(__UpperCamelCase				)
							_snake_case			 = Hasher.hash(__UpperCamelCase				)
							_snake_case			 = Hasher.hash(__UpperCamelCase				)
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)






					@require_tiktoken
					def        __SCREAMING_SNAKE_CASE			(					self				):
							import tiktoken

							_snake_case			 = tiktoken.get_encoding("gpt2"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = tiktoken.get_encoding("r50k_base"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = tiktoken.get_encoding("gpt2"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					@require_torch
					def        __SCREAMING_SNAKE_CASE			(					self				):
							import torch

							_snake_case			 = torch.tensor([1.0]				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = torch.tensor([2.0]				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = torch.tensor([1.0]				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






					@require_spacy
					@require_spacy_model("en_core_web_sm"				)
					@require_spacy_model("fr_core_news_sm"				)
					@pytest.mark.integration
					def        __SCREAMING_SNAKE_CASE			(					self				):
							import spacy

							_snake_case			 = spacy.load("en_core_web_sm"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = spacy.load("fr_core_news_sm"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							_snake_case			 = spacy.load("en_core_web_sm"				)
							_snake_case			 = mda(datasets.utils.py_utils.dumps(__UpperCamelCase				)				).hexdigest()
							self.assertEqual(__UpperCamelCase,						__UpperCamelCase				)
							self.assertNotEqual(__UpperCamelCase,						__UpperCamelCase				)






@pytest.mark.integration
def 	_lowercase	(			lowercase_						):



		'''simple docstring'''







		_snake_case			 = tmp_path / "dir1"
		_snake_case			 = tmp_path / "dir2"
		dira.mkdir()
		dira.mkdir()
		_snake_case			 = "script.py"
		_snake_case			 = dedent(
		    "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "						)
		_snake_case			 = dira / script_filename
		_snake_case			 = dira / script_filename
		with script_patha.open("w"						) as f:
				f.write(lowercase_						)
		with script_patha.open("w"						) as f:
				f.write(lowercase_						)
		_snake_case			 = subprocess.check_output(["python", str(lowercase_						)]						)
		_snake_case			 = subprocess.check_output(["python", str(lowercase_						)]						)
		assert fingerprinta == fingerprinta






def 	_lowercase	(			):



		'''simple docstring'''







		_snake_case			 = {"a": [0, 1, 2]}
		_snake_case			 = DatasetChild(InMemoryTable.from_pydict(lowercase_						)						)
		_snake_case			 = dataset.funca()._fingerprint
		assert expected_fingerprint == dataset.funca()._fingerprint
		assert expected_fingerprint != dataset.funca()._fingerprint

		with Pool(2						) as p:
				assert expected_fingerprint == p.apply_async(dataset.funca						).get()._fingerprint
				assert expected_fingerprint != p.apply_async(dataset.funca						).get()._fingerprint






def 	_lowercase	(			):



		'''simple docstring'''







		_snake_case			 = {"a": [0, 1, 2]}





		class   lowerCamelCase_							( datasets.Dataset	):




							'''simple docstring'''






							@fingerprint_transform(inplace=__UpperCamelCase				)
							def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
									return DummyDatasetChild(self.data,						fingerprint=__UpperCamelCase				)




		_snake_case			 = DummyDatasetChild(InMemoryTable.from_pydict(lowercase_						)						).func()





		class   lowerCamelCase_							( datasets.Dataset	):




							'''simple docstring'''






							@fingerprint_transform(inplace=__UpperCamelCase,						version="1.0.0"				)
							def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
									return DummyDatasetChild(self.data,						fingerprint=__UpperCamelCase				)




		_snake_case			 = DummyDatasetChild(InMemoryTable.from_pydict(lowercase_						)						).func()





		class   lowerCamelCase_							( datasets.Dataset	):




							'''simple docstring'''






							@fingerprint_transform(inplace=__UpperCamelCase,						version="2.0.0"				)
							def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
									return DummyDatasetChild(self.data,						fingerprint=__UpperCamelCase				)




		_snake_case			 = DummyDatasetChild(InMemoryTable.from_pydict(lowercase_						)						).func()

		assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}						) == 3






def 	_lowercase	(			):



		'''simple docstring'''







		_snake_case			 = Hasher()
		hasher.update(lambda lowercase_						: x						)











'''simple docstring'''







import warnings

from ...utils import logging
from .image_processing_vilt import ViltImageProcessor


_lowerCamelCase							:		Optional[Any]										=						logging.get_logger(__name__)





class   lowerCamelCase_							( UpperCAmelCase_							):




					'''simple docstring'''






					def __init__(					self,						*__UpperCamelCase,						**__UpperCamelCase				):
							warnings.warn(
							    "The class ViltFeatureExtractor is deprecated and will be removed in version 5 of Transformers. Please"
							    " use ViltImageProcessor instead.",						__UpperCamelCase,						)
							super().__init__(*__UpperCamelCase,						**__UpperCamelCase				)





