





'''simple docstring'''

import json
import os
import pickle
import subprocess
from hashlib import mda
from pathlib import Path
from tempfile import gettempdir
from textwrap import dedent
from types import CodeType, FunctionType
from unittest import TestCase
from unittest.mock import patch

import pytest
from multiprocess import Pool
from packaging import version

import datasets
from datasets.fingerprint import Hasher, fingerprint_transform
from datasets.table import InMemoryTable

from .utils import (
    require_regex,
    require_spacy,
    require_spacy_model,
    require_tiktoken,
    require_torch,
    require_transformers,
)






class      UpperCAmelCase			:
 def __init__(       self   :List[str]		, lowercase_   :str							)->							Tuple:
     A__       =    foo



 def __call__(       self   :Tuple							)->							Optional[int]:
     return self.foo






class      UpperCAmelCase			( datasets.Dataset       ):
 @fingerprint_transform(inplace=lowercase_							)
 def   UpperCAmelCase_							(       self   :List[str]		, lowercase_   :str		, *lowercase_   :str		, **lowercase_   :Optional[int]							)->							Optional[Any]:
     return DatasetChild(self.data		, fingerprint=lowercase_							)



 @fingerprint_transform(inplace=lowercase_							)
 def   UpperCAmelCase_							(       self   :List[Any]		, lowercase_   :List[Any]		, *lowercase_   :List[Any]		, **lowercase_   :List[str]							)->							List[str]:
     return DatasetChild(self.data		, fingerprint=lowercase_							)






class      UpperCAmelCase			:
 def __init__(       self   :Union[str, Any]		, lowercase_   :List[Any]							)->							str:
     A__       =    callable
 def __call__(       self   :Dict		, *lowercase_   :Union[str, Any]		, **lowercase_   :Optional[int]							)->							List[str]:
     if self.callable is not None:
         return self.callable(*lowercase_		, **lowercase_							)



 def __getstate__(       self   :List[str]							)->							Optional[int]:
     raise pickle.PicklingError()






class      UpperCAmelCase			( UpperCamelCase__       ):
 @require_transformers
 @pytest.mark.integration
 def   UpperCAmelCase_							(       self   :List[Any]							)->							Tuple:
     from transformers import AutoTokenizer

     def encode(lowercase_   :Dict							):
         return tokenizer(lowercase_							)

     # TODO: add hash consistency tests across sessions
     A__       =    AutoTokenizer.from_pretrained("bert-base-uncased"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(lambda lowercase_							: tokenizer(lowercase_							)							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    AutoTokenizer.from_pretrained("bert-base-cased"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(lambda lowercase_							: tokenizer(lowercase_							)							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    AutoTokenizer.from_pretrained("bert-base-uncased"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(lambda lowercase_							: tokenizer(lowercase_							)							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 @require_transformers
 @pytest.mark.integration
 def   UpperCAmelCase_							(       self   :List[Any]							)->							int:
     from transformers import AutoTokenizer

     A__       =    AutoTokenizer.from_pretrained("gpt2"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     tokenizer("Hello world !"							)  # call once to change the tokenizer's cache
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)



 @require_regex
 def   UpperCAmelCase_							(       self   :str							)->							List[Any]:
     import regex

     A__       =    regex.Regex("foo"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    regex.Regex("bar"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    regex.Regex("foo"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)






class      UpperCAmelCase			( UpperCamelCase__       ):
 def   UpperCAmelCase_							(       self   :List[Any]							)->							List[Any]:
     def func():
         return foo

     A__       =    [0]
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    [1]
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    [0]
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 def   UpperCAmelCase_							(       self   :List[str]							)->							Tuple:
     def func():
         pass

     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()

     def func():
         pass

     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
 def   UpperCAmelCase_							(       self   :List[Any]							)->							Optional[Any]:
     A__       =    mda(datasets.utils.py_utils.dumps(Foo([0]							)							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(Foo([1]							)							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(Foo([0]							)							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 def   UpperCAmelCase_							(       self   :Union[str, Any]							)->							Tuple:
     A__       =    mda(datasets.utils.py_utils.dumps(Foo([0]							).__call__							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(Foo([1]							).__call__							)							).hexdigest()
     A__       =    mda(datasets.utils.py_utils.dumps(Foo([0]							).__call__							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 def   UpperCAmelCase_							(       self   :Optional[Any]							)->							Optional[Any]:
     A__       =    (
         "co_argcount",
         "co_kwonlyargcount",
         "co_nlocals",
         "co_stacksize",
         "co_flags",
         "co_code",
         "co_consts",
         "co_names",
         "co_varnames",
         "co_filename",
         "co_name",
         "co_firstlineno",
         "co_lnotab",
         "co_freevars",
         "co_cellvars",
     )

     def _create_code_pyaa(*lowercase_   :List[Any]							):
         if hasattr(args[-3]		, "encode"							):
             A__       =    list(lowercase_							)
             A__       =    args[-3].encode()  # co_lnotab
             A__       =    args[-10].encode()  # co_code
         if hasattr(lowercase_		, "co_posonlyargcount"							):
             if len(lowercase_							) == 16:
                 return CodeType(*lowercase_							)
             elif len(lowercase_							) == 15:
                 return CodeType(args[0]		, 0		, *args[1:]							)
             return CodeType(args[0]		, 0		, 0		, *args[1:]							)
         elif hasattr(lowercase_		, "co_kwonlyargcount"							):
             if len(lowercase_							) == 16:
                 return CodeType(args[0]		, *args[2:]							)
             elif len(lowercase_							) == 15:
                 return CodeType(*lowercase_							)
             return CodeType(args[0]		, 0		, *args[1:]							)
         if len(lowercase_							) == 16:
             return CodeType(args[0]		, *args[3:]							)
         elif len(lowercase_							) == 15:
             return CodeType(args[0]		, *args[2:]							)
         return CodeType(*lowercase_							)

     def create_ipython_func(lowercase_   :Optional[Any]		, lowercase_   :int							):
         def func():
             return returned_obj

         A__       =    func.__code__
         # Use _create_code from dill in order to make it work for different python versions
         if datasets.config.PY_VERSION < version.parse("3.8"							):
             A__       =    _create_code_pyaa(
                 *[getattr(lowercase_		, lowercase_							) if k != "co_filename" else co_filename for k in code_args_pyaa]							)
         else:
             A__       =    code.replace(co_filename=lowercase_							)
         return FunctionType(lowercase_		, func.__globals__		, func.__name__		, func.__defaults__		, func.__closure__							)

     A__,			A__       =    "<ipython-input-2-e0383a102aae>", [0]
     A__       =    mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_		, lowercase_							)							)							).hexdigest()
     A__,			A__       =    "<ipython-input-2-e0383a102aae>", [1]
     A__       =    mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_		, lowercase_							)							)							).hexdigest()
     A__,			A__       =    "<ipython-input-5-713f6613acf3>", [0]
     A__       =    mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_		, lowercase_							)							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)

     A__,			A__       =    os.path.join(gettempdir()		, "ipykernel_12345"		, "321456789.py"							), [0]
     A__       =    mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_		, lowercase_							)							)							).hexdigest()
     A__,			A__       =    os.path.join(gettempdir()		, "ipykernel_12345"		, "321456789.py"							), [1]
     A__       =    mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_		, lowercase_							)							)							).hexdigest()
     A__,			A__       =    os.path.join(gettempdir()		, "ipykernel_12345"		, "654123987.py"							), [0]
     A__       =    mda(datasets.utils.py_utils.dumps(create_ipython_func(lowercase_		, lowercase_							)							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)



 def   UpperCAmelCase_							(       self   :str							)->							Union[str, Any]:
     A__,			A__       =    [0], [1]

     def func():
         return foo, bar

     A__       =    "__main__"

     def globalvars_mocka_side_effect(lowercase_   :List[Any]		, *lowercase_   :Tuple		, **lowercase_   :Optional[int]							):
         return {"foo": foo, "bar": bar}

     def globalvars_mocka_side_effect(lowercase_   :int		, *lowercase_   :int		, **lowercase_   :Optional[Any]							):
         return {"bar": bar, "foo": foo}

     with patch("dill.detect.globalvars"		, side_effect=lowercase_							) as globalvars_mocka:
         A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
         self.assertGreater(globalvars_mocka.call_count		, 0							)
     with patch("dill.detect.globalvars"		, side_effect=lowercase_							) as globalvars_mocka:
         A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
         self.assertGreater(globalvars_mocka.call_count		, 0							)
     self.assertEqual(lowercase_		, lowercase_							)






class      UpperCAmelCase			( UpperCamelCase__       ):
 def   UpperCAmelCase_							(       self   :Dict							)->							Optional[int]:
     A__       =    Hasher.hash("hello"							)
     A__       =    Hasher.hash("hello"							)
     A__       =    Hasher.hash("there"							)
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 def   UpperCAmelCase_							(       self   :List[str]							)->							Any:
     A__       =    Hasher.hash(Foo("hello"							)							)
     A__       =    Hasher.hash(Foo("hello"							)							)
     A__       =    Hasher.hash(Foo("there"							)							)
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 def   UpperCAmelCase_							(       self   :List[Any]							)->							Tuple:
     A__       =    Hasher()
     for x in ["hello", Foo("hello"							)]:
         hasher.update(lowercase_							)
     A__       =    hasher.hexdigest()
     A__       =    Hasher()
     for x in ["hello", Foo("hello"							)]:
         hasher.update(lowercase_							)
     A__       =    hasher.hexdigest()
     A__       =    Hasher()
     for x in ["there", Foo("there"							)]:
         hasher.update(lowercase_							)
     A__       =    hasher.hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 def   UpperCAmelCase_							(       self   :int							)->							int:
     with self.assertRaises(pickle.PicklingError							):
         Hasher.hash(UnpicklableCallable(Foo("hello"							)							)							)
 def   UpperCAmelCase_							(       self   :Tuple							)->							Optional[int]:
     A__       =    "abc"
     A__       =    [string, string]  # two strings have the same ids
     A__       =    [string, string]
     A__       =    json.loads(F"[\"{string}\", \"{string}\"]"							)  # two strings have different ids
     self.assertIs(obja[0]		, lowercase_							)
     self.assertIs(obja[0]		, obja[1]							)
     self.assertIs(obja[0]		, lowercase_							)
     self.assertIs(obja[0]		, obja[1]							)
     self.assertIsNot(obja[0]		, lowercase_							)
     self.assertIsNot(obja[0]		, obja[1]							)
     A__       =    Hasher.hash(lowercase_							)
     A__       =    Hasher.hash(lowercase_							)
     A__       =    Hasher.hash(lowercase_							)
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertEqual(lowercase_		, lowercase_							)
 @require_tiktoken
 def   UpperCAmelCase_							(       self   :Union[str, Any]							)->							Tuple:
     import tiktoken

     A__       =    tiktoken.get_encoding("gpt2"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    tiktoken.get_encoding("r50k_base"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    tiktoken.get_encoding("gpt2"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)
 @require_torch
 def   UpperCAmelCase_							(       self   :Optional[int]							)->							int:
     import torch

     A__       =    torch.tensor([1.0]							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    torch.tensor([2.0]							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    torch.tensor([1.0]							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)



 @require_spacy
 @require_spacy_model("en_core_web_sm"							)
 @require_spacy_model("fr_core_news_sm"							)
 @pytest.mark.integration
 def   UpperCAmelCase_							(       self   :List[str]							)->							int:
     import spacy

     A__       =    spacy.load("en_core_web_sm"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    spacy.load("fr_core_news_sm"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     A__       =    spacy.load("en_core_web_sm"							)
     A__       =    mda(datasets.utils.py_utils.dumps(lowercase_							)							).hexdigest()
     self.assertEqual(lowercase_		, lowercase_							)
     self.assertNotEqual(lowercase_		, lowercase_							)





@pytest.mark.integration
def 		UpperCamelCase       (							_lowerCamelCase      :      Path ):
    A__       =    tmp_path / "dir1"
    A__       =    tmp_path / "dir2"
    dira.mkdir()
    dira.mkdir()
    A__       =    "script.py"
    A__       =    dedent(
        "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    " )
    A__       =    dira / script_filename
    A__       =    dira / script_filename
    with script_patha.open("w" ) as f:
        f.write(_lowerCamelCase )
    with script_patha.open("w" ) as f:
        f.write(_lowerCamelCase )
    A__       =    subprocess.check_output(["python", str(_lowerCamelCase )] )
    A__       =    subprocess.check_output(["python", str(_lowerCamelCase )] )
    assert fingerprinta == fingerprinta





def 		UpperCamelCase       (							):
    A__       =    {"a": [0, 1, 2]}
    A__       =    DatasetChild(InMemoryTable.from_pydict(_lowerCamelCase ) )
    A__       =    dataset.funca()._fingerprint
    assert expected_fingerprint == dataset.funca()._fingerprint
    assert expected_fingerprint != dataset.funca()._fingerprint

    with Pool(2 ) as p:
        assert expected_fingerprint == p.apply_async(dataset.funca ).get()._fingerprint
        assert expected_fingerprint != p.apply_async(dataset.funca ).get()._fingerprint





def 		UpperCamelCase       (							):
    A__       =    {"a": [0, 1, 2]}






    class      UpperCAmelCase			( datasets.Dataset       ):
     @fingerprint_transform(inplace=lowercase_							)
     def   UpperCAmelCase_							(       self   :Tuple		, lowercase_   :Any							)->							str:
         return DummyDatasetChild(self.data		, fingerprint=lowercase_							)

    A__       =    DummyDatasetChild(InMemoryTable.from_pydict(_lowerCamelCase ) ).func()






    class      UpperCAmelCase			( datasets.Dataset       ):
     @fingerprint_transform(inplace=lowercase_		, version="1.0.0"							)
     def   UpperCAmelCase_							(       self   :Optional[int]		, lowercase_   :str							)->							Union[str, Any]:
         return DummyDatasetChild(self.data		, fingerprint=lowercase_							)

    A__       =    DummyDatasetChild(InMemoryTable.from_pydict(_lowerCamelCase ) ).func()






    class      UpperCAmelCase			( datasets.Dataset       ):
     @fingerprint_transform(inplace=lowercase_		, version="2.0.0"							)
     def   UpperCAmelCase_							(       self   :Optional[Any]		, lowercase_   :List[Any]							)->							Union[str, Any]:
         return DummyDatasetChild(self.data		, fingerprint=lowercase_							)

    A__       =    DummyDatasetChild(InMemoryTable.from_pydict(_lowerCamelCase ) ).func()

    assert len({fingeprint_no_version, fingeprint_a, fingeprint_a} ) == 3





def 		UpperCamelCase       (							):
    # AttributeError: module 'dill._dill' has no attribute 'stack'
    A__       =    Hasher()
    hasher.update(lambda _lowerCamelCase : x )










'''simple docstring'''

import numpy as np
import PIL
import torch
import torchvision.transforms as T
import torchvision.transforms.functional as TF
from PIL import Image





def 		UpperCamelCase       (							_lowerCamelCase      :      Optional[Any]		, _lowerCamelCase      :      Union[str, Any]=2_56 ):
    A__       =    min(img.size )

    if s < target_image_size:
        raise ValueError(F"min dim for image {s} < {target_image_size}" )

    A__       =    target_image_size / s
    A__       =    (round(r * img.size[1] ), round(r * img.size[0] ))
    A__       =    TF.resize(_lowerCamelCase		, _lowerCamelCase		, interpolation=PIL.Image.LANCZOS )
    A__       =    TF.center_crop(_lowerCamelCase		, output_size=2 * [target_image_size] )
    A__       =    torch.unsqueeze(T.ToTensor()(_lowerCamelCase )		, 0 )
    return img





def 		UpperCamelCase       (							_lowerCamelCase      :      int ):
    A__       =    2.0 * x - 1.0
    return x





def 		UpperCamelCase       (							_lowerCamelCase      :      Optional[Any]		, _lowerCamelCase      :      Tuple=True		, _lowerCamelCase      :      int="RGB" ):
    A__       =    x.detach().cpu()
    if process:
        A__       =    post_process_tensor(_lowerCamelCase )
    A__       =    x.numpy()
    if process:
        A__       =    (2_55 * x).astype(np.uinta )
    A__       =    Image.fromarray(_lowerCamelCase )
    if not x.mode == mode:
        A__       =    x.convert(_lowerCamelCase )
    return x





def 		UpperCamelCase       (							_lowerCamelCase      :      Any ):
    A__       =    torch.clamp(_lowerCamelCase		, -1.0		, 1.0 )
    A__       =    (x + 1.0) / 2.0
    A__       =    x.permute(1		, 2		, 0 )
    return x





def 		UpperCamelCase       (							_lowerCamelCase      :      List[Any] ):
    A__       =    post_process_tensor(x.squeeze() )
    return x.permute(2		, 0		, 1 ).unsqueeze(0 )



