code: 
  '''simple docstring'''
  
  
  
  
  
  from __future__ import annotations
  
  import inspect
  import unittest
  
  from transformers import MobileViTConfig
  from transformers.file_utils import is_tf_available, is_vision_available
  from transformers.testing_utils import require_tf, slow
  
  from ...test_configuration_common import ConfigTester
  from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
  from ...test_pipeline_mixin import PipelineTesterMixin
  
  
  if is_tf_available():
       import numpy as np
       import tensorflow as tf
  
       from transformers import TFMobileViTForImageClassification, TFMobileViTForSemanticSegmentation, TFMobileViTModel
       from transformers.models.mobilevit.modeling_tf_mobilevit import TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST
  
  
  if is_vision_available():
       from PIL import Image
  
       from transformers import MobileViTImageProcessor
  
  
  
  class  lowerCAmelCase__   (       UpperCamelCase       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    def 		_lowerCamelCase	(self  : Dict    )   ->       Union[str, Any]:
          snake_case_						       =							self.config_class(**self.inputs_dict    )
          self.parent.assertTrue(hasattr(lowerCAmelCase_	, "hidden_sizes"    )    )
          self.parent.assertTrue(hasattr(lowerCAmelCase_	, "neck_hidden_sizes"    )    )
          self.parent.assertTrue(hasattr(lowerCAmelCase_	, "num_attention_heads"    )    )
  
  
  
  class  lowerCAmelCase__   :
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    def __init__(self  : Dict	, lowerCAmelCase_  : List[str]	, lowerCAmelCase_  : Optional[Any]=13	, lowerCAmelCase_  : str=32	, lowerCAmelCase_  : Dict=2	, lowerCAmelCase_  : Optional[Any]=3	, lowerCAmelCase_  : List[str]=6_40	, lowerCAmelCase_  : Dict=4	, lowerCAmelCase_  : Union[str, Any]="silu"	, lowerCAmelCase_  : Dict=3	, lowerCAmelCase_  : Union[str, Any]=32	, lowerCAmelCase_  : Tuple=0.1	, lowerCAmelCase_  : Any=0.1	, lowerCAmelCase_  : Optional[Any]=0.1	, lowerCAmelCase_  : Optional[Any]=0.02	, lowerCAmelCase_  : Optional[Any]=True	, lowerCAmelCase_  : Union[str, Any]=True	, lowerCAmelCase_  : str=10	, lowerCAmelCase_  : Any=None	, )   ->       Union[str, Any]:
          snake_case_						       =							parent
          snake_case_						       =							batch_size
          snake_case_						       =							image_size
          snake_case_						       =							patch_size
          snake_case_						       =							num_channels
          snake_case_						       =							last_hidden_size
          snake_case_						       =							num_attention_heads
          snake_case_						       =							hidden_act
          snake_case_						       =							conv_kernel_size
          snake_case_						       =							output_stride
          snake_case_						       =							hidden_dropout_prob
          snake_case_						       =							attention_probs_dropout_prob
          snake_case_						       =							classifier_dropout_prob
          snake_case_						       =							use_labels
          snake_case_						       =							is_training
          snake_case_						       =							num_labels
          snake_case_						       =							initializer_range
          snake_case_						       =							scope
  
    def 		_lowerCamelCase	(self  : Any    )   ->       int:
          snake_case_						       =							floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )
  
          snake_case_						       =							None
          snake_case_						       =							None
          if self.use_labels:
                snake_case_						       =							ids_tensor([self.batch_size]	, self.num_labels    )
                snake_case_						       =							ids_tensor([self.batch_size, self.image_size, self.image_size]	, self.num_labels    )
  
          snake_case_						       =							self.get_config()
  
          return config, pixel_values, labels, pixel_labels
  
    def 		_lowerCamelCase	(self  : Dict    )   ->       str:
          return MobileViTConfig(
              image_size=self.image_size	, patch_size=self.patch_size	, num_channels=self.num_channels	, num_attention_heads=self.num_attention_heads	, hidden_act=self.hidden_act	, conv_kernel_size=self.conv_kernel_size	, output_stride=self.output_stride	, hidden_dropout_prob=self.hidden_dropout_prob	, attention_probs_dropout_prob=self.attention_probs_dropout_prob	, classifier_dropout_prob=self.classifier_dropout_prob	, initializer_range=self.initializer_range	, )
  
    def 		_lowerCamelCase	(self  : Union[str, Any]	, lowerCAmelCase_  : List[str]	, lowerCAmelCase_  : Optional[int]	, lowerCAmelCase_  : Optional[Any]	, lowerCAmelCase_  : Tuple    )   ->       Any:
          snake_case_						       =							TFMobileViTModel(config=lowerCAmelCase_    )
          snake_case_						       =							model(lowerCAmelCase_	, training=lowerCAmelCase_    )
          snake_case_						       =							snake_case_						       =							self.image_size // self.output_stride
          self.parent.assertEqual(
              result.last_hidden_state.shape	, (self.batch_size, self.last_hidden_size, expected_height, expected_width)    )
  
    def 		_lowerCamelCase	(self  : Union[str, Any]	, lowerCAmelCase_  : Union[str, Any]	, lowerCAmelCase_  : Any	, lowerCAmelCase_  : Any	, lowerCAmelCase_  : List[Any]    )   ->       str:
          snake_case_						       =							self.num_labels
          snake_case_						       =							TFMobileViTForImageClassification(lowerCAmelCase_    )
          snake_case_						       =							model(lowerCAmelCase_	, labels=lowerCAmelCase_	, training=lowerCAmelCase_    )
          self.parent.assertEqual(result.logits.shape	, (self.batch_size, self.num_labels)    )
  
    def 		_lowerCamelCase	(self  : str	, lowerCAmelCase_  : str	, lowerCAmelCase_  : List[str]	, lowerCAmelCase_  : str	, lowerCAmelCase_  : List[str]    )   ->       List[Any]:
          snake_case_						       =							self.num_labels
          snake_case_						       =							TFMobileViTForSemanticSegmentation(lowerCAmelCase_    )
          snake_case_						       =							snake_case_						       =							self.image_size // self.output_stride
  
          snake_case_						       =							model(lowerCAmelCase_	, training=lowerCAmelCase_    )
          self.parent.assertEqual(
              result.logits.shape	, (self.batch_size, self.num_labels, expected_height, expected_width)    )
  
          snake_case_						       =							model(lowerCAmelCase_	, labels=lowerCAmelCase_	, training=lowerCAmelCase_    )
          self.parent.assertEqual(
              result.logits.shape	, (self.batch_size, self.num_labels, expected_height, expected_width)    )
  
  
    def 		_lowerCamelCase	(self  : str    )   ->       Union[str, Any]:
          snake_case_						       =							self.prepare_config_and_inputs()
          snake_case_       ,	snake_case_       ,	snake_case_       ,	snake_case_						       =							config_and_inputs
          snake_case_						       =							{"pixel_values": pixel_values}
          return config, inputs_dict
  
  
  
  @require_tf
  class  lowerCAmelCase__   (       UpperCamelCase					,							UpperCamelCase					,							unittest.TestCase       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    _UpperCamelCase						= (
        (TFMobileViTModel, TFMobileViTForImageClassification, TFMobileViTForSemanticSegmentation)
        if is_tf_available()
        else ()
    )
    _UpperCamelCase						= (
        {'feature-extraction': TFMobileViTModel, 'image-classification': TFMobileViTForImageClassification}
        if is_tf_available()
        else {}
    )
  
    _UpperCamelCase						= False
    _UpperCamelCase						= False
    _UpperCamelCase						= False
    _UpperCamelCase						= False
    _UpperCamelCase						= False
  
    def 		_lowerCamelCase	(self  : List[Any]    )   ->       Optional[int]:
          snake_case_						       =							TFMobileViTModelTester(self    )
          snake_case_						       =							TFMobileViTConfigTester(self	, config_class=lowerCAmelCase_	, has_text_modality=lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : str    )   ->       Tuple:
          self.config_tester.run_common_tests()
  
    @unittest.skip(reason="MobileViT does not use inputs_embeds"    )
    def 		_lowerCamelCase	(self  : int    )   ->       List[Any]:
          pass
  
    @unittest.skip(reason="MobileViT does not support input and output embeddings"    )
    def 		_lowerCamelCase	(self  : List[Any]    )   ->       int:
          pass
  
    @unittest.skip(reason="MobileViT does not output attentions"    )
    def 		_lowerCamelCase	(self  : Tuple    )   ->       Tuple:
          pass
  
    def 		_lowerCamelCase	(self  : Any    )   ->       Union[str, Any]:
          snake_case_       ,	snake_case_						       =							self.model_tester.prepare_config_and_inputs_for_common()
  
          for model_class in self.all_model_classes:
                snake_case_						       =							model_class(lowerCAmelCase_    )
                snake_case_						       =							inspect.signature(model.call    )
                # signature.parameters is an OrderedDict => so arg_names order is deterministic
                snake_case_						       =							[*signature.parameters.keys()]
  
                snake_case_						       =							["pixel_values"]
                self.assertListEqual(arg_names[:1]	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : Any    )   ->       Dict:
          snake_case_						       =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_model(*lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : List[str]    )   ->       int:
          def check_hidden_states_output(lowerCAmelCase_  : Optional[int]	, lowerCAmelCase_  : Tuple	, lowerCAmelCase_  : Tuple    ):
                snake_case_						       =							model_class(lowerCAmelCase_    )
  
                snake_case_						       =							model(**self._prepare_for_class(lowerCAmelCase_	, lowerCAmelCase_    )    )
  
                snake_case_						       =							outputs.hidden_states
  
                snake_case_						       =							5
                self.assertEqual(len(lowerCAmelCase_    )	, lowerCAmelCase_    )
  
                # MobileViT's feature maps are of shape (batch_size, num_channels, height, width)
                # with the width and height being successively divided by 2.
                snake_case_						       =							2
                for i in range(len(lowerCAmelCase_    )    ):
                      self.assertListEqual(
                          list(hidden_states[i].shape[-2:]    )	, [self.model_tester.image_size // divisor, self.model_tester.image_size // divisor]	, )
                      divisor *= 2
  
                self.assertEqual(self.model_tester.output_stride	, divisor // 2    )
  
          snake_case_       ,	snake_case_						       =							self.model_tester.prepare_config_and_inputs_for_common()
  
          for model_class in self.all_model_classes:
                snake_case_						       =							True
                check_hidden_states_output(lowerCAmelCase_	, lowerCAmelCase_	, lowerCAmelCase_    )
  
                # check that output_hidden_states also work using config
                del inputs_dict["output_hidden_states"]
                snake_case_						       =							True
  
                check_hidden_states_output(lowerCAmelCase_	, lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : str    )   ->       Any:
          snake_case_						       =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_image_classification(*lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       Optional[int]:
          snake_case_						       =							self.model_tester.prepare_config_and_inputs()
          self.model_tester.create_and_check_for_semantic_segmentation(*lowerCAmelCase_    )
  
    @unittest.skipIf(
        not is_tf_available() or len(tf.config.list_physical_devices("GPU"    )    ) == 0	, reason="TF does not support backprop for grouped convolutions on CPU."	, )
    def 		_lowerCamelCase	(self  : List[str]    )   ->       Dict:
          super().test_dataset_conversion()
  
    def 		_lowerCamelCase	(self  : List[str]	, lowerCAmelCase_  : Optional[Any]	, lowerCAmelCase_  : Dict	, lowerCAmelCase_  : List[str]=2E-1	, lowerCAmelCase_  : Any=2E-1    )   ->       Optional[Any]:
          self.assertTrue(np.allclose(lowerCAmelCase_	, lowerCAmelCase_	, atol=lowerCAmelCase_	, rtol=lowerCAmelCase_    )    )
  
    @unittest.skipIf(
        not is_tf_available() or len(tf.config.list_physical_devices("GPU"    )    ) == 0	, reason="TF does not support backprop for grouped convolutions on CPU."	, )
    @slow
    def 		_lowerCamelCase	(self  : Tuple    )   ->       Union[str, Any]:
          snake_case_       ,	snake_case_						       =							self.model_tester.prepare_config_and_inputs_for_common()
  
          for model_class in self.all_model_classes:
                # Since `TFMobileViTModel` cannot operate with the default `fit()` method.
                if model_class.__name__ != "TFMobileViTModel":
                      snake_case_						       =							model_class(lowerCAmelCase_    )
                      if getattr(lowerCAmelCase_	, "hf_compute_loss"	, lowerCAmelCase_    ):
                            super().test_keras_fit()
  
    def 		_lowerCamelCase	(self  : List[Any]    )   ->       List[str]:
          snake_case_       ,	snake_case_						       =							self.model_tester.prepare_config_and_inputs_for_common()
          for model_class in self.all_model_classes:
                # set an ignore index to correctly test the masked loss used in
                # `TFMobileViTForSemanticSegmentation`.
                if model_class.__name__ != "TFMobileViTForSemanticSegmentation":
                      snake_case_						       =							5
  
                snake_case_						       =							model_class(lowerCAmelCase_    )
                if getattr(lowerCAmelCase_	, "hf_compute_loss"	, lowerCAmelCase_    ):
                      # The number of elements in the loss should be the same as the number of elements in the label
                      snake_case_						       =							self._prepare_for_class(inputs_dict.copy()	, lowerCAmelCase_	, return_labels=lowerCAmelCase_    )
                      snake_case_						       =							prepared_for_class[
                          sorted(prepared_for_class.keys() - inputs_dict.keys()	, reverse=lowerCAmelCase_    )[0]
                      ]
                      snake_case_						       =							added_label.shape.as_list()[:1]
  
                      # Test that model correctly compute the loss with kwargs
                      snake_case_						       =							self._prepare_for_class(inputs_dict.copy()	, lowerCAmelCase_	, return_labels=lowerCAmelCase_    )
                      snake_case_						       =							{"input_ids", "pixel_values", "input_features"}
                      snake_case_						       =							possible_input_names.intersection(set(lowerCAmelCase_    )    ).pop()
                      snake_case_						       =							prepared_for_class.pop(lowerCAmelCase_    )
  
                      snake_case_						       =							model(lowerCAmelCase_	, **lowerCAmelCase_    )[0]
                      self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )
  
                      # Test that model correctly compute the loss when we mask some positions
                      snake_case_						       =							self._prepare_for_class(inputs_dict.copy()	, lowerCAmelCase_	, return_labels=lowerCAmelCase_    )
                      snake_case_						       =							{"input_ids", "pixel_values", "input_features"}
                      snake_case_						       =							possible_input_names.intersection(set(lowerCAmelCase_    )    ).pop()
                      snake_case_						       =							prepared_for_class.pop(lowerCAmelCase_    )
                      if "labels" in prepared_for_class:
                            snake_case_						       =							prepared_for_class["labels"].numpy()
                            if len(labels.shape    ) > 1 and labels.shape[1] != 1:
                                  # labels[0] = -100
                                  snake_case_						       =							tf.convert_to_tensor(lowerCAmelCase_    )
                                  snake_case_						       =							model(lowerCAmelCase_	, **lowerCAmelCase_    )[0]
                                  self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )
                                  self.assertTrue(not np.any(np.isnan(loss.numpy()    )    )    )
  
                  # Test that model correctly compute the loss with a dict
                      snake_case_						       =							self._prepare_for_class(inputs_dict.copy()	, lowerCAmelCase_	, return_labels=lowerCAmelCase_    )
                      snake_case_						       =							model(lowerCAmelCase_    )[0]
                      self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )
  
                      # Test that model correctly compute the loss with a tuple
                      snake_case_						       =							self._prepare_for_class(inputs_dict.copy()	, lowerCAmelCase_	, return_labels=lowerCAmelCase_    )
  
                      # Get keys that were added with the _prepare_for_class function
                      snake_case_						       =							prepared_for_class.keys() - inputs_dict.keys()
                      snake_case_						       =							inspect.signature(model.call    ).parameters
                      snake_case_						       =							list(signature.keys()    )
  
                      # Create a dictionary holding the location of the tensors in the tuple
                      snake_case_						       =							{0: input_name}
                      for label_key in label_keys:
                            snake_case_						       =							signature_names.index(lowerCAmelCase_    )
                            snake_case_						       =							label_key
                      snake_case_						       =							sorted(tuple_index_mapping.items()    )
                      # Initialize a list with their default values, update the values and convert to a tuple
                      snake_case_						       =							[]
  
                      for name in signature_names:
                            if name != "kwargs":
                                  list_input.append(signature[name].default    )
  
                      for index, value in sorted_tuple_index_mapping:
                            snake_case_						       =							prepared_for_class[value]
  
                      snake_case_						       =							tuple(lowerCAmelCase_    )
  
                      # Send to model
                      snake_case_						       =							model(tuple_input[:-1]    )[0]
  
                      self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )
  
  
    @slow
    def 		_lowerCamelCase	(self  : List[str]    )   ->       List[Any]:
          for model_name in TF_MOBILEVIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                snake_case_						       =							TFMobileViTModel.from_pretrained(lowerCAmelCase_    )
                self.assertIsNotNone(lowerCAmelCase_    )
  
  def   snake_case_       (							):
        snake_case_						       =							Image.open("./tests/fixtures/tests_samples/COCO/000000039769.png"			)
        return image
  
  
  
  
  @require_tf
  class  lowerCAmelCase__   (       unittest.TestCase       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    @slow
    def 		_lowerCamelCase	(self  : str    )   ->       Tuple:
          snake_case_						       =							TFMobileViTForImageClassification.from_pretrained("apple/mobilevit-xx-small"    )
  
          snake_case_						       =							MobileViTImageProcessor.from_pretrained("apple/mobilevit-xx-small"    )
          snake_case_						       =							prepare_img()
          snake_case_						       =							image_processor(images=lowerCAmelCase_	, return_tensors="tf"    )
  
          # forward pass
          snake_case_						       =							model(**lowerCAmelCase_	, training=lowerCAmelCase_    )
  
          # verify the logits
          snake_case_						       =							tf.TensorShape((1, 10_00)    )
          self.assertEqual(outputs.logits.shape	, lowerCAmelCase_    )
  
          snake_case_						       =							tf.constant([-1.9_364, -1.2_327, -0.4_653]    )
  
          tf.debugging.assert_near(outputs.logits[0, :3]	, lowerCAmelCase_	, atol=1E-4	, rtol=1E-04    )
  
  
    @slow
    def 		_lowerCamelCase	(self  : int    )   ->       Optional[Any]:
          # `from_pt` will be removed
          snake_case_						       =							TFMobileViTForSemanticSegmentation.from_pretrained("apple/deeplabv3-mobilevit-xx-small"    )
  
          snake_case_						       =							MobileViTImageProcessor.from_pretrained("apple/deeplabv3-mobilevit-xx-small"    )
  
          snake_case_						       =							prepare_img()
          snake_case_						       =							image_processor(images=lowerCAmelCase_	, return_tensors="tf"    )
  
          # forward pass
          snake_case_						       =							model(inputs.pixel_values	, training=lowerCAmelCase_    )
          snake_case_						       =							outputs.logits
  
          # verify the logits
          snake_case_						       =							tf.TensorShape((1, 21, 32, 32)    )
          self.assertEqual(logits.shape	, lowerCAmelCase_    )
  
          snake_case_						       =							tf.constant(
              [
                  [[6.9_713, 6.9_786, 7.2_422], [7.2_893, 7.2_825, 7.4_446], [7.6_580, 7.8_797, 7.9_420]],
                  [[-10.6_869, -10.3_250, -10.3_471], [-10.4_228, -9.9_868, -9.7_132], [-11.0_405, -11.0_221, -10.7_318]],
                  [[-3.3_089, -2.8_539, -2.6_740], [-3.2_706, -2.5_621, -2.5_108], [-3.2_534, -2.6_615, -2.6_651]],
              ]    )
  
          tf.debugging.assert_near(logits[0, :3, :3, :3]	, lowerCAmelCase_	, rtol=1E-4	, atol=1E-4    )
  
  
style_context: 
  '''simple docstring'''
  
  
  
  
  
  import json
  import os
  import pickle
  import subprocess
  from hashlib import mda
  from pathlib import Path
  from tempfile import gettempdir
  from textwrap import dedent
  from types import CodeType, FunctionType
  from unittest import TestCase
  from unittest.mock import patch
  
  import pytest
  from multiprocess import Pool
  from packaging import version
  
  import datasets
  from datasets.fingerprint import Hasher, fingerprint_transform
  from datasets.table import InMemoryTable
  
  from .utils import (
      require_regex,
      require_spacy,
      require_spacy_model,
      require_tiktoken,
      require_torch,
      require_transformers,
  )
  
  
  
  class  lowerCAmelCase__   :
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    def __init__(self  : int	, lowerCAmelCase_  : Optional[Any]    )   ->       Tuple:
          snake_case_						       =							foo
  
  
    def __call__(self  : List[Any]    )   ->       Optional[int]:
          return self.foo
  
  
  
  class  lowerCAmelCase__   (       datasets.Dataset       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    @fingerprint_transform(inplace=lowerCAmelCase_    )
    def 		_lowerCamelCase	(self  : Tuple	, lowerCAmelCase_  : Any	, *lowerCAmelCase_  : Union[str, Any]	, **lowerCAmelCase_  : Any    )   ->       str:
          return DatasetChild(self.data	, fingerprint=lowerCAmelCase_    )
  
  
    @fingerprint_transform(inplace=lowerCAmelCase_    )
    def 		_lowerCamelCase	(self  : str	, lowerCAmelCase_  : Optional[Any]	, *lowerCAmelCase_  : List[Any]	, **lowerCAmelCase_  : Any    )   ->       List[str]:
          return DatasetChild(self.data	, fingerprint=lowerCAmelCase_    )
  
  
  
  class  lowerCAmelCase__   :
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    def __init__(self  : int	, lowerCAmelCase_  : List[str]    )   ->       Dict:
          snake_case_						       =							callable
  
    def __call__(self  : List[str]	, *lowerCAmelCase_  : Any	, **lowerCAmelCase_  : Tuple    )   ->       Dict:
          if self.callable is not None:
                return self.callable(*lowerCAmelCase_	, **lowerCAmelCase_    )
  
  
    def __getstate__(self  : List[str]    )   ->       Tuple:
          raise pickle.PicklingError()
  
  
  
  class  lowerCAmelCase__   (       UpperCamelCase       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    @require_transformers
    @pytest.mark.integration
    def 		_lowerCamelCase	(self  : str    )   ->       str:
          from transformers import AutoTokenizer
  
          def encode(lowerCAmelCase_  : str    ):
                return tokenizer(lowerCAmelCase_    )
  
          # TODO: add hash consistency tests across sessions
          snake_case_						       =							AutoTokenizer.from_pretrained("bert-base-uncased"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lambda lowerCAmelCase_    : tokenizer(lowerCAmelCase_    )    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							AutoTokenizer.from_pretrained("bert-base-cased"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lambda lowerCAmelCase_    : tokenizer(lowerCAmelCase_    )    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							AutoTokenizer.from_pretrained("bert-base-uncased"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lambda lowerCAmelCase_    : tokenizer(lowerCAmelCase_    )    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    @require_transformers
    @pytest.mark.integration
    def 		_lowerCamelCase	(self  : Optional[Any]    )   ->       Optional[int]:
          from transformers import AutoTokenizer
  
          snake_case_						       =							AutoTokenizer.from_pretrained("gpt2"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          tokenizer("Hello world !"    )  # call once to change the tokenizer's cache
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
  
    @require_regex
    def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       Optional[int]:
          import regex
  
          snake_case_						       =							regex.Regex("foo"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							regex.Regex("bar"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							regex.Regex("foo"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
  
  
  class  lowerCAmelCase__   (       UpperCamelCase       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    def 		_lowerCamelCase	(self  : Dict    )   ->       Optional[Any]:
          def func():
                return foo
  
          snake_case_						       =							[0]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							[1]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							[0]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : str    )   ->       Tuple:
          def func():
                pass
  
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
  
          def func():
                pass
  
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : List[Any]    )   ->       Union[str, Any]:
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    )    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([1]    )    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    )    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       int:
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    ).__call__    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([1]    ).__call__    )    ).hexdigest()
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(Foo([0]    ).__call__    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : int    )   ->       Optional[Any]:
          snake_case_						       =							(
              "co_argcount",
              "co_kwonlyargcount",
              "co_nlocals",
              "co_stacksize",
              "co_flags",
              "co_code",
              "co_consts",
              "co_names",
              "co_varnames",
              "co_filename",
              "co_name",
              "co_firstlineno",
              "co_lnotab",
              "co_freevars",
              "co_cellvars",
          )
  
          def _create_code_pyaa(*lowerCAmelCase_  : Any    ):
                if hasattr(args[-3]	, "encode"    ):
                      snake_case_						       =							list(lowerCAmelCase_    )
                      snake_case_						       =							args[-3].encode()  # co_lnotab
                      snake_case_						       =							args[-10].encode()  # co_code
                if hasattr(lowerCAmelCase_	, "co_posonlyargcount"    ):
                      if len(lowerCAmelCase_    ) == 16:
                            return CodeType(*lowerCAmelCase_    )
                      elif len(lowerCAmelCase_    ) == 15:
                            return CodeType(args[0]	, 0	, *args[1:]    )
                      return CodeType(args[0]	, 0	, 0	, *args[1:]    )
                elif hasattr(lowerCAmelCase_	, "co_kwonlyargcount"    ):
                      if len(lowerCAmelCase_    ) == 16:
                            return CodeType(args[0]	, *args[2:]    )
                      elif len(lowerCAmelCase_    ) == 15:
                            return CodeType(*lowerCAmelCase_    )
                      return CodeType(args[0]	, 0	, *args[1:]    )
                if len(lowerCAmelCase_    ) == 16:
                      return CodeType(args[0]	, *args[3:]    )
                elif len(lowerCAmelCase_    ) == 15:
                      return CodeType(args[0]	, *args[2:]    )
                return CodeType(*lowerCAmelCase_    )
  
          def create_ipython_func(lowerCAmelCase_  : Tuple	, lowerCAmelCase_  : Any    ):
                def func():
                      return returned_obj
  
                snake_case_						       =							func.__code__
                # Use _create_code from dill in order to make it work for different python versions
                if datasets.config.PY_VERSION < version.parse("3.8"    ):
                      snake_case_						       =							_create_code_pyaa(
                          *[getattr(lowerCAmelCase_	, lowerCAmelCase_    ) if k != "co_filename" else co_filename for k in code_args_pyaa]    )
                else:
                      snake_case_						       =							code.replace(co_filename=lowerCAmelCase_    )
                return FunctionType(lowerCAmelCase_	, func.__globals__	, func.__name__	, func.__defaults__	, func.__closure__    )
  
          snake_case_       ,	snake_case_						       =							"<ipython-input-2-e0383a102aae>", [0]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
          snake_case_       ,	snake_case_						       =							"<ipython-input-2-e0383a102aae>", [1]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
          snake_case_       ,	snake_case_						       =							"<ipython-input-5-713f6613acf3>", [0]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
          snake_case_       ,	snake_case_						       =							os.path.join(gettempdir()	, "ipykernel_12345"	, "321456789.py"    ), [0]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
          snake_case_       ,	snake_case_						       =							os.path.join(gettempdir()	, "ipykernel_12345"	, "321456789.py"    ), [1]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
          snake_case_       ,	snake_case_						       =							os.path.join(gettempdir()	, "ipykernel_12345"	, "654123987.py"    ), [0]
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(create_ipython_func(lowerCAmelCase_	, lowerCAmelCase_    )    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
  
    def 		_lowerCamelCase	(self  : int    )   ->       List[Any]:
          snake_case_       ,	snake_case_						       =							[0], [1]
  
          def func():
                return foo, bar
  
          snake_case_						       =							"__main__"
  
          def globalvars_mocka_side_effect(lowerCAmelCase_  : str	, *lowerCAmelCase_  : Tuple	, **lowerCAmelCase_  : List[Any]    ):
                return {"foo": foo, "bar": bar}
  
          def globalvars_mocka_side_effect(lowerCAmelCase_  : List[str]	, *lowerCAmelCase_  : List[str]	, **lowerCAmelCase_  : Optional[Any]    ):
                return {"bar": bar, "foo": foo}
  
          with patch("dill.detect.globalvars"	, side_effect=lowerCAmelCase_    ) as globalvars_mocka:
                snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
                self.assertGreater(globalvars_mocka.call_count	, 0    )
          with patch("dill.detect.globalvars"	, side_effect=lowerCAmelCase_    ) as globalvars_mocka:
                snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
                self.assertGreater(globalvars_mocka.call_count	, 0    )
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
  
  
  class  lowerCAmelCase__   (       UpperCamelCase       ):
  
  
  
    '''simple docstring'''
  
  
  
  
  
  
    def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       Tuple:
          snake_case_						       =							Hasher.hash("hello"    )
          snake_case_						       =							Hasher.hash("hello"    )
          snake_case_						       =							Hasher.hash("there"    )
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : Any    )   ->       int:
          snake_case_						       =							Hasher.hash(Foo("hello"    )    )
          snake_case_						       =							Hasher.hash(Foo("hello"    )    )
          snake_case_						       =							Hasher.hash(Foo("there"    )    )
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : List[Any]    )   ->       List[Any]:
          snake_case_						       =							Hasher()
          for x in ["hello", Foo("hello"    )]:
                hasher.update(lowerCAmelCase_    )
          snake_case_						       =							hasher.hexdigest()
          snake_case_						       =							Hasher()
          for x in ["hello", Foo("hello"    )]:
                hasher.update(lowerCAmelCase_    )
          snake_case_						       =							hasher.hexdigest()
          snake_case_						       =							Hasher()
          for x in ["there", Foo("there"    )]:
                hasher.update(lowerCAmelCase_    )
          snake_case_						       =							hasher.hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    def 		_lowerCamelCase	(self  : str    )   ->       List[Any]:
          with self.assertRaises(pickle.PicklingError    ):
                Hasher.hash(UnpicklableCallable(Foo("hello"    )    )    )
  
    def 		_lowerCamelCase	(self  : Dict    )   ->       Optional[int]:
          snake_case_						       =							"abc"
          snake_case_						       =							[string, string]  # two strings have the same ids
          snake_case_						       =							[string, string]
          snake_case_						       =							json.loads(f'''["{string}", "{string}"]'''    )  # two strings have different ids
          self.assertIs(obja[0]	, lowerCAmelCase_    )
          self.assertIs(obja[0]	, obja[1]    )
          self.assertIs(obja[0]	, lowerCAmelCase_    )
          self.assertIs(obja[0]	, obja[1]    )
          self.assertIsNot(obja[0]	, lowerCAmelCase_    )
          self.assertIsNot(obja[0]	, obja[1]    )
          snake_case_						       =							Hasher.hash(lowerCAmelCase_    )
          snake_case_						       =							Hasher.hash(lowerCAmelCase_    )
          snake_case_						       =							Hasher.hash(lowerCAmelCase_    )
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    @require_tiktoken
    def 		_lowerCamelCase	(self  : Union[str, Any]    )   ->       Union[str, Any]:
          import tiktoken
  
          snake_case_						       =							tiktoken.get_encoding("gpt2"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							tiktoken.get_encoding("r50k_base"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							tiktoken.get_encoding("gpt2"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
    @require_torch
    def 		_lowerCamelCase	(self  : Optional[int]    )   ->       str:
          import torch
  
          snake_case_						       =							torch.tensor([1.0]    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							torch.tensor([2.0]    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							torch.tensor([1.0]    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
  
    @require_spacy
    @require_spacy_model("en_core_web_sm"    )
    @require_spacy_model("fr_core_news_sm"    )
    @pytest.mark.integration
    def 		_lowerCamelCase	(self  : int    )   ->       Union[str, Any]:
          import spacy
  
          snake_case_						       =							spacy.load("en_core_web_sm"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							spacy.load("fr_core_news_sm"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          snake_case_						       =							spacy.load("en_core_web_sm"    )
          snake_case_						       =							mda(datasets.utils.py_utils.dumps(lowerCAmelCase_    )    ).hexdigest()
          self.assertEqual(lowerCAmelCase_	, lowerCAmelCase_    )
          self.assertNotEqual(lowerCAmelCase_	, lowerCAmelCase_    )
  
  @pytest.mark.integration
  def   snake_case_       (							_a     :							Path			):
        snake_case_						       =							tmp_path / "dir1"
        snake_case_						       =							tmp_path / "dir2"
        dira.mkdir()
        dira.mkdir()
        snake_case_						       =							"script.py"
        snake_case_						       =							dedent(
            "\n    from datasets.fingerprint import Hasher\n    def foo():\n        pass\n    print(Hasher.hash(foo))\n    "			)
        snake_case_						       =							dira / script_filename
        snake_case_						       =							dira / script_filename
        with script_patha.open("w"			) as f:
              f.write(_a			)
        with script_patha.open("w"			) as f:
              f.write(_a			)
        snake_case_						       =							subprocess.check_output(["python", str(_a			)]			)
        snake_case_						       =							subprocess.check_output(["python", str(_a			)]			)
        assert fingerprinta == fingerprinta
  def   snake_case_       (							):
        snake_case_						       =							{"a": [0, 1, 2]}
        snake_case_						       =							DatasetChild(InMemoryTable.from_pydict(_a			)			)
        snake_case_						       =							dataset.funca()._fingerprint
        assert expected_fingerprint == dataset.funca()._fingerprint
        assert expected_fingerprint != dataset.funca()._fingerprint
  
        with Pool(2			) as p:
              assert expected_fingerprint == p.apply_async(dataset.funca			).get()._fingerprint
              assert expected_fingerprint != p.apply_async(dataset.funca			).get()._fingerprint
  def   snake_case_       (							):
        snake_case_						       =							{"a": [0, 1, 2]}
  
  
  
        class  lowerCAmelCase__   (       datasets.Dataset       ):
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          @fingerprint_transform(inplace=lowerCAmelCase_    )
          def 		_lowerCamelCase	(self  : int	, lowerCAmelCase_  : str    )   ->       List[Any]:
                return DummyDatasetChild(self.data	, fingerprint=lowerCAmelCase_    )
  
        snake_case_						       =							DummyDatasetChild(InMemoryTable.from_pydict(_a			)			).func()
  
  
  
        class  lowerCAmelCase__   (       datasets.Dataset       ):
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          @fingerprint_transform(inplace=lowerCAmelCase_	, version="1.0.0"    )
          def 		_lowerCamelCase	(self  : Dict	, lowerCAmelCase_  : Any    )   ->       Any:
                return DummyDatasetChild(self.data	, fingerprint=lowerCAmelCase_    )
  
        snake_case_						       =							DummyDatasetChild(InMemoryTable.from_pydict(_a			)			).func()
  
  
  
        class  lowerCAmelCase__   (       datasets.Dataset       ):
  
  
  
          '''simple docstring'''
  
  
  
  
  
  
          @fingerprint_transform(inplace=lowerCAmelCase_	, version="2.0.0"    )
          def 		_lowerCamelCase	(self  : Union[str, Any]	, lowerCAmelCase_  : Any    )   ->       int:
                return DummyDatasetChild(self.data	, fingerprint=lowerCAmelCase_    )
  
        snake_case_						       =							DummyDatasetChild(InMemoryTable.from_pydict(_a			)			).func()
  
        assert len({fingeprint_no_version, fingeprint_a, fingeprint_a}			) == 3
  
  
  
  
  def   snake_case_       (							):
        # AttributeError: module 'dill._dill' has no attribute 'stack'
        snake_case_						       =							Hasher()
        hasher.update(lambda _a			: x			)
  
  