





"""simple docstring"""
import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases



def      __lowerCamelCase   (							a_     :		Optional[Any] ,   a_     :		Dict							)			->      str:
      assert isinstance(a_ ,   a_							)
      assert dataset.num_rows == 4
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_1", "col_2", "col_3"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize('''keep_in_memory''' ,   [False, True]							)
def      __lowerCamelCase   (							a_     :		Optional[Any] ,   a_     :		Dict ,   a_     :		Any							)			->      Optional[Any]:
      __SCREAMING_SNAKE_CASE   :Optional[int]						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :List[Any]						=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            __SCREAMING_SNAKE_CASE   :List[Any]						=  JsonDatasetReader(a_ ,   cache_dir=a_ ,   keep_in_memory=a_							).read()
      _check_json_dataset(a_ ,   a_							)



@pytest.mark.parametrize(
    '''features''' ,   [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,   )
def      __lowerCamelCase   (							a_     :		Union[str, Any] ,   a_     :		List[Any] ,   a_     :		List[str]							)			->      Any:
      __SCREAMING_SNAKE_CASE   :Dict						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :int						=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      __SCREAMING_SNAKE_CASE   :Optional[Any]						=  features.copy() if features else default_expected_features
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  (
          Features({feature: Value(a_							) for feature, dtype in features.items()}							) if features is not None else None
      )
      __SCREAMING_SNAKE_CASE   :Optional[int]						=  JsonDatasetReader(a_ ,   features=a_ ,   cache_dir=a_							).read()
      _check_json_dataset(a_ ,   a_							)



@pytest.mark.parametrize(
    '''features''' ,   [
        None,
        {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''},
    ] ,   )
def      __lowerCamelCase   (							a_     :		Union[str, Any] ,   a_     :		Tuple ,   a_     :		Tuple							)			->      Any:
      __SCREAMING_SNAKE_CASE   :int						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  {'''col_3''': '''float64''', '''col_1''': '''string''', '''col_2''': '''int64'''}
      __SCREAMING_SNAKE_CASE   :Tuple						=  features.copy() if features else default_expected_features
      __SCREAMING_SNAKE_CASE   :Dict						=  (
          Features({feature: Value(a_							) for feature, dtype in features.items()}							) if features is not None else None
      )
      __SCREAMING_SNAKE_CASE   :Tuple						=  JsonDatasetReader(a_ ,   features=a_ ,   cache_dir=a_							).read()
      assert isinstance(a_ ,   a_							)
      assert dataset.num_rows == 2
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_3", "col_1", "col_2"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype



def      __lowerCamelCase   (							a_     :		int ,   a_     :		List[Any]							)			->      Union[str, Any]:
      # jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
      __SCREAMING_SNAKE_CASE   :List[Any]						=  {'''col_2''': '''int64''', '''col_3''': '''float64''', '''col_1''': '''string'''}
      __SCREAMING_SNAKE_CASE   :Any						=  features.copy()
      __SCREAMING_SNAKE_CASE   :Optional[Any]						=  (
          Features({feature: Value(a_							) for feature, dtype in features.items()}							) if features is not None else None
      )
      __SCREAMING_SNAKE_CASE   :Dict						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  JsonDatasetReader(a_ ,   features=a_ ,   cache_dir=a_							).read()
      assert isinstance(a_ ,   a_							)
      assert dataset.num_rows == 2
      assert dataset.num_columns == 3
      assert dataset.column_names == ["col_2", "col_3", "col_1"]
      for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize('''split''' ,   [None, NamedSplit('''train'''							), '''train''', '''test''']							)
def      __lowerCamelCase   (							a_     :		Union[str, Any] ,   a_     :		str ,   a_     :		str							)			->      List[Any]:
      __SCREAMING_SNAKE_CASE   :int						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :Optional[Any]						=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      __SCREAMING_SNAKE_CASE   :Dict						=  JsonDatasetReader(a_ ,   cache_dir=a_ ,   split=a_							).read()
      _check_json_dataset(a_ ,   a_							)
      assert dataset.split == split if split else "train"



@pytest.mark.parametrize('''path_type''' ,   [str, list]							)
def      __lowerCamelCase   (							a_     :		List[Any] ,   a_     :		str ,   a_     :		Optional[Any]							)			->      Dict:
      if issubclass(a_ ,   a_							):
            __SCREAMING_SNAKE_CASE   :Any						=  jsonl_path
      elif issubclass(a_ ,   a_							):
            __SCREAMING_SNAKE_CASE   :Dict						=  [jsonl_path]
      __SCREAMING_SNAKE_CASE   :List[str]						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :int						=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      __SCREAMING_SNAKE_CASE   :str						=  JsonDatasetReader(a_ ,   cache_dir=a_							).read()
      _check_json_dataset(a_ ,   a_							)



def      __lowerCamelCase   (							a_     :		Optional[Any] ,   a_     :		Union[str, Any] ,   a_     :		Dict=("train",)							)			->      str:
      assert isinstance(a_ ,   a_							)
      for split in splits:
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  dataset_dict[split]
            assert dataset.num_rows == 4
            assert dataset.num_columns == 3
            assert dataset.column_names == ["col_1", "col_2", "col_3"]
            for feature, expected_dtype in expected_features.items():
                  assert dataset.features[feature].dtype == expected_dtype



@pytest.mark.parametrize('''keep_in_memory''' ,   [False, True]							)
def      __lowerCamelCase   (							a_     :		int ,   a_     :		Union[str, Any] ,   a_     :		Optional[Any]							)			->      Optional[Any]:
      __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :Tuple						=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
            __SCREAMING_SNAKE_CASE   :Any						=  JsonDatasetReader({'''train''': jsonl_path} ,   cache_dir=a_ ,   keep_in_memory=a_							).read()
      _check_json_datasetdict(a_ ,   a_							)



@pytest.mark.parametrize(
    '''features''' ,   [
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ] ,   )
def      __lowerCamelCase   (							a_     :		Dict ,   a_     :		int ,   a_     :		List[Any]							)			->      Optional[Any]:
      __SCREAMING_SNAKE_CASE   :List[Any]						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :Optional[int]						=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      __SCREAMING_SNAKE_CASE   :Dict						=  features.copy() if features else default_expected_features
      __SCREAMING_SNAKE_CASE   :Tuple						=  (
          Features({feature: Value(a_							) for feature, dtype in features.items()}							) if features is not None else None
      )
      __SCREAMING_SNAKE_CASE   :List[Any]						=  JsonDatasetReader({'''train''': jsonl_path} ,   features=a_ ,   cache_dir=a_							).read()
      _check_json_datasetdict(a_ ,   a_							)



@pytest.mark.parametrize('''split''' ,   [None, NamedSplit('''train'''							), '''train''', '''test''']							)
def      __lowerCamelCase   (							a_     :		int ,   a_     :		Any ,   a_     :		List[Any]							)			->      Any:
      if split:
            __SCREAMING_SNAKE_CASE   :Any						=  {split: jsonl_path}
      else:
            __SCREAMING_SNAKE_CASE   :Tuple						=  '''train'''
            __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  {'''train''': jsonl_path, '''test''': jsonl_path}
      __SCREAMING_SNAKE_CASE   :str						=  tmp_path / '''cache'''
      __SCREAMING_SNAKE_CASE   :Any						=  {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
      __SCREAMING_SNAKE_CASE   :Dict						=  JsonDatasetReader(a_ ,   cache_dir=a_							).read()
      _check_json_datasetdict(a_ ,   a_ ,   splits=list(path.keys()							)							)
      assert all(dataset[split].split == split for split in path.keys()							)



def      __lowerCamelCase   (							a_     :		List[Any]							)			->      Tuple:
      return json.load(a_							)



def      __lowerCamelCase   (							a_     :		Dict							)			->      Optional[int]:
      return [json.loads(a_							) for line in buffer]







class 					_SCREAMING_SNAKE_CASE:




       @pytest.mark.parametrize('''lines, load_json_function'''		,[(True, load_json_lines), (False, load_json)]							)
       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)			->      Union[str, Any]:





             """simple docstring"""






             with io.BytesIO() as buffer:
                   JsonDatasetWriter(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,lines=SCREAMING_SNAKE_CASE__							).write()
                   buffer.seek(0							)
                   __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  load_json_function(SCREAMING_SNAKE_CASE__							)
             assert isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)
             assert isinstance(exported_content[0]		,SCREAMING_SNAKE_CASE__							)
             assert len(SCREAMING_SNAKE_CASE__							) == 10




       @pytest.mark.parametrize(
           '''orient, container, keys, len_at'''		,[
               ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
               ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
               ('''index''', dict, set('''0123456789'''							), None),
               ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
               ('''values''', list, None, None),
               ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
           ]		,)
       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)			->      Any:





             """simple docstring"""






             with io.BytesIO() as buffer:
                   JsonDatasetWriter(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,lines=SCREAMING_SNAKE_CASE__		,orient=SCREAMING_SNAKE_CASE__							).write()
                   buffer.seek(0							)
                   __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  load_json(SCREAMING_SNAKE_CASE__							)
             assert isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)
             if keys:
                   if container is dict:
                         assert exported_content.keys() == keys
                   else:
                         assert exported_content[0].keys() == keys
             else:
                   assert not hasattr(SCREAMING_SNAKE_CASE__		,'''keys'''							) and not hasattr(exported_content[0]		,'''keys'''							)
             if len_at:
                   assert len(exported_content[len_at]							) == 10
             else:
                   assert len(SCREAMING_SNAKE_CASE__							) == 10




       @pytest.mark.parametrize('''lines, load_json_function'''		,[(True, load_json_lines), (False, load_json)]							)
       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)			->      Optional[Any]:





             """simple docstring"""






             with io.BytesIO() as buffer:
                   JsonDatasetWriter(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,lines=SCREAMING_SNAKE_CASE__		,num_proc=2							).write()
                   buffer.seek(0							)
                   __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  load_json_function(SCREAMING_SNAKE_CASE__							)
             assert isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)
             assert isinstance(exported_content[0]		,SCREAMING_SNAKE_CASE__							)
             assert len(SCREAMING_SNAKE_CASE__							) == 10




       @pytest.mark.parametrize(
           '''orient, container, keys, len_at'''		,[
               ('''records''', list, {'''tokens''', '''labels''', '''answers''', '''id'''}, None),
               ('''split''', dict, {'''columns''', '''data'''}, '''data'''),
               ('''index''', dict, set('''0123456789'''							), None),
               ('''columns''', dict, {'''tokens''', '''labels''', '''answers''', '''id'''}, '''tokens'''),
               ('''values''', list, None, None),
               ('''table''', dict, {'''schema''', '''data'''}, '''data'''),
           ]		,)
       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)			->      str:





             """simple docstring"""






             with io.BytesIO() as buffer:
                   JsonDatasetWriter(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,lines=SCREAMING_SNAKE_CASE__		,orient=SCREAMING_SNAKE_CASE__		,num_proc=2							).write()
                   buffer.seek(0							)
                   __SCREAMING_SNAKE_CASE   :Optional[int]						=  load_json(SCREAMING_SNAKE_CASE__							)
             assert isinstance(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)
             if keys:
                   if container is dict:
                         assert exported_content.keys() == keys
                   else:
                         assert exported_content[0].keys() == keys
             else:
                   assert not hasattr(SCREAMING_SNAKE_CASE__		,'''keys'''							) and not hasattr(exported_content[0]		,'''keys'''							)
             if len_at:
                   assert len(exported_content[len_at]							) == 10
             else:
                   assert len(SCREAMING_SNAKE_CASE__							) == 10




       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__							)			->      Optional[int]:





             """simple docstring"""






             with pytest.raises(SCREAMING_SNAKE_CASE__							):
                   with io.BytesIO() as buffer:
                         JsonDatasetWriter(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,num_proc=0							)




       @pytest.mark.parametrize('''compression, extension'''		,[('''gzip''', '''gz'''), ('''bz2''', '''bz2'''), ('''xz''', '''xz''')]							)
       def     _UpperCamelCase     (						self		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__							)			->      Any:





             """simple docstring"""






             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  tmp_path_factory.mktemp('''data'''							) / f'''test.json.{extension}'''
             __SCREAMING_SNAKE_CASE   :int						=  str(shared_datadir / f'''test_file.json.{extension}'''							)
             JsonDatasetWriter(SCREAMING_SNAKE_CASE__		,SCREAMING_SNAKE_CASE__		,compression=SCREAMING_SNAKE_CASE__							).write()

             with fsspec.open(SCREAMING_SNAKE_CASE__		,'''rb'''		,compression='''infer'''							) as f:
                   __SCREAMING_SNAKE_CASE   :List[Any]						=  f.read()
             with fsspec.open(SCREAMING_SNAKE_CASE__		,'''rb'''		,compression='''infer'''							) as f:
                   __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  f.read()
             assert exported_content == original_content






"""simple docstring"""
from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCamelCase_								=		logging.get_logger(__name__)

lowerCamelCase_								=		{
    "microsoft/beit-base-patch16-224-pt22k": (
        "https://huggingface.co/microsoft/beit-base-patch16-224-pt22k/resolve/main/config.json"
    ),
    # See all BEiT models at https://huggingface.co/models?filter=beit
}






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      str							=  '''beit'''




       def __init__(						self		,SCREAMING_SNAKE_CASE__=81_92		,SCREAMING_SNAKE_CASE__=7_68		,SCREAMING_SNAKE_CASE__=12		,SCREAMING_SNAKE_CASE__=12		,SCREAMING_SNAKE_CASE__=30_72		,SCREAMING_SNAKE_CASE__="gelu"		,SCREAMING_SNAKE_CASE__=0.0		,SCREAMING_SNAKE_CASE__=0.0		,SCREAMING_SNAKE_CASE__=0.0_2		,SCREAMING_SNAKE_CASE__=1E-12		,SCREAMING_SNAKE_CASE__=2_24		,SCREAMING_SNAKE_CASE__=16		,SCREAMING_SNAKE_CASE__=3		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=0.1		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=[3, 5, 7, 11]		,SCREAMING_SNAKE_CASE__=[1, 2, 3, 6]		,SCREAMING_SNAKE_CASE__=True		,SCREAMING_SNAKE_CASE__=0.4		,SCREAMING_SNAKE_CASE__=2_56		,SCREAMING_SNAKE_CASE__=1		,SCREAMING_SNAKE_CASE__=False		,SCREAMING_SNAKE_CASE__=2_55		,**SCREAMING_SNAKE_CASE__		,)			->      Optional[Any]:





             """simple docstring"""






             super().__init__(**SCREAMING_SNAKE_CASE__							)

             __SCREAMING_SNAKE_CASE   :Any						=  vocab_size
             __SCREAMING_SNAKE_CASE   :Dict						=  hidden_size
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  num_hidden_layers
             __SCREAMING_SNAKE_CASE   :List[str]						=  num_attention_heads
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  intermediate_size
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  hidden_act
             __SCREAMING_SNAKE_CASE   :Tuple						=  hidden_dropout_prob
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  attention_probs_dropout_prob
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  initializer_range
             __SCREAMING_SNAKE_CASE   :str						=  layer_norm_eps

             __SCREAMING_SNAKE_CASE   :int						=  image_size
             __SCREAMING_SNAKE_CASE   :Tuple						=  patch_size
             __SCREAMING_SNAKE_CASE   :Any						=  num_channels
             __SCREAMING_SNAKE_CASE   :Any						=  use_mask_token
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_absolute_position_embeddings
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_relative_position_bias
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  use_shared_relative_position_bias
             __SCREAMING_SNAKE_CASE   :List[str]						=  layer_scale_init_value
             __SCREAMING_SNAKE_CASE   :Optional[Any]						=  drop_path_rate
             __SCREAMING_SNAKE_CASE   :str						=  use_mean_pooling
             # decode head attributes (semantic segmentation)
             __SCREAMING_SNAKE_CASE   :Dict						=  out_indices
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  pool_scales
             # auxiliary head attributes (semantic segmentation)
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  use_auxiliary_head
             __SCREAMING_SNAKE_CASE   :Union[str, Any]						=  auxiliary_loss_weight
             __SCREAMING_SNAKE_CASE   :Dict						=  auxiliary_channels
             __SCREAMING_SNAKE_CASE   :Optional[int]						=  auxiliary_num_convs
             __SCREAMING_SNAKE_CASE   :List[str]						=  auxiliary_concat_input
             __SCREAMING_SNAKE_CASE   :List[Any]						=  semantic_loss_ignore_index






class 					_SCREAMING_SNAKE_CASE(   A  ):
       SCREAMING_SNAKE_CASE_   :      List[Any]							=  version.parse('''1.11'''  )




       @property
       def     _UpperCamelCase     (						self							)			->      Mapping[str, Mapping[int, str]]:





             """simple docstring"""






             return OrderedDict(
                 [
                     ('''pixel_values''', {0: '''batch''', 1: '''num_channels''', 2: '''height''', 3: '''width'''}),
                 ]							)




       @property
       def     _UpperCamelCase     (						self							)			->      float:





             """simple docstring"""






             return 1E-4