import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast





@dataclass
class 					lowercase__						(   datasets.BuilderConfig):
						UpperCamelCase_     =		None






class 					lowercase__						(   datasets.ArrowBasedBuilder):
						UpperCamelCase_     =		PandasConfig


						def        __A       (       self	:				Optional[Any]						):





												'''simple docstring'''





												return datasets.DatasetInfo(features=self.config.features						)


						def        __A       (       self	:				Tuple    ,		UpperCamelCase__	:				Any						):





												'''simple docstring'''





												if not self.config.data_files:
																		raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""						)
												SCREAMING_SNAKE_CASE							:    Tuple							=							dl_manager.download_and_extract(self.config.data_files						)
												if isinstance(UpperCamelCase__    ,		(str, list, tuple)						):
																		SCREAMING_SNAKE_CASE							:    int							=							data_files
																		if isinstance(UpperCamelCase__    ,		UpperCamelCase__						):
																								SCREAMING_SNAKE_CASE							:    List[str]							=							[files]
																		# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																		SCREAMING_SNAKE_CASE							:    Dict							=							[dl_manager.iter_files(UpperCamelCase__						) for file in files]
																		return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,		gen_kwargs={'''files''': files}						)]
												SCREAMING_SNAKE_CASE							:    List[Any]							=							[]
												for split_name, files in data_files.items():
																		if isinstance(UpperCamelCase__    ,		UpperCamelCase__						):
																								SCREAMING_SNAKE_CASE							:    List[str]							=							[files]
																		# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																		SCREAMING_SNAKE_CASE							:    Union[str, Any]							=							[dl_manager.iter_files(UpperCamelCase__						) for file in files]
																		splits.append(datasets.SplitGenerator(name=UpperCamelCase__    ,		gen_kwargs={'''files''': files}						)						)
												return splits


						def        __A       (       self	:				int    ,		UpperCamelCase__	:				pa.Table						):





												'''simple docstring'''





												if self.config.features is not None:
																		# more expensive cast to support nested features with keys in a different order
																		# allows str <-> int/float or str to Audio for example
																		SCREAMING_SNAKE_CASE							:    int							=							table_cast(UpperCamelCase__    ,		self.config.features.arrow_schema						)
												return pa_table



						def        __A       (       self	:				Union[str, Any]    ,		UpperCamelCase__	:				Union[str, Any]						):





												'''simple docstring'''





												for i, file in enumerate(itertools.chain.from_iterable(UpperCamelCase__						)						):
																		with open(UpperCamelCase__    ,		'''rb'''						) as f:
																								SCREAMING_SNAKE_CASE							:    List[str]							=							pa.Table.from_pandas(pd.read_pickle(UpperCamelCase__						)						)
																								yield i, self._cast_table(UpperCamelCase__						)






import pytest

from datasets.splits import SplitDict, SplitInfo
from datasets.utils.py_utils import asdict
@pytest.mark.parametrize(
    '''split_dict'''		,  [
        SplitDict(),
        SplitDict({'''train''': SplitInfo(name='''train'''		,  num_bytes=1_337		,  num_examples=42		,  dataset_name='''my_dataset'''						)}						),
        SplitDict({'''train''': SplitInfo(name='''train'''		,  num_bytes=1_337		,  num_examples=42						)}						),
        SplitDict({'''train''': SplitInfo()}						),
    ]		,  )
def       A	(    _lowercase						):
						SCREAMING_SNAKE_CASE							:    Tuple							=							split_dict._to_yaml_list()
						assert len(_lowercase						) == len(_lowercase						)
						SCREAMING_SNAKE_CASE							:    Tuple							=							SplitDict._from_yaml_list(_lowercase						)
						for split_name, split_info in split_dict.items():
												# dataset_name field is deprecated, and is therefore not part of the YAML dump
												SCREAMING_SNAKE_CASE							:    Any							=							None
												# the split name of split_dict takes over the name of the split info object
												SCREAMING_SNAKE_CASE							:    Optional[Any]							=							split_name
						assert split_dict == reloaded



@pytest.mark.parametrize(
    '''split_info'''		,  [SplitInfo(), SplitInfo(dataset_name=_lowercase						), SplitInfo(dataset_name='''my_dataset'''						)]						)
def       A	(    _lowercase						):
						# For backward compatibility, we need asdict(split_dict) to return split info dictrionaries with the "dataset_name"
						# field even if it's deprecated. This way old versionso of `datasets` can still reload dataset_infos.json files
						SCREAMING_SNAKE_CASE							:    List[Any]							=							asdict(SplitDict({'''train''': split_info}						)						)
						assert "dataset_name" in split_dict_asdict["train"]
						assert split_dict_asdict["train"]["dataset_name"] == split_info.dataset_name





