





'''simple docstring'''


import csv
import json
import os

import datasets


# TODO: Add BibTeX citation
# Find for instance the citation on arxiv or on the dataset repo/website
lowerCAmelCase      :  List[str]      							='''\
@InProceedings{huggingface:dataset,
title = {A great new dataset},
author={huggingface, Inc.
},
year={2020}
}
'''

# TODO: Add description of the dataset here
# You can copy an official description
lowerCAmelCase      :  Any      							='''\
This new dataset is designed to solve this great NLP task and is crafted with a lot of care.
'''

# TODO: Add a link to an official homepage for the dataset here
lowerCAmelCase      :  Union[str, Any]      							=''''''

# TODO: Add the licence for the dataset here if you can find it
lowerCAmelCase      :  Union[str, Any]      							=''''''

# TODO: Add link to the official dataset URLs here
# The HuggingFace Datasets library doesn't host the datasets but only points to the original files.
# This can be an arbitrary nested dict/list of URLs (see below in `_split_generators` method)
lowerCAmelCase      :  List[str]      							={
    '''first_domain''': '''https://huggingface.co/great-new-dataset-first_domain.zip''',
    '''second_domain''': '''https://huggingface.co/great-new-dataset-second_domain.zip''',
}
class    a_     (       datasets.GeneratorBasedBuilder							):
     __A  				=    datasets.Version("1.1.0"							)

     # This is an example of a dataset with multiple configurations.
     # If you don't want/need to define several sub-sets in your dataset,
     # just remove the BUILDER_CONFIG_CLASS and the BUILDER_CONFIGS attributes.

     # If you need to make complex sub-parts in the datasets with configurable options
     # You can create your own builder configuration class to store attribute, inheriting from datasets.BuilderConfig
     # BUILDER_CONFIG_CLASS = MyBuilderConfig

     # You will be able to load one or the other configurations in the following list with
     # data = datasets.load_dataset('my_dataset', 'first_domain')
     # data = datasets.load_dataset('my_dataset', 'second_domain')
     __A  				=    [
         datasets.BuilderConfig(name="first_domain"			,		version=_lowerCAmelCase			,		description="This part of my dataset covers a first domain"							),
         datasets.BuilderConfig(name="second_domain"			,		version=_lowerCAmelCase			,		description="This part of my dataset covers a second domain"							),
     ]

     __A  				=    "first_domain"  # It's not mandatory to have a default configuration. Just use one if it make sense.





     def 							lowercase__   (				self	:      List[Any]		):





        """simple docstring"""




        if self.config.name == "first_domain":  # This is the name of the configuration selected in BUILDER_CONFIGS above
           lowercase_    :List[str]					    =				datasets.Features(
               {
                   "sentence": datasets.Value("string"		),
                   "option1": datasets.Value("string"		),
                   "answer": datasets.Value("string"		)
                   # These are the features of your dataset like images, labels ...
               }		)
        else:  # This is an example to show how to have different features for "first_domain" and "second_domain"
           lowercase_    :str					    =				datasets.Features(
               {
                   "sentence": datasets.Value("string"		),
                   "option2": datasets.Value("string"		),
                   "second_domain_answer": datasets.Value("string"		)
                   # These are the features of your dataset like images, labels ...
               }		)
        return datasets.DatasetInfo(
            # This is the description that will appear on the datasets page.
            description=_DESCRIPTION			,   features=lowercase			,   homepage=_HOMEPAGE			,   license=_LICENSE			,   citation=_CITATION			,   )





     def 							lowercase__   (				self	:      Dict			,   lowercase	:      List[str]		):





        """simple docstring"""




        lowercase_    :List[Any]					    =				_URLS[self.config.name]
        lowercase_    :Optional[int]					    =				dl_manager.download_and_extract(lowercase		)
        return [
            datasets.SplitGenerator(
                name=datasets.Split.TRAIN			,   gen_kwargs={
                    "filepath": os.path.join(lowercase			,   "train.jsonl"		),
                    "split": "train",
                }			,   ),
            datasets.SplitGenerator(
                name=datasets.Split.VALIDATION			,   gen_kwargs={
                    "filepath": os.path.join(lowercase			,   "dev.jsonl"		),
                    "split": "dev",
                }			,   ),
            datasets.SplitGenerator(
                name=datasets.Split.TEST			,   gen_kwargs={
                    "filepath": os.path.join(lowercase			,   "test.jsonl"		),
                    "split": "test"
                }			,   ),
        ]







     def 							lowercase__   (				self	:      Tuple			,   lowercase	:      int			,   lowercase	:      Union[str, Any]		):





        """simple docstring"""




        with open(lowercase			,   encoding="utf-8"		) as f:
           for key, row in enumerate(lowercase		):
              lowercase_    :List[str]					    =				json.loads(lowercase		)
              if self.config.name == "first_domain":
                 # Yields examples as (key, example) tuples
                 yield key, {
                     "sentence": data["sentence"],
                     "option1": data["option1"],
                     "answer": "" if split == "test" else data["answer"],
                 }
              else:
                 yield key, {
                     "sentence": data["sentence"],
                     "option2": data["option2"],
                     "second_domain_answer": "" if split == "test" else data["second_domain_answer"],
                 }









'''simple docstring'''


import multiprocessing
import os
from typing import BinaryIO, Optional, Union

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.csv.csv import Csv
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class    a_     (       _lowerCAmelCase							):





     def __init__(				self	:      Dict			,   lowercase	:      NestedDataStructureLike[PathLike]			,   lowercase	:      Optional[NamedSplit] = None			,   lowercase	:      Optional[Features] = None			,   lowercase	:      str = None			,   lowercase	:      bool = False			,   lowercase	:      bool = False			,   lowercase	:      Optional[int] = None			,   **lowercase	:      Any			,   ):





        """simple docstring"""




        super().__init__(
            lowercase			,   split=lowercase			,   features=lowercase			,   cache_dir=lowercase			,   keep_in_memory=lowercase			,   streaming=lowercase			,   num_proc=lowercase			,   **lowercase			,   )
        lowercase_    :Any					    =				path_or_paths if isinstance(lowercase			,   lowercase		) else {self.split: path_or_paths}
        lowercase_    :Any					    =				Csv(
            cache_dir=lowercase			,   data_files=lowercase			,   features=lowercase			,   **lowercase			,   )







     def 							lowercase__   (				self	:      Any		):





        """simple docstring"""




        if self.streaming:
           lowercase_    :Union[str, Any]					    =				self.builder.as_streaming_dataset(split=self.split		)
        # Build regular (map-style) dataset
        else:
           lowercase_    :Dict					    =				None
           lowercase_    :List[str]					    =				None
           lowercase_    :List[Any]					    =				None
           lowercase_    :Union[str, Any]					    =				None

           self.builder.download_and_prepare(
               download_config=lowercase			,   download_mode=lowercase			,   verification_mode=lowercase			,   base_path=lowercase			,   num_proc=self.num_proc			,   )
           lowercase_    :int					    =				self.builder.as_dataset(
               split=self.split			,   verification_mode=lowercase			,   in_memory=self.keep_in_memory		)
        return dataset







class    a_     :





     def __init__(				self	:      Dict			,   lowercase	:      Dataset			,   lowercase	:      Union[PathLike, BinaryIO]			,   lowercase	:      Optional[int] = None			,   lowercase	:      Optional[int] = None			,   **lowercase	:      Any			,   ):





        """simple docstring"""




        if num_proc is not None and num_proc <= 0:
           raise ValueError(F'num_proc {num_proc} must be an integer > 0.'		)

        lowercase_    :str					    =				dataset
        lowercase_    :List[Any]					    =				path_or_buf
        lowercase_    :Any					    =				batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        lowercase_    :int					    =				num_proc
        lowercase_    :Optional[Any]					    =				"utf-8"
        lowercase_    :Dict					    =				to_csv_kwargs





     def 							lowercase__   (				self	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    :int					    =				self.to_csv_kwargs.pop("path_or_buf"			,   lowercase		)
        lowercase_    :List[str]					    =				self.to_csv_kwargs.pop("header"			,   lowercase		)
        lowercase_    :Optional[Any]					    =				self.to_csv_kwargs.pop("index"			,   lowercase		)

        if isinstance(self.path_or_buf			,   (str, bytes, os.PathLike)		):
           with open(self.path_or_buf			,   "wb+"		) as buffer:
              lowercase_    :List[Any]					    =				self._write(file_obj=lowercase			,   header=lowercase			,   index=lowercase			,   **self.to_csv_kwargs		)
        else:
           lowercase_    :Any					    =				self._write(file_obj=self.path_or_buf			,   header=lowercase			,   index=lowercase			,   **self.to_csv_kwargs		)
        return written





     def 							lowercase__   (				self	:      List[Any]			,   lowercase	:      Union[str, Any]		):





        """simple docstring"""




        lowercase_    ,							lowercase_    ,							lowercase_    ,							lowercase_    :Optional[Any]					    =				args

        lowercase_    :Dict					    =				query_table(
            table=self.dataset.data			,   key=slice(lowercase			,   offset + self.batch_size		)			,   indices=self.dataset._indices			,   )
        lowercase_    :List[str]					    =				batch.to_pandas().to_csv(
            path_or_buf=lowercase			,   header=header if (offset == 0) else False			,   index=lowercase			,   **lowercase		)
        return csv_str.encode(self.encoding		)







     def 							lowercase__   (				self	:      Dict			,   lowercase	:      BinaryIO			,   lowercase	:      Dict			,   lowercase	:      int			,   **lowercase	:      Any		):





        """simple docstring"""




        lowercase_    :Dict					    =				0

        if self.num_proc is None or self.num_proc == 1:
           for offset in logging.tqdm(
               range(0			,   len(self.dataset		)			,   self.batch_size		)			,   unit="ba"			,   disable=not logging.is_progress_bar_enabled()			,   desc="Creating CSV from Arrow format"			,   ):
              lowercase_    :Optional[Any]					    =				self._batch_csv((offset, header, index, to_csv_kwargs)		)
              written += file_obj.write(lowercase		)

        else:
           lowercase_    ,							lowercase_    :Optional[int]					    =				len(self.dataset		), self.batch_size
           with multiprocessing.Pool(self.num_proc		) as pool:
              for csv_str in logging.tqdm(
                  pool.imap(
                      self._batch_csv			,   [(offset, header, index, to_csv_kwargs) for offset in range(0			,   lowercase			,   lowercase		)]			,   )			,   total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,   unit="ba"			,   disable=not logging.is_progress_bar_enabled()			,   desc="Creating CSV from Arrow format"			,   ):
                 written += file_obj.write(lowercase		)

        return written


