


'''simple docstring'''
# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import importlib.metadata
import json
import os
from dataclasses import dataclass
from typing import Any, Dict, Union

from packaging import version

from ..utils import is_torch_available, logging


if is_torch_available():
  import torch


UpperCamelCase__    					=      logging.get_logger(__name__)




@dataclass
class 			lowerCamelCase_   :






      def __init__(					self				:    str						,						_A				:    Union[str, Any]=False						,						_A				:    Tuple=False						,						_A				:    Union[str, Any]=6.0						,						_A				:    Any=None						,						_A				:    Tuple=False						,						_A				:    Optional[int]=False						,						_A				:    Tuple=None						,						_A				:    Union[str, Any]="fp4"						,						_A				:    Tuple=False						,						**_A				:    str						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				load_in_abit
          UpperCAmelCase__			:		str													=				load_in_abit
          UpperCAmelCase__			:		List[str]													=				llm_inta_threshold
          UpperCAmelCase__			:		Tuple													=				llm_inta_skip_modules
          UpperCAmelCase__			:		str													=				llm_inta_enable_fpaa_cpu_offload
          UpperCAmelCase__			:		Tuple													=				llm_inta_has_fpaa_weight
          UpperCAmelCase__			:		Tuple													=				bnb_abit_quant_type
          UpperCAmelCase__			:		int													=				bnb_abit_use_double_quant

          if bnb_abit_compute_dtype is None:
              UpperCAmelCase__			:		str													=				torch.floataa
          elif isinstance(_A						,						_A ):
              UpperCAmelCase__			:		Optional[int]													=				getattr(_A						,						_A )
          elif isinstance(_A						,						torch.dtype ):
              UpperCAmelCase__			:		Union[str, Any]													=				bnb_abit_compute_dtype
          else:
              raise ValueError('''bnb_4bit_compute_dtype must be a string or a torch.dtype''' )

          self.post_init()






      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          if not isinstance(self.llm_inta_threshold						,						_A ):
              raise ValueError('''llm_int8_threshold must be a float''' )

          if self.llm_inta_skip_modules is not None and not isinstance(self.llm_inta_skip_modules						,						_A ):
              raise ValueError('''llm_int8_skip_modules must be a list of strings''' )
          if not isinstance(self.llm_inta_enable_fpaa_cpu_offload						,						_A ):
              raise ValueError('''llm_int8_enable_fp32_cpu_offload must be a boolean''' )

          if not isinstance(self.llm_inta_has_fpaa_weight						,						_A ):
              raise ValueError('''llm_int8_has_fp16_weight must be a boolean''' )

          if self.bnb_abit_compute_dtype is not None and not isinstance(self.bnb_abit_compute_dtype						,						torch.dtype ):
              raise ValueError('''bnb_4bit_compute_dtype must be torch.dtype''' )

          if not isinstance(self.bnb_abit_quant_type						,						_A ):
              raise ValueError('''bnb_4bit_quant_type must be a string''' )

          if not isinstance(self.bnb_abit_use_double_quant						,						_A ):
              raise ValueError('''bnb_4bit_use_double_quant must be a boolean''' )

          if self.load_in_abit and not version.parse(importlib.metadata.version('''bitsandbytes''' ) ) >= version.parse(
              '''0.39.0''' ):
              raise ValueError(
                  '''4 bit quantization requires bitsandbytes>=0.39.0 - please upgrade your bitsandbytes version''' )






      def lowercase_							(					self				:    Optional[int] ):


          '''simple docstring'''


          return self.load_in_abit or self.load_in_abit






      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          if self.load_in_abit:
              return "llm_int8"
          elif self.load_in_abit and self.bnb_abit_quant_type == "fp4":
              return "fp4"
          elif self.load_in_abit and self.bnb_abit_quant_type == "nf4":
              return "nf4"
          else:
              return None






      @classmethod
      def lowercase_							(					cls				:    Tuple						,						_A				:    int						,						_A				:    Union[str, Any]						,						**_A				:    Tuple ):


          '''simple docstring'''


          UpperCAmelCase__			:		Dict													=				cls(**_A )

          UpperCAmelCase__			:		int													=				[]
          for key, value in kwargs.items():
              if hasattr(_A						,						_A ):
                  setattr(_A						,						_A						,						_A )
                  to_remove.append(_A )
          for key in to_remove:
              kwargs.pop(_A						,						_A )

          if return_unused_kwargs:
              return config, kwargs
          else:
              return config






      def lowercase_							(					self				:    Optional[int]						,						_A				:    Union[str, os.PathLike] ):


          '''simple docstring'''


          with open(_A						,						'''w'''						,						encoding='''utf-8''' ) as writer:
              UpperCAmelCase__			:		List[str]													=				self.to_dict()
              UpperCAmelCase__			:		Optional[int]													=				json.dumps(_A						,						indent=2						,						sort_keys=_A ) + '''\n'''

              writer.write(_A )






      def lowercase_							(					self				:    str ):


          '''simple docstring'''


          UpperCAmelCase__			:		Tuple													=				copy.deepcopy(self.__dict__ )
          UpperCAmelCase__			:		List[str]													=				str(output['''bnb_4bit_compute_dtype'''] ).split('''.''' )[1]

          return output






      def __repr__(					self				:    Tuple ):


          '''simple docstring'''


          return f"""{self.__class__.__name__} {self.to_json_string()}"""






      def lowercase_							(					self				:    Optional[Any]						,						_A				:    bool = True ):


          '''simple docstring'''


          if use_diff is True:
              UpperCAmelCase__			:		str													=				self.to_diff_dict()
          else:
              UpperCAmelCase__			:		List[Any]													=				self.to_dict()
          return json.dumps(_A						,						indent=2						,						sort_keys=_A ) + "\n"






      def lowercase_							(					self				:    List[Any] ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[Any]													=				self.to_dict()

          # get the default config dict
          UpperCAmelCase__			:		str													=				BitsAndBytesConfig().to_dict()

          UpperCAmelCase__			:		Union[str, Any]													=				{}

          # only serialize values that differ from the default config
          for key, value in config_dict.items():
              if value != default_config_dict[key]:
                  UpperCAmelCase__			:		str													=				value

          return serializable_config_dict




'''simple docstring'''
import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader




class 			lowerCamelCase_   (	__a      ):






      def __init__(					self				:    List[str]						,						_A				:    NestedDataStructureLike[PathLike]						,						_A				:    Optional[NamedSplit] = None						,						_A				:    Optional[Features] = None						,						_A				:    str = None						,						_A				:    bool = False						,						_A				:    bool = False						,						_A				:    Optional[str] = None						,						_A				:    Optional[int] = None						,						**_A				:    Union[str, Any]						,						):


          '''simple docstring'''


          super().__init__(
              _A						,						split=_A						,						features=_A						,						cache_dir=_A						,						keep_in_memory=_A						,						streaming=_A						,						num_proc=_A						,						**_A						,						)
          UpperCAmelCase__			:		List[str]													=				field
          UpperCAmelCase__			:		Optional[Any]													=				path_or_paths if isinstance(_A						,						_A ) else {self.split: path_or_paths}
          UpperCAmelCase__			:		Any													=				Json(
              cache_dir=_A						,						data_files=_A						,						features=_A						,						field=_A						,						**_A						,						)






      def lowercase_							(					self				:    Dict ):


          '''simple docstring'''


          if self.streaming:
              UpperCAmelCase__			:		int													=				self.builder.as_streaming_dataset(split=self.split )
          # Build regular (map-style) dataset
          else:
              UpperCAmelCase__			:		List[str]													=				None
              UpperCAmelCase__			:		Dict													=				None
              UpperCAmelCase__			:		Optional[int]													=				None
              UpperCAmelCase__			:		List[Any]													=				None

              self.builder.download_and_prepare(
                  download_config=_A						,						download_mode=_A						,						verification_mode=_A						,						base_path=_A						,						num_proc=self.num_proc						,						)
              UpperCAmelCase__			:		str													=				self.builder.as_dataset(
                  split=self.split						,						verification_mode=_A						,						in_memory=self.keep_in_memory )
          return dataset




class 			lowerCamelCase_   :






      def __init__(					self				:    Optional[Any]						,						_A				:    Dataset						,						_A				:    Union[PathLike, BinaryIO]						,						_A				:    Optional[int] = None						,						_A				:    Optional[int] = None						,						**_A				:    List[str]						,						):


          '''simple docstring'''


          if num_proc is not None and num_proc <= 0:
              raise ValueError(f"""num_proc {num_proc} must be an integer > 0.""" )

          UpperCAmelCase__			:		Dict													=				dataset
          UpperCAmelCase__			:		Any													=				path_or_buf
          UpperCAmelCase__			:		List[str]													=				batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
          UpperCAmelCase__			:		Tuple													=				num_proc
          UpperCAmelCase__			:		Any													=				'''utf-8'''
          UpperCAmelCase__			:		Optional[int]													=				to_json_kwargs






      def lowercase_							(					self				:    int ):


          '''simple docstring'''


          UpperCAmelCase__			:		List[str]													=				self.to_json_kwargs.pop('''path_or_buf'''						,						_A )
          UpperCAmelCase__			:		Optional[int]													=				self.to_json_kwargs.pop('''orient'''						,						'''records''' )
          UpperCAmelCase__			:		Tuple													=				self.to_json_kwargs.pop('''lines'''						,						True if orient == '''records''' else False )
          UpperCAmelCase__			:		str													=				self.to_json_kwargs.pop('''index'''						,						False if orient in ['''split''', '''table'''] else True )
          UpperCAmelCase__			:		Optional[Any]													=				self.to_json_kwargs.pop('''compression'''						,						_A )

          if compression not in [None, "infer", "gzip", "bz2", "xz"]:
              raise NotImplementedError(f"""`datasets` currently does not support {compression} compression""" )

          if isinstance(self.path_or_buf						,						(str, bytes, os.PathLike) ):
              with fsspec.open(self.path_or_buf						,						'''wb'''						,						compression=_A ) as buffer:
                  UpperCAmelCase__			:		Union[str, Any]													=				self._write(file_obj=_A						,						orient=_A						,						lines=_A						,						index=_A						,						**self.to_json_kwargs )
          else:
              if compression:
                  raise NotImplementedError(
                      f"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
                      ''' was passed. Please provide a local path instead.''' )
              UpperCAmelCase__			:		Union[str, Any]													=				self._write(
                  file_obj=self.path_or_buf						,						orient=_A						,						lines=_A						,						index=_A						,						**self.to_json_kwargs )
          return written






      def lowercase_							(					self				:    Optional[int]						,						_A				:    Optional[Any] ):


          '''simple docstring'''


          UpperCAmelCase__						,						UpperCAmelCase__						,						UpperCAmelCase__						,						UpperCAmelCase__						,						UpperCAmelCase__			:		Dict													=				args

          UpperCAmelCase__			:		Dict													=				query_table(
              table=self.dataset.data						,						key=slice(_A						,						offset + self.batch_size )						,						indices=self.dataset._indices						,						)
          UpperCAmelCase__			:		str													=				batch.to_pandas().to_json(
              path_or_buf=_A						,						orient=_A						,						lines=_A						,						index=_A						,						**_A )
          if not json_str.endswith('''\n''' ):
              json_str += "\n"
          return json_str.encode(self.encoding )






      def lowercase_							(					self				:    Union[str, Any]						,						_A				:    BinaryIO						,						_A				:    Optional[int]						,						_A				:    int						,						_A				:    Any						,						**_A				:    Any						,						):


          '''simple docstring'''


          UpperCAmelCase__			:		Optional[Any]													=				0

          if self.num_proc is None or self.num_proc == 1:
              for offset in logging.tqdm(
                  range(0						,						len(self.dataset )						,						self.batch_size )						,						unit='''ba'''						,						disable=not logging.is_progress_bar_enabled()						,						desc='''Creating json from Arrow format'''						,						):
                  UpperCAmelCase__			:		Optional[int]													=				self._batch_json((offset, orient, lines, index, to_json_kwargs) )
                  written += file_obj.write(_A )
          else:
              UpperCAmelCase__						,						UpperCAmelCase__			:		str													=				len(self.dataset ), self.batch_size
              with multiprocessing.Pool(self.num_proc ) as pool:
                  for json_str in logging.tqdm(
                      pool.imap(
                          self._batch_json						,						[(offset, orient, lines, index, to_json_kwargs) for offset in range(0						,						_A						,						_A )]						,						)						,						total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size						,						unit='''ba'''						,						disable=not logging.is_progress_bar_enabled()						,						desc='''Creating json from Arrow format'''						,						):
                      written += file_obj.write(_A )

          return written
