

'''simple docstring'''

import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


a   :			Optional[int]         = datasets.utils.logging.get_logger(__name__)







@dataclass
class   UpperCamelCase__    (   datasets.BuilderConfig   ):


		"""simple docstring"""

		SCREAMING_SNAKE_CASE__  :	Optional[Any]   =							None
		SCREAMING_SNAKE_CASE__  :	Tuple   =							"utf-8"
		SCREAMING_SNAKE_CASE__  :	Dict   =							None
		SCREAMING_SNAKE_CASE__  :	Any   =							None
		SCREAMING_SNAKE_CASE__  :	List[str]   =							True  # deprecated
		SCREAMING_SNAKE_CASE__  :	List[Any]   =							None  # deprecated
		SCREAMING_SNAKE_CASE__  :	str   =							10 << 20  # 10MB
		SCREAMING_SNAKE_CASE__  :	Any   =							None







class   UpperCamelCase__    (   datasets.ArrowBasedBuilder   ):


		"""simple docstring"""

		SCREAMING_SNAKE_CASE__  :	List[Any]   =							JsonConfig







		def       A_       ( self			):


							'''simple docstring'''

							if self.config.block_size is not None:
												logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"			)
												UpperCAmelCase		:							Tuple   					=   self.config.block_size
							if self.config.use_threads is not True:
												logger.warning(
												    "The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore."			)
							if self.config.newlines_in_values is not None:
												raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"			)
							return datasets.DatasetInfo(features=self.config.features			)







		def       A_       ( self    ,			snake_case			):


							'''simple docstring'''

							if not self.config.data_files:
												raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"			)
							UpperCAmelCase		:							Tuple   					=   dl_manager.download_and_extract(self.config.data_files			)
							if isinstance(snake_case    ,			(str, list, tuple)			):
												UpperCAmelCase		:							Optional[int]   					=   data_files
												if isinstance(snake_case    ,			snake_case			):
																	UpperCAmelCase		:							Optional[Any]   					=   [files]
												UpperCAmelCase		:							Any   					=   [dl_manager.iter_files(snake_case			) for file in files]
												return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,			gen_kwargs={"files": files}			)]
							UpperCAmelCase		:							List[str]   					=   []
							for split_name, files in data_files.items():
												if isinstance(snake_case    ,			snake_case			):
																	UpperCAmelCase		:							List[str]   					=   [files]
												UpperCAmelCase		:							Union[str, Any]   					=   [dl_manager.iter_files(snake_case			) for file in files]
												splits.append(datasets.SplitGenerator(name=snake_case    ,			gen_kwargs={"files": files}			)			)
							return splits







		def       A_       ( self    ,			snake_case			):


							'''simple docstring'''

							if self.config.features is not None:
												# adding missing columns
												for column_name in set(self.config.features			) - set(pa_table.column_names			):
																	UpperCAmelCase		:							Tuple   					=   self.config.features.arrow_schema.field(snake_case			).type
																	UpperCAmelCase		:							Tuple   					=   pa_table.append_column(snake_case    ,			pa.array([None] * len(snake_case			)    ,			type=snake_case			)			)
												# more expensive cast to support nested structures with keys in a different order
												# allows str <-> int/float or str to Audio for example
												UpperCAmelCase		:							List[str]   					=   table_cast(snake_case    ,			self.config.features.arrow_schema			)
							return pa_table







		def       A_       ( self    ,			snake_case			):


							'''simple docstring'''

							for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case			)			):
												# If the file is one json object and if we need to look at the list of items in one specific field
												if self.config.field is not None:
																	with open(snake_case    ,			encoding=self.config.encoding    ,			errors=self.config.encoding_errors			) as f:
																						UpperCAmelCase		:							str   					=   json.load(snake_case			)

																	# We keep only the field we are interested in
																	UpperCAmelCase		:							List[Any]   					=   dataset[self.config.field]

																	# We accept two format: a list of dicts or a dict of lists
																	if isinstance(snake_case    ,			(list, tuple)			):
																						UpperCAmelCase		:							str   					=   set().union(*[row.keys() for row in dataset]			)
																						UpperCAmelCase		:							List[Any]   					=   {col: [row.get(snake_case			) for row in dataset] for col in keys}
																	else:
																						UpperCAmelCase		:							Any   					=   dataset
																	UpperCAmelCase		:							Dict   					=   pa.Table.from_pydict(snake_case			)
																	yield file_idx, self._cast_table(snake_case			)

												# If the file has one json object per line
												else:
																	with open(snake_case    ,			"rb"			) as f:
																						UpperCAmelCase		:							Union[str, Any]   					=   0
																						# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																						# Set a default minimum value of 16kB if the chunk size is really small
																						UpperCAmelCase		:							Any   					=   max(self.config.chunksize // 3_2    ,			1_6 << 1_0			)
																						UpperCAmelCase		:							str   					=   (
																						    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																						)
																						while True:
																											UpperCAmelCase		:							int   					=   f.read(self.config.chunksize			)
																											if not batch:
																																break
																											# Finish current line
																											try:
																																batch += f.readline()
																											except (AttributeError, io.UnsupportedOperation):
																																batch += readline(snake_case			)
																											# PyArrow only accepts utf-8 encoded bytes
																											if self.config.encoding != "utf-8":
																																UpperCAmelCase		:							Tuple   					=   batch.decode(self.config.encoding    ,			errors=snake_case			).encode("utf-8"			)
																											try:
																																while True:
																																					try:
																																										UpperCAmelCase		:							Optional[Any]   					=   paj.read_json(
																																										    io.BytesIO(snake_case			)    ,			read_options=paj.ReadOptions(block_size=snake_case			)			)
																																										break
																																					except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																										if (
																																										    isinstance(snake_case    ,			pa.ArrowInvalid			)
																																										    and "straddling" not in str(snake_case			)
																																										    or block_size > len(snake_case			)
																																										):
																																															raise
																																										else:
																																															# Increase the block size in case it was too small.
																																															# The block size will be reset for the next file.
																																															logger.debug(
																																															    f"Batch of {len(snake_case			)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."			)
																																															block_size *= 2
																											except pa.ArrowInvalid as e:
																																try:
																																					with open(
																																					    snake_case    ,			encoding=self.config.encoding    ,			errors=self.config.encoding_errors			) as f:
																																										UpperCAmelCase		:							Dict   					=   json.load(snake_case			)
																																except json.JSONDecodeError:
																																					logger.error(f"Failed to read file \'{file}\' with error {type(snake_case			)}: {e}"			)
																																					raise e
																																# If possible, parse the file as a list of json objects and exit the loop
																																if isinstance(snake_case    ,			snake_case			):  # list is the only sequence type supported in JSON
																																					try:
																																										UpperCAmelCase		:							Dict   					=   set().union(*[row.keys() for row in dataset]			)
																																										UpperCAmelCase		:							Any   					=   {col: [row.get(snake_case			) for row in dataset] for col in keys}
																																										UpperCAmelCase		:							List[Any]   					=   pa.Table.from_pydict(snake_case			)
																																					except (pa.ArrowInvalid, AttributeError) as e:
																																										logger.error(f"Failed to read file \'{file}\' with error {type(snake_case			)}: {e}"			)
																																										raise ValueError(f"Not able to read records in the JSON file at {file}."			) from None
																																					yield file_idx, self._cast_table(snake_case			)
																																					break
																																else:
																																					logger.error(f"Failed to read file \'{file}\' with error {type(snake_case			)}: {e}"			)
																																					raise ValueError(
																																					    f"Not able to read records in the JSON file at {file}. "
																																					    f"You should probably indicate the field of the JSON file containing your records. "
																																					    f"This JSON file contain the following fields: {str(list(dataset.keys()			)			)}. "
																																					    f"Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. "			) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																											yield (file_idx, batch_idx), self._cast_table(snake_case			)
																											batch_idx += 1


class      A__							:






				def __init__(						self					:      Dict    ,     a					:      list[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		len(a   )
							lowerCAmelCase__      :      Optional[Any]         =		[0] * len_array

							if len_array > 0:
										lowerCAmelCase__      :      List[Any]         =		array[0]

							for i in range(1    ,     a   ):
										lowerCAmelCase__      :      Optional[int]         =		self.prefix_sum[i - 1] + array[i]






				def 							_lowerCamelCase		(						self					:      int    ,     a					:      int    ,     a					:      int   ):






							'''simple docstring'''







							if start == 0:
										return self.prefix_sum[end]

							return self.prefix_sum[end] - self.prefix_sum[start - 1]






				def 							_lowerCamelCase		(						self					:      Optional[Any]    ,     a					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		{0}
							for sum_item in self.prefix_sum:
										if sum_item - target_sum in sums:
													return True

										sums.add(a   )

							return False


if __name__ == "__main__":
					import doctest

					doctest.testmod()