


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


snake_case							:       str     			=      datasets.utils.logging.get_logger(__name__)







@dataclass
class        snake_case_   (datasets.BuilderConfig      ):
					UpperCAmelCase__					:    Optional[datasets.Features]          =			None
					UpperCAmelCase__					:    str          =			"utf-8"
					UpperCAmelCase__					:    Optional[str]          =			None
					UpperCAmelCase__					:    Optional[str]          =			None
					UpperCAmelCase__					:    bool          =			True  # deprecated
					UpperCAmelCase__					:    Optional[int]          =			None  # deprecated
					UpperCAmelCase__					:    int          =			1_0 << 2_0  # 10MB
					UpperCAmelCase__					:    Optional[bool]          =			None







class        snake_case_   (datasets.ArrowBasedBuilder      ):
					UpperCAmelCase__					:    Any          =			JsonConfig
					def  lowerCamelCase__(      self   :Dict      )					->    Optional[int]:
												if self.config.block_size is not None:
																			logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'      )
																			a__             =     self.config.block_size
												if self.config.use_threads is not True:
																			logger.warning(
																			    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'      )
												if self.config.newlines_in_values is not None:
																			raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'      )
												return datasets.DatasetInfo(features=self.config.features      )
					def  lowerCamelCase__(      self   :List[Any]			,__snake_case   :int      )					->    Optional[Any]:
												if not self.config.data_files:
																			raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'      )
												a__             =     dl_manager.download_and_extract(self.config.data_files      )
												if isinstance(__snake_case			,(str, list, tuple)      ):
																			a__             =     data_files
																			if isinstance(__snake_case			,__snake_case      ):
																										a__             =     [files]
																			a__             =     [dl_manager.iter_files(__snake_case      ) for file in files]
																			return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,gen_kwargs={'files': files}      )]
												a__             =     []
												for split_name, files in data_files.items():
																			if isinstance(__snake_case			,__snake_case      ):
																										a__             =     [files]
																			a__             =     [dl_manager.iter_files(__snake_case      ) for file in files]
																			splits.append(datasets.SplitGenerator(name=__snake_case			,gen_kwargs={'files': files}      )      )
												return splits
					def  lowerCamelCase__(      self   :int			,__snake_case   :pa.Table      )					->    pa.Table:
												if self.config.features is not None:
																			# adding missing columns
																			for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
																										a__             =     self.config.features.arrow_schema.field(__snake_case      ).type
																										a__             =     pa_table.append_column(__snake_case			,pa.array([None] * len(__snake_case      )			,type=__snake_case      )      )
																			# more expensive cast to support nested structures with keys in a different order
																			# allows str <-> int/float or str to Audio for example
																			a__             =     table_cast(__snake_case			,self.config.features.arrow_schema      )
												return pa_table



					def  lowerCamelCase__(      self   :Dict			,__snake_case   :List[str]      )					->    List[Any]:
												for file_idx, file in enumerate(itertools.chain.from_iterable(__snake_case      )      ):
																			# If the file is one json object and if we need to look at the list of items in one specific field
																			if self.config.field is not None:
																										with open(__snake_case			,encoding=self.config.encoding			,errors=self.config.encoding_errors      ) as f:
																																	a__             =     json.load(__snake_case      )

																										# We keep only the field we are interested in
																										a__             =     dataset[self.config.field]

																										# We accept two format: a list of dicts or a dict of lists
																										if isinstance(__snake_case			,(list, tuple)      ):
																																	a__             =     set().union(*[row.keys() for row in dataset]      )
																																	a__             =     {col: [row.get(__snake_case      ) for row in dataset] for col in keys}
																										else:
																																	a__             =     dataset
																										a__             =     pa.Table.from_pydict(__snake_case      )
																										yield file_idx, self._cast_table(__snake_case      )

																			# If the file has one json object per line
																			else:
																										with open(__snake_case			,'rb'      ) as f:
																																	a__             =     0
																																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																																	# Set a default minimum value of 16kB if the chunk size is really small
																																	a__             =     max(self.config.chunksize // 32			,16 << 10      )
																																	a__             =     (
																																	    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																																	)
																																	while True:
																																								a__             =     f.read(self.config.chunksize      )
																																								if not batch:
																																															break
																																								# Finish current line
																																								try:
																																															batch += f.readline()
																																								except (AttributeError, io.UnsupportedOperation):
																																															batch += readline(__snake_case      )
																																								# PyArrow only accepts utf-8 encoded bytes
																																								if self.config.encoding != "utf-8":
																																															a__             =     batch.decode(self.config.encoding			,errors=__snake_case      ).encode('utf-8'      )
																																								try:
																																															while True:
																																																						try:
																																																													a__             =     paj.read_json(
																																																													    io.BytesIO(__snake_case      )			,read_options=paj.ReadOptions(block_size=__snake_case      )      )
																																																													break
																																																						except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																													if (
																																																													    isinstance(__snake_case			,pa.ArrowInvalid      )
																																																													    and "straddling" not in str(__snake_case      )
																																																													    or block_size > len(__snake_case      )
																																																													):
																																																																				raise
																																																													else:
																																																																				# Increase the block size in case it was too small.
																																																																				# The block size will be reset for the next file.
																																																																				logger.debug(
																																																																				    F'Batch of {len(__snake_case      )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'      )
																																																																				block_size *= 2
																																								except pa.ArrowInvalid as e:
																																															try:
																																																						with open(
																																																						    __snake_case			,encoding=self.config.encoding			,errors=self.config.encoding_errors      ) as f:
																																																													a__             =     json.load(__snake_case      )
																																															except json.JSONDecodeError:
																																																						logger.error(F'Failed to read file \'{file}\' with error {type(__snake_case      )}: {e}'      )
																																																						raise e
																																															# If possible, parse the file as a list of json objects and exit the loop
																																															if isinstance(__snake_case			,__snake_case      ):  # list is the only sequence type supported in JSON
																																																						try:
																																																													a__             =     set().union(*[row.keys() for row in dataset]      )
																																																													a__             =     {col: [row.get(__snake_case      ) for row in dataset] for col in keys}
																																																													a__             =     pa.Table.from_pydict(__snake_case      )
																																																						except (pa.ArrowInvalid, AttributeError) as e:
																																																													logger.error(F'Failed to read file \'{file}\' with error {type(__snake_case      )}: {e}'      )
																																																													raise ValueError(F'Not able to read records in the JSON file at {file}.'      ) from None
																																																						yield file_idx, self._cast_table(__snake_case      )
																																																						break
																																															else:
																																																						logger.error(F'Failed to read file \'{file}\' with error {type(__snake_case      )}: {e}'      )
																																																						raise ValueError(
																																																						    F'Not able to read records in the JSON file at {file}. '
																																																						    F'You should probably indicate the field of the JSON file containing your records. '
																																																						    F'This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. '
																																																						    F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																								yield (file_idx, batch_idx), self._cast_table(__snake_case      )
																																								batch_idx += 1








import argparse


snake_case							:       int     			=      '''docs/source/_static/js/custom.js'''






def 							__lowercase   ( __lowerCAmelCase							:				Optional[Any]   ):
							with open(__lowerCAmelCase					,    encoding='utf-8'					,    newline='\n'   ) as f:
														a__             =     f.readlines()
							a__             =     0

							# First let's put the right version
							while not lines[index].startswith('const stableVersion ='   ):
														index += 1
							a__             =     F'const stableVersion = "v{version}"\n'

							# Then update the dictionary
							while not lines[index].startswith('const versionMapping = {'   ):
														index += 1

							# We go until the end
							while not lines[index].startswith('}'   ):
														index += 1
							# We add the new version at the end
							lines[index - 1] += F'    "v{version}": "v{version}",\n'

							with open(__lowerCAmelCase					,    'w'					,    encoding='utf-8'					,    newline='\n'   ) as f:
														f.writelines(__lowerCAmelCase   )


if __name__ == "__main__":
						snake_case							:       Optional[Any]     			=      argparse.ArgumentParser()
						parser.add_argument('''--version''', help='''Release version.''')
						snake_case							:       Optional[int]     			=      parser.parse_args()
						update_custom_js(args.version)




