
import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCamelCase__          =		datasets.utils.logging.get_logger(__name__)







@dataclass
class      A__							(     datasets.BuilderConfig	):
				lowercase			    =  None
				lowercase			    =  "utf-8"
				lowercase			    =  None
				lowercase			    =  None
				lowercase			    =  True  # deprecated
				lowercase			    =  None  # deprecated
				lowercase			    =  10 << 20  # 10MB
				lowercase			    =  None







class      A__							(     datasets.ArrowBasedBuilder	):
				lowercase			    =  JsonConfig






				def 							_lowerCamelCase		(						self					:      Dict   ):






							'''simple docstring'''







							if self.config.block_size is not None:
										logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'   )
										lowerCAmelCase__      :      Tuple         =		self.config.block_size
							if self.config.use_threads is not True:
										logger.warning(
										    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'   )
							if self.config.newlines_in_values is not None:
										raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'   )
							return datasets.DatasetInfo(features=self.config.features   )






				def 							_lowerCamelCase		(						self					:      str    ,     a					:      List[Any]   ):






							'''simple docstring'''







							if not self.config.data_files:
										raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''   )
							lowerCAmelCase__      :      Tuple         =		dl_manager.download_and_extract(self.config.data_files   )
							if isinstance(a    ,     (str, list, tuple)   ):
										lowerCAmelCase__      :      Optional[int]         =		data_files
										if isinstance(a    ,     a   ):
													lowerCAmelCase__      :      Optional[Any]         =		[files]
										lowerCAmelCase__      :      Any         =		[dl_manager.iter_files(a   ) for file in files]
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,     gen_kwargs={'files': files}   )]
							lowerCAmelCase__      :      List[str]         =		[]
							for split_name, files in data_files.items():
										if isinstance(a    ,     a   ):
													lowerCAmelCase__      :      List[str]         =		[files]
										lowerCAmelCase__      :      Union[str, Any]         =		[dl_manager.iter_files(a   ) for file in files]
										splits.append(datasets.SplitGenerator(name=a    ,     gen_kwargs={'files': files}   )   )
							return splits






				def 							_lowerCamelCase		(						self					:      Any    ,     a					:      pa.Table   ):






							'''simple docstring'''







							if self.config.features is not None:
										# adding missing columns
										for column_name in set(self.config.features   ) - set(pa_table.column_names   ):
													lowerCAmelCase__      :      Tuple         =		self.config.features.arrow_schema.field(a   ).type
													lowerCAmelCase__      :      Tuple         =		pa_table.append_column(a    ,     pa.array([None] * len(a   )    ,     type=a   )   )
										# more expensive cast to support nested structures with keys in a different order
										# allows str <-> int/float or str to Audio for example
										lowerCAmelCase__      :      List[str]         =		table_cast(a    ,     self.config.features.arrow_schema   )
							return pa_table






				def 							_lowerCamelCase		(						self					:      Any    ,     a					:      List[str]   ):






							'''simple docstring'''







							for file_idx, file in enumerate(itertools.chain.from_iterable(a   )   ):
										# If the file is one json object and if we need to look at the list of items in one specific field
										if self.config.field is not None:
													with open(a    ,     encoding=self.config.encoding    ,     errors=self.config.encoding_errors   ) as f:
																lowerCAmelCase__      :      str         =		json.load(a   )

													# We keep only the field we are interested in
													lowerCAmelCase__      :      List[Any]         =		dataset[self.config.field]

													# We accept two format: a list of dicts or a dict of lists
													if isinstance(a    ,     (list, tuple)   ):
																lowerCAmelCase__      :      str         =		set().union(*[row.keys() for row in dataset]   )
																lowerCAmelCase__      :      List[Any]         =		{col: [row.get(a   ) for row in dataset] for col in keys}
													else:
																lowerCAmelCase__      :      Any         =		dataset
													lowerCAmelCase__      :      Dict         =		pa.Table.from_pydict(a   )
													yield file_idx, self._cast_table(a   )

										# If the file has one json object per line
										else:
													with open(a    ,     'rb'   ) as f:
																lowerCAmelCase__      :      Union[str, Any]         =		0
																# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																# Set a default minimum value of 16kB if the chunk size is really small
																lowerCAmelCase__      :      Any         =		max(self.config.chunksize // 32    ,     16 << 10   )
																lowerCAmelCase__      :      str         =		(
																    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																)
																while True:
																			lowerCAmelCase__      :      int         =		f.read(self.config.chunksize   )
																			if not batch:
																						break
																			# Finish current line
																			try:
																						batch += f.readline()
																			except (AttributeError, io.UnsupportedOperation):
																						batch += readline(a   )
																			# PyArrow only accepts utf-8 encoded bytes
																			if self.config.encoding != "utf-8":
																						lowerCAmelCase__      :      Tuple         =		batch.decode(self.config.encoding    ,     errors=a   ).encode('utf-8'   )
																			try:
																						while True:
																									try:
																												lowerCAmelCase__      :      Optional[Any]         =		paj.read_json(
																												    io.BytesIO(a   )    ,     read_options=paj.ReadOptions(block_size=a   )   )
																												break
																									except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																												if (
																												    isinstance(a    ,     pa.ArrowInvalid   )
																												    and "straddling" not in str(a   )
																												    or block_size > len(a   )
																												):
																															raise
																												else:
																															# Increase the block size in case it was too small.
																															# The block size will be reset for the next file.
																															logger.debug(
																															    f'''Batch of {len(a   )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''   )
																															block_size *= 2
																			except pa.ArrowInvalid as e:
																						try:
																									with open(
																									    a    ,     encoding=self.config.encoding    ,     errors=self.config.encoding_errors   ) as f:
																												lowerCAmelCase__      :      Dict         =		json.load(a   )
																						except json.JSONDecodeError:
																									logger.error(f'''Failed to read file \'{file}\' with error {type(a   )}: {e}'''   )
																									raise e
																						# If possible, parse the file as a list of json objects and exit the loop
																						if isinstance(a    ,     a   ):  # list is the only sequence type supported in JSON
																									try:
																												lowerCAmelCase__      :      Dict         =		set().union(*[row.keys() for row in dataset]   )
																												lowerCAmelCase__      :      Any         =		{col: [row.get(a   ) for row in dataset] for col in keys}
																												lowerCAmelCase__      :      List[Any]         =		pa.Table.from_pydict(a   )
																									except (pa.ArrowInvalid, AttributeError) as e:
																												logger.error(f'''Failed to read file \'{file}\' with error {type(a   )}: {e}'''   )
																												raise ValueError(f'''Not able to read records in the JSON file at {file}.'''   ) from None
																									yield file_idx, self._cast_table(a   )
																									break
																						else:
																									logger.error(f'''Failed to read file \'{file}\' with error {type(a   )}: {e}'''   )
																									raise ValueError(
																									    f'''Not able to read records in the JSON file at {file}. '''
																									    f'''You should probably indicate the field of the JSON file containing your records. '''
																									    f'''This JSON file contain the following fields: {str(list(dataset.keys()   )   )}. '''
																									    f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''   ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																			yield (file_idx, batch_idx), self._cast_table(a   )
																			batch_idx += 1

from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowerCamelCase__          =		logging.get_logger(__name__)

lowerCamelCase__          =		{
    """EleutherAI/gpt-j-6B""": """https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json""",
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}







class      A__							(     __magic_name__	):
				lowercase			    =  'gptj'
				lowercase			    =  {
				    'max_position_embeddings': 'n_positions',
				    'hidden_size': 'n_embd',
				    'num_attention_heads': 'n_head',
				    'num_hidden_layers': 'n_layer',
				}






				def __init__(						self					:      Optional[Any]    ,     a					:      Dict=50_400    ,     a					:      Union[str, Any]=2_048    ,     a					:      List[str]=4_096    ,     a					:      Any=28    ,     a					:      Optional[Any]=16    ,     a					:      Optional[Any]=64    ,     a					:      int=None    ,     a					:      Any="gelu_new"    ,     a					:      Union[str, Any]=0.0    ,     a					:      List[Any]=0.0    ,     a					:      List[Any]=0.0    ,     a					:      Optional[Any]=1E-5    ,     a					:      Optional[int]=0.0_2    ,     a					:      int=True    ,     a					:      str=50_256    ,     a					:      str=50_256    ,     a					:      Any=False    ,     **a					:      Dict    ,     ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[Any]         =		vocab_size
							lowerCAmelCase__      :      List[Any]         =		n_positions
							lowerCAmelCase__      :      str         =		n_embd
							lowerCAmelCase__      :      str         =		n_layer
							lowerCAmelCase__      :      str         =		n_head
							lowerCAmelCase__      :      Dict         =		n_inner
							lowerCAmelCase__      :      Union[str, Any]         =		rotary_dim
							lowerCAmelCase__      :      Optional[int]         =		activation_function
							lowerCAmelCase__      :      Any         =		resid_pdrop
							lowerCAmelCase__      :      int         =		embd_pdrop
							lowerCAmelCase__      :      int         =		attn_pdrop
							lowerCAmelCase__      :      List[Any]         =		layer_norm_epsilon
							lowerCAmelCase__      :      str         =		initializer_range
							lowerCAmelCase__      :      Dict         =		use_cache

							lowerCAmelCase__      :      str         =		bos_token_id
							lowerCAmelCase__      :      int         =		eos_token_id

							super().__init__(
							    bos_token_id=a    ,     eos_token_id=a    ,     tie_word_embeddings=a    ,     **a   )







class      A__							(     __magic_name__	):






				def __init__(						self					:      str    ,     a					:      PretrainedConfig    ,     a					:      str = "default"    ,     a					:      List[PatchingSpec] = None    ,     a					:      bool = False    ,     ):






							'''simple docstring'''







							super().__init__(a    ,     task=a    ,     patching_specs=a    ,     use_past=a   )
							if not getattr(self._config    ,     'pad_token_id'    ,     a   ):
										# TODO: how to do that better?
										lowerCAmelCase__      :      int         =		0






				@property
				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Dict         =		OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}   )
							if self.use_past:
										self.fill_with_past_key_values_(a    ,     direction='inputs'   )
										lowerCAmelCase__      :      Optional[Any]         =		{0: 'batch', 1: 'past_sequence + sequence'}
							else:
										lowerCAmelCase__      :      Tuple         =		{0: 'batch', 1: 'sequence'}

							return common_inputs






				@property
				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							return self._config.n_layer






				@property
				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							return self._config.n_head






				def 							_lowerCamelCase		(						self					:      Tuple    ,     a					:      PreTrainedTokenizer    ,     a					:      int = -1    ,     a					:      int = -1    ,     a					:      bool = False    ,     a					:      Optional[TensorType] = None    ,     ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		super(a    ,     self   ).generate_dummy_inputs(
							    a    ,     batch_size=a    ,     seq_length=a    ,     is_pair=a    ,     framework=a   )

							# We need to order the input in the way they appears in the forward()
							lowerCAmelCase__      :      Optional[int]         =		OrderedDict({'input_ids': common_inputs['input_ids']}   )

							# Need to add the past_keys
							if self.use_past:
										if not is_torch_available():
													raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'   )
										else:
													import torch

													lowerCAmelCase__      ,				lowerCAmelCase__      :      int         =		common_inputs['input_ids'].shape
													# Not using the same length for past_key_values
													lowerCAmelCase__      :      Optional[int]         =		seqlen + 2
													lowerCAmelCase__      :      Tuple         =		(
													    batch,
													    self.num_attention_heads,
													    past_key_values_length,
													    self._config.hidden_size // self.num_attention_heads,
													)
													lowerCAmelCase__      :      Tuple         =		[
													    (torch.zeros(a   ), torch.zeros(a   )) for _ in range(self.num_layers   )
													]

							lowerCAmelCase__      :      Any         =		common_inputs['attention_mask']
							if self.use_past:
										lowerCAmelCase__      :      List[str]         =		ordered_inputs['attention_mask'].dtype
										lowerCAmelCase__      :      Optional[Any]         =		torch.cat(
										    [ordered_inputs['attention_mask'], torch.ones(a    ,     a    ,     dtype=a   )]    ,     dim=1   )

							return ordered_inputs






				@property
				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							return 13