



'''simple docstring'''






import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


A_							: List[str]						 =	datasets.utils.logging.get_logger(__name__)



@dataclass
class 							lowercase       (  datasets.BuilderConfig				):


 """simple docstring"""


 UpperCAmelCase							=  None
 UpperCAmelCase							=  "utf-8"
 UpperCAmelCase							=  None
 UpperCAmelCase							=  None
 UpperCAmelCase							=  True  # deprecated
 UpperCAmelCase							=  None  # deprecated
 UpperCAmelCase							=  10 << 20  # 10MB
 UpperCAmelCase							=  None







class 							lowercase       (  datasets.ArrowBasedBuilder				):


 """simple docstring"""


 UpperCAmelCase							=  JsonConfig







 def _snake_case (							self							)		->   Optional[Any]:
        if self.config.block_size is not None:
               logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""							)
               _UpperCAmelCase  :						str			     = self.config.block_size
        if self.config.use_threads is not True:
               logger.warning(
                   """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""							)
        if self.config.newlines_in_values is not None:
               raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""							)
        return datasets.DatasetInfo(features=self.config.features							)







 def _snake_case (							self  ,a_							)		->   List[Any]:
        if not self.config.data_files:
               raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''							)
        _UpperCAmelCase  :						Optional[Any]			     = dl_manager.download_and_extract(self.config.data_files							)
        if isinstance(a_  ,(str, list, tuple)							):
               _UpperCAmelCase  :						List[Any]			     = data_files
               if isinstance(a_  ,a_							):
                      _UpperCAmelCase  :						Any			     = [files]
               _UpperCAmelCase  :						str			     = [dl_manager.iter_files(a_							) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,gen_kwargs={"""files""": files}							)]
        _UpperCAmelCase  :						Optional[Any]			     = []
        for split_name, files in data_files.items():
               if isinstance(a_  ,a_							):
                      _UpperCAmelCase  :						Union[str, Any]			     = [files]
               _UpperCAmelCase  :						List[str]			     = [dl_manager.iter_files(a_							) for file in files]
               splits.append(datasets.SplitGenerator(name=a_  ,gen_kwargs={"""files""": files}							)							)
        return splits







 def _snake_case (							self  ,a_							)		->   pa.Table:
        if self.config.features is not None:
               # adding missing columns
               for column_name in set(self.config.features							) - set(pa_table.column_names							):
                      _UpperCAmelCase  :						Tuple			     = self.config.features.arrow_schema.field(a_							).type
                      _UpperCAmelCase  :						Dict			     = pa_table.append_column(a_  ,pa.array([None] * len(a_							)  ,type=a_							)							)
               # more expensive cast to support nested structures with keys in a different order
               # allows str <-> int/float or str to Audio for example
               _UpperCAmelCase  :						List[str]			     = table_cast(a_  ,self.config.features.arrow_schema							)
        return pa_table







 def _snake_case (							self  ,a_							)		->   Union[str, Any]:
        for file_idx, file in enumerate(itertools.chain.from_iterable(a_							)							):
               # If the file is one json object and if we need to look at the list of items in one specific field
               if self.config.field is not None:
                      with open(a_  ,encoding=self.config.encoding  ,errors=self.config.encoding_errors							) as f:
                             _UpperCAmelCase  :						Any			     = json.load(a_							)

                      # We keep only the field we are interested in
                      _UpperCAmelCase  :						Any			     = dataset[self.config.field]

                      # We accept two format: a list of dicts or a dict of lists
                      if isinstance(a_  ,(list, tuple)							):
                             _UpperCAmelCase  :						Union[str, Any]			     = set().union(*[row.keys() for row in dataset]							)
                             _UpperCAmelCase  :						Dict			     = {col: [row.get(a_							) for row in dataset] for col in keys}
                      else:
                             _UpperCAmelCase  :						List[Any]			     = dataset
                      _UpperCAmelCase  :						Tuple			     = pa.Table.from_pydict(a_							)
                      yield file_idx, self._cast_table(a_							)

               # If the file has one json object per line
               else:
                      with open(a_  ,"""rb"""							) as f:
                             _UpperCAmelCase  :						Optional[Any]			     = 0
                             # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                             # Set a default minimum value of 16kB if the chunk size is really small
                             _UpperCAmelCase  :						List[Any]			     = max(self.config.chunksize // 32  ,16 << 10							)
                             _UpperCAmelCase  :						Dict			     = (
                                 self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                             )
                             while True:
                                    _UpperCAmelCase  :						int			     = f.read(self.config.chunksize							)
                                    if not batch:
                                           break
                                    # Finish current line
                                    try:
                                           batch += f.readline()
                                    except (AttributeError, io.UnsupportedOperation):
                                           batch += readline(a_							)
                                    # PyArrow only accepts utf-8 encoded bytes
                                    if self.config.encoding != "utf-8":
                                           _UpperCAmelCase  :						int			     = batch.decode(self.config.encoding  ,errors=a_							).encode("""utf-8"""							)
                                    try:
                                           while True:
                                                  try:
                                                         _UpperCAmelCase  :						Optional[int]			     = paj.read_json(
                                                             io.BytesIO(a_							)  ,read_options=paj.ReadOptions(block_size=a_							)							)
                                                         break
                                                  except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                         if (
                                                             isinstance(a_  ,pa.ArrowInvalid							)
                                                             and "straddling" not in str(a_							)
                                                             or block_size > len(a_							)
                                                         ):
                                                                raise
                                                         else:
                                                                # Increase the block size in case it was too small.
                                                                # The block size will be reset for the next file.
                                                                logger.debug(
                                                                    f'''Batch of {len(a_							)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''							)
                                                                block_size *= 2
                                    except pa.ArrowInvalid as e:
                                           try:
                                                  with open(
                                                      a_  ,encoding=self.config.encoding  ,errors=self.config.encoding_errors							) as f:
                                                         _UpperCAmelCase  :						List[Any]			     = json.load(a_							)
                                           except json.JSONDecodeError:
                                                  logger.error(f'''Failed to read file \'{file}\' with error {type(a_							)}: {e}'''							)
                                                  raise e
                                           # If possible, parse the file as a list of json objects and exit the loop
                                           if isinstance(a_  ,a_							):  # list is the only sequence type supported in JSON
                                                  try:
                                                         _UpperCAmelCase  :						Any			     = set().union(*[row.keys() for row in dataset]							)
                                                         _UpperCAmelCase  :						int			     = {col: [row.get(a_							) for row in dataset] for col in keys}
                                                         _UpperCAmelCase  :						Any			     = pa.Table.from_pydict(a_							)
                                                  except (pa.ArrowInvalid, AttributeError) as e:
                                                         logger.error(f'''Failed to read file \'{file}\' with error {type(a_							)}: {e}'''							)
                                                         raise ValueError(f'''Not able to read records in the JSON file at {file}.'''							) from None
                                                  yield file_idx, self._cast_table(a_							)
                                                  break
                                           else:
                                                  logger.error(f'''Failed to read file \'{file}\' with error {type(a_							)}: {e}'''							)
                                                  raise ValueError(
                                                      f'''Not able to read records in the JSON file at {file}. '''
                                                      f'''You should probably indicate the field of the JSON file containing your records. '''
                                                      f'''This JSON file contain the following fields: {str(list(dataset.keys()							)							)}. '''
                                                      f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''							) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                    yield (file_idx, batch_idx), self._cast_table(a_							)
                                    batch_idx += 1








'''simple docstring'''






from collections import OrderedDict
from typing import TYPE_CHECKING, Any, List, Mapping, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import TensorType, logging


if TYPE_CHECKING:
   from ...onnx.config import PatchingSpec
   from ...tokenization_utils_base import PreTrainedTokenizerBase


A_							: Any						 =	logging.get_logger(__name__)

A_							: Optional[int]						 =	{
    """allenai/longformer-base-4096""": """https://huggingface.co/allenai/longformer-base-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096""": """https://huggingface.co/allenai/longformer-large-4096/resolve/main/config.json""",
    """allenai/longformer-large-4096-finetuned-triviaqa""": (
        """https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/config.json"""
    ),
    """allenai/longformer-base-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
    """allenai/longformer-large-4096-extra.pos.embd.only""": (
        """https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/config.json"""
    ),
}



class 							lowercase       (  _lowerCamelCase				):


 """simple docstring"""


 UpperCAmelCase							=  """longformer"""







 def __init__(							self  ,a_ = 512  ,a_ = 2  ,a_ = 1  ,a_ = 0  ,a_ = 2  ,a_ = 30_522  ,a_ = 768  ,a_ = 12  ,a_ = 12  ,a_ = 3_072  ,a_ = "gelu"  ,a_ = 0.1  ,a_ = 0.1  ,a_ = 512  ,a_ = 2  ,a_ = 0.02  ,a_ = 1E-1_2  ,a_ = False  ,**a_  ,)		->   List[Any]:
        super().__init__(pad_token_id=a_  ,**a_							)

        _UpperCAmelCase  :						List[Any]			     = attention_window
        _UpperCAmelCase  :						Any			     = sep_token_id
        _UpperCAmelCase  :						Dict			     = bos_token_id
        _UpperCAmelCase  :						Tuple			     = eos_token_id
        _UpperCAmelCase  :						Tuple			     = vocab_size
        _UpperCAmelCase  :						Optional[Any]			     = hidden_size
        _UpperCAmelCase  :						Optional[int]			     = num_hidden_layers
        _UpperCAmelCase  :						Union[str, Any]			     = num_attention_heads
        _UpperCAmelCase  :						Optional[int]			     = hidden_act
        _UpperCAmelCase  :						str			     = intermediate_size
        _UpperCAmelCase  :						List[Any]			     = hidden_dropout_prob
        _UpperCAmelCase  :						Union[str, Any]			     = attention_probs_dropout_prob
        _UpperCAmelCase  :						List[str]			     = max_position_embeddings
        _UpperCAmelCase  :						Optional[int]			     = type_vocab_size
        _UpperCAmelCase  :						Any			     = initializer_range
        _UpperCAmelCase  :						Optional[int]			     = layer_norm_eps
        _UpperCAmelCase  :						Union[str, Any]			     = onnx_export







class 							lowercase       (  _lowerCamelCase				):


 """simple docstring"""


 def __init__(							self  ,a_  ,a_ = "default"  ,a_ = None							)		->   int:
        super().__init__(a_  ,a_  ,a_							)
        _UpperCAmelCase  :						Tuple			     = True







 @property
 def _snake_case (							self							)		->   Mapping[str, Mapping[int, str]]:
        if self.task == "multiple-choice":
               _UpperCAmelCase  :						Optional[int]			     = {0: """batch""", 1: """choice""", 2: """sequence"""}
        else:
               _UpperCAmelCase  :						Tuple			     = {0: """batch""", 1: """sequence"""}
        return OrderedDict(
            [
                ("""input_ids""", dynamic_axis),
                ("""attention_mask""", dynamic_axis),
                ("""global_attention_mask""", dynamic_axis),
            ]							)







 @property
 def _snake_case (							self							)		->   Mapping[str, Mapping[int, str]]:
        _UpperCAmelCase  :						str			     = super().outputs
        if self.task == "default":
               _UpperCAmelCase  :						int			     = {0: """batch"""}
        return outputs







 @property
 def _snake_case (							self							)		->   float:
        return 1E-4







 @property
 def _snake_case (							self							)		->   int:
        # needs to be >= 14 to support tril operator
        return max(super().default_onnx_opset  ,14							)







 def _snake_case (							self  ,a_  ,a_ = -1  ,a_ = -1  ,a_ = False  ,a_ = None  ,)		->   Mapping[str, Any]:
        _UpperCAmelCase  :						List[str]			     = super().generate_dummy_inputs(
            preprocessor=a_  ,batch_size=a_  ,seq_length=a_  ,is_pair=a_  ,framework=a_							)
        import torch

        # for some reason, replacing this code by inputs["global_attention_mask"] = torch.randint(2, inputs["input_ids"].shape, dtype=torch.int64)
        # makes the export fail randomly
        _UpperCAmelCase  :						int			     = torch.zeros_like(inputs["""input_ids"""]							)
        # make every second token global
        _UpperCAmelCase  :						List[str]			     = 1

        return inputs



