





'''simple docstring'''




import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast


@dataclass
class  SCREAMING_SNAKE_CASE    (datasets.BuilderConfig							):
     lowerCAmelCase					=       None




class  SCREAMING_SNAKE_CASE    (datasets.ArrowBasedBuilder							):
     lowerCAmelCase					=       PandasConfig


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          return datasets.DatasetInfo(features=self.config.features)


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          if not self.config.data_files:
               raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}')
          __A						: Dict      =       dl_manager.download_and_extract(self.config.data_files)
          if isinstance(_UpperCAmelCase						,	(str, list, tuple)):
               __A						: Union[str, Any]      =       data_files
               if isinstance(_UpperCAmelCase						,	_UpperCAmelCase):
                    __A						: Optional[int]      =       [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __A						: Optional[Any]      =       [dl_manager.iter_files(_UpperCAmelCase) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,	gen_kwargs={'files': files})]
          __A						: Tuple      =       []
          for split_name, files in data_files.items():
               if isinstance(_UpperCAmelCase						,	_UpperCAmelCase):
                    __A						: Dict      =       [files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               __A						: Optional[Any]      =       [dl_manager.iter_files(_UpperCAmelCase) for file in files]
               splits.append(datasets.SplitGenerator(name=_UpperCAmelCase						,	gen_kwargs={'files': files}))
          return splits


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          if self.config.features is not None:
               # more expensive cast to support nested features with keys in a different order
               # allows str <-> int/float or str to Audio for example
               __A						: List[str]      =       table_cast(_UpperCAmelCase						,	self.config.features.arrow_schema)
          return pa_table


     def 							SCREAMING_SNAKE_CASE  (						self						,	_UpperCAmelCase):

          '''simple docstring'''






          for i, file in enumerate(itertools.chain.from_iterable(_UpperCAmelCase)):
               with open(_UpperCAmelCase						,	'rb') as f:
                    __A						: Optional[int]      =       pa.Table.from_pandas(pd.read_pickle(_UpperCAmelCase))
                    yield i, self._cast_table(_UpperCAmelCase)






'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__    :      List[Any]						=    logging.get_logger(__name__)

lowercase__    :      str						=    {}


class  SCREAMING_SNAKE_CASE    (a__							):
     lowerCAmelCase					=       '''llama'''
     lowerCAmelCase					=       ['''past_key_values''']


     def __init__(						self						,	_UpperCAmelCase=3_2000						,	_UpperCAmelCase=4096						,	_UpperCAmelCase=1_1008						,	_UpperCAmelCase=32						,	_UpperCAmelCase=32						,	_UpperCAmelCase=None						,	_UpperCAmelCase="silu"						,	_UpperCAmelCase=2048						,	_UpperCAmelCase=0.02						,	_UpperCAmelCase=1e-6						,	_UpperCAmelCase=True						,	_UpperCAmelCase=0						,	_UpperCAmelCase=1						,	_UpperCAmelCase=2						,	_UpperCAmelCase=1						,	_UpperCAmelCase=False						,	_UpperCAmelCase=None						,	**_UpperCAmelCase						,	):

          '''simple docstring'''






          __A						: Union[str, Any]      =       vocab_size
          __A						: Union[str, Any]      =       max_position_embeddings
          __A						: Any      =       hidden_size
          __A						: Optional[Any]      =       intermediate_size
          __A						: str      =       num_hidden_layers
          __A						: Optional[Any]      =       num_attention_heads

          # for backward compatibility
          if num_key_value_heads is None:
               __A						: List[Any]      =       num_attention_heads

          __A						: int      =       num_key_value_heads
          __A						: List[Any]      =       hidden_act
          __A						: Union[str, Any]      =       initializer_range
          __A						: List[Any]      =       rms_norm_eps
          __A						: Any      =       pretraining_tp
          __A						: Optional[Any]      =       use_cache
          __A						: Dict      =       rope_scaling
          self._rope_scaling_validation()

          super().__init__(
              pad_token_id=_UpperCAmelCase						,	bos_token_id=_UpperCAmelCase						,	eos_token_id=_UpperCAmelCase						,	tie_word_embeddings=_UpperCAmelCase						,	**_UpperCAmelCase						,	)


     def 							SCREAMING_SNAKE_CASE  (						self):

          '''simple docstring'''






          if self.rope_scaling is None:
               return

          if not isinstance(self.rope_scaling						,	_UpperCAmelCase) or len(self.rope_scaling) != 2:
               raise ValueError(
                   '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                   F'got {self.rope_scaling}')
          __A						: Optional[Any]      =       self.rope_scaling.get('type'						,	_UpperCAmelCase)
          __A						: Tuple      =       self.rope_scaling.get('factor'						,	_UpperCAmelCase)
          if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
               raise ValueError(
                   F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}')
          if rope_scaling_factor is None or not isinstance(_UpperCAmelCase						,	_UpperCAmelCase) or rope_scaling_factor <= 1.0:
               raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}')