


"""simple docstring"""







import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__lowercase    					=      datasets.utils.logging.get_logger(__name__)






@dataclass
class 						_A  (  datasets.BuilderConfig  ):







    """simple docstring"""




    UpperCAmelCase       :							int	=       1_0_0_0_0
    UpperCAmelCase       :							Optional[List[str]]	=       None
    UpperCAmelCase       :							Optional[datasets.Features]	=       None






class 						_A  (  datasets.ArrowBasedBuilder  ):







    """simple docstring"""




    UpperCAmelCase       :							str	=       ParquetConfig






    def 	__snake_case							(					self				:    Tuple):
      return datasets.DatasetInfo(features=self.config.features)






    def 	__snake_case							(					self				:    List[Any]				, __UpperCAmelCase				:    str):
      if not self.config.data_files:
        raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}''')
      a						:		str      							=			dl_manager.download_and_extract(self.config.data_files)
      if isinstance(__UpperCAmelCase				, (str, list, tuple)):
        a						:		Dict      							=			data_files
        if isinstance(__UpperCAmelCase				, __UpperCAmelCase):
          a						:		str      							=			[files]
        # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
        a						:		List[Any]      							=			[dl_manager.iter_files(__UpperCAmelCase) for file in files]
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN				, gen_kwargs={"files": files})]
      a						:		Dict      							=			[]
      for split_name, files in data_files.items():
        if isinstance(__UpperCAmelCase				, __UpperCAmelCase):
          a						:		Optional[int]      							=			[files]
        # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
        a						:		Tuple      							=			[dl_manager.iter_files(__UpperCAmelCase) for file in files]
        # Infer features is they are stoed in the arrow schema
        if self.info.features is None:
          for file in itertools.chain.from_iterable(__UpperCAmelCase):
            with open(__UpperCAmelCase				, "rb") as f:
              a						:		Tuple      							=			datasets.Features.from_arrow_schema(pq.read_schema(__UpperCAmelCase))
            break
        splits.append(datasets.SplitGenerator(name=__UpperCAmelCase				, gen_kwargs={"files": files}))
      return splits






    def 	__snake_case							(					self				:    List[str]				, __UpperCAmelCase				:    pa.Table):
      if self.info.features is not None:
        # more expensive cast to support nested features with keys in a different order
        # allows str <-> int/float or str to Audio for example
        a						:		Optional[int]      							=			table_cast(__UpperCAmelCase				, self.info.features.arrow_schema)
      return pa_table






    def 	__snake_case							(					self				:    Union[str, Any]				, __UpperCAmelCase				:    int):
      a						:		Tuple      							=			self.info.features.arrow_schema if self.info.features is not None else None
      if self.info.features is not None and self.config.columns is not None:
        if sorted(field.name for field in schema) != sorted(self.config.columns):
          raise ValueError(
              f'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\'''')
      for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCAmelCase)):
        with open(__UpperCAmelCase				, "rb") as f:
          a						:		Tuple      							=			pq.ParquetFile(__UpperCAmelCase)
          try:
            for batch_idx, record_batch in enumerate(
                parquet_file.iter_batches(batch_size=self.config.batch_size				, columns=self.config.columns)):
              a						:		Optional[Any]      							=			pa.Table.from_batches([record_batch])
              # Uncomment for debugging (will print the Arrow table size and elements)
              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
              yield f'''{file_idx}_{batch_idx}''', self._cast_table(__UpperCAmelCase)
          except ValueError as e:
            logger.error(f'''Failed to read file \'{file}\' with error {type(__UpperCAmelCase)}: {e}''')
            raise








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


_A   =     {}

try:
			if not is_sentencepiece_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     ["NllbTokenizer"]

try:
			if not is_tokenizers_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_A   =     ["NllbTokenizerFast"]


if TYPE_CHECKING:
			try:
						if not is_sentencepiece_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb import NllbTokenizer

			try:
						if not is_tokenizers_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .tokenization_nllb_fast import NllbTokenizerFast

else:
			import sys

			_A   =     _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



