





import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_a 						=      datasets.utils.logging.get_logger(__name__)






@dataclass
class 					__A    (    datasets.BuilderConfig ):

			'''simple docstring'''







			lowerCAmelCase_							     =						10000
			lowerCAmelCase_							     =						None
			lowerCAmelCase_							     =						None






class 					__A    (    datasets.ArrowBasedBuilder ):

			'''simple docstring'''







			lowerCAmelCase_							     =						ParquetConfig





			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				return datasets.DatasetInfo(features=self.config.features							)





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				if not self.config.data_files:
					raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'							)
				lowerCamelCase__								=		dl_manager.download_and_extract(self.config.data_files							)
				if isinstance(__lowerCAmelCase       ,  (str, list, tuple)							):
					lowerCamelCase__								=		data_files
					if isinstance(__lowerCAmelCase       ,  __lowerCAmelCase							):
						lowerCamelCase__								=		[files]
					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
					lowerCamelCase__								=		[dl_manager.iter_files(__lowerCAmelCase							) for file in files]
					return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,  gen_kwargs={'''files''': files}							)]
				lowerCamelCase__								=		[]
				for split_name, files in data_files.items():
					if isinstance(__lowerCAmelCase       ,  __lowerCAmelCase							):
						lowerCamelCase__								=		[files]
					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
					lowerCamelCase__								=		[dl_manager.iter_files(__lowerCAmelCase							) for file in files]
					# Infer features is they are stoed in the arrow schema
					if self.info.features is None:
						for file in itertools.chain.from_iterable(__lowerCAmelCase							):
							with open(__lowerCAmelCase       ,  '''rb'''							) as f:
								lowerCamelCase__								=		datasets.Features.from_arrow_schema(pq.read_schema(__lowerCAmelCase							)							)
							break
					splits.append(datasets.SplitGenerator(name=__lowerCAmelCase       ,  gen_kwargs={'''files''': files}							)							)
				return splits





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				if self.info.features is not None:
					# more expensive cast to support nested features with keys in a different order
					# allows str <-> int/float or str to Audio for example
					lowerCamelCase__								=		table_cast(__lowerCAmelCase       ,  self.info.features.arrow_schema							)
				return pa_table





			def 		__lowerCamelCase				(    self       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		self.info.features.arrow_schema if self.info.features is not None else None
				if self.info.features is not None and self.config.columns is not None:
					if sorted(field.name for field in schema							) != sorted(self.config.columns							):
						raise ValueError(
						    F'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''							)
				for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCAmelCase							)							):
					with open(__lowerCAmelCase       ,  '''rb'''							) as f:
						lowerCamelCase__								=		pq.ParquetFile(__lowerCAmelCase							)
						try:
							for batch_idx, record_batch in enumerate(
							    parquet_file.iter_batches(batch_size=self.config.batch_size       ,  columns=self.config.columns							)							):
								lowerCamelCase__								=		pa.Table.from_batches([record_batch]							)
								# Uncomment for debugging (will print the Arrow table size and elements)
								# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
								# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
								yield F'{file_idx}_{batch_idx}', self._cast_table(__lowerCAmelCase							)
						except ValueError as e:
							logger.error(F'Failed to read file \'{file}\' with error {type(__lowerCAmelCase							)}: {e}'							)
							raise










from typing import List, Optional, Union

from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType






class 					__A    (    lowerCAmelCase ):

			'''simple docstring'''







			lowerCAmelCase_							     =						["""image_processor""", """tokenizer"""]
			lowerCAmelCase_							     =						"""BlipImageProcessor"""
			lowerCAmelCase_							     =						("""BertTokenizer""", """BertTokenizerFast""")





			def __init__(    self       ,  __lowerCAmelCase       ,  __lowerCAmelCase							):



				'''simple docstring'''







				lowerCamelCase__								=		False
				super().__init__(__lowerCAmelCase       ,  __lowerCAmelCase							)
				lowerCamelCase__								=		self.image_processor





			def __call__(    self       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = True       ,  __lowerCAmelCase = False       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = 0       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = None       ,  __lowerCAmelCase = False       ,  __lowerCAmelCase = False       ,  __lowerCAmelCase = False       ,  __lowerCAmelCase = False       ,  __lowerCAmelCase = False       ,  __lowerCAmelCase = True       ,  __lowerCAmelCase = None       ,  **__lowerCAmelCase       ,  ):



				'''simple docstring'''







				if images is None and text is None:
					raise ValueError('''You have to specify either images or text.'''							)

				# Get only text
				if images is None:
					lowerCamelCase__								=		self.tokenizer
					lowerCamelCase__								=		self.tokenizer(
					    text=__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase       ,  padding=__lowerCAmelCase       ,  truncation=__lowerCAmelCase       ,  max_length=__lowerCAmelCase       ,  stride=__lowerCAmelCase       ,  pad_to_multiple_of=__lowerCAmelCase       ,  return_attention_mask=__lowerCAmelCase       ,  return_overflowing_tokens=__lowerCAmelCase       ,  return_special_tokens_mask=__lowerCAmelCase       ,  return_offsets_mapping=__lowerCAmelCase       ,  return_token_type_ids=__lowerCAmelCase       ,  return_length=__lowerCAmelCase       ,  verbose=__lowerCAmelCase       ,  return_tensors=__lowerCAmelCase       ,  **__lowerCAmelCase       ,  )
					return text_encoding

				# add pixel_values
				lowerCamelCase__								=		self.image_processor(__lowerCAmelCase       ,  return_tensors=__lowerCAmelCase							)

				if text is not None:
					lowerCamelCase__								=		self.tokenizer(
					    text=__lowerCAmelCase       ,  add_special_tokens=__lowerCAmelCase       ,  padding=__lowerCAmelCase       ,  truncation=__lowerCAmelCase       ,  max_length=__lowerCAmelCase       ,  stride=__lowerCAmelCase       ,  pad_to_multiple_of=__lowerCAmelCase       ,  return_attention_mask=__lowerCAmelCase       ,  return_overflowing_tokens=__lowerCAmelCase       ,  return_special_tokens_mask=__lowerCAmelCase       ,  return_offsets_mapping=__lowerCAmelCase       ,  return_token_type_ids=__lowerCAmelCase       ,  return_length=__lowerCAmelCase       ,  verbose=__lowerCAmelCase       ,  return_tensors=__lowerCAmelCase       ,  **__lowerCAmelCase       ,  )
				else:
					lowerCamelCase__								=		None

				if text_encoding is not None:
					encoding_image_processor.update(__lowerCAmelCase							)

				return encoding_image_processor





			def 		__lowerCamelCase				(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.batch_decode(*__lowerCAmelCase       ,  **__lowerCAmelCase							)





			def 		__lowerCamelCase				(    self       ,  *__lowerCAmelCase       ,  **__lowerCAmelCase							):



				'''simple docstring'''







				return self.tokenizer.decode(*__lowerCAmelCase       ,  **__lowerCAmelCase							)





			@property
			def 		__lowerCamelCase				(    self							):



				'''simple docstring'''







				lowerCamelCase__								=		self.tokenizer.model_input_names
				lowerCamelCase__								=		self.image_processor.model_input_names
				return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)



