

import argparse
import json
import os

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ConvNextImageProcessor, ConvNextVaConfig, ConvNextVaForImageClassification
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
lowerCAmelCase							:						Optional[Any]     			=		logging.get_logger(__name__)







def     A_   (					_UpperCAmelCase					):
      SCREAMING_SNAKE_CASE_:				Tuple							= ConvNextVaConfig()

      if "atto" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				str							= [2, 2, 6, 2]
            SCREAMING_SNAKE_CASE_:				str							= [40, 80, 1_60, 3_20]
      if "femto" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				List[str]							= [2, 2, 6, 2]
            SCREAMING_SNAKE_CASE_:				Dict							= [48, 96, 1_92, 3_84]
      if "pico" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				Any							= [2, 2, 6, 2]
            SCREAMING_SNAKE_CASE_:				Dict							= [64, 1_28, 2_56, 5_12]
      if "nano" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				Tuple							= [2, 2, 8, 2]
            SCREAMING_SNAKE_CASE_:				str							= [80, 1_60, 3_20, 6_40]
      if "tiny" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				List[str]							= [3, 3, 9, 3]
            SCREAMING_SNAKE_CASE_:				Dict							= [96, 1_92, 3_84, 7_68]
      if "base" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				List[Any]							= [3, 3, 27, 3]
            SCREAMING_SNAKE_CASE_:				Tuple							= [1_28, 2_56, 5_12, 10_24]
      if "large" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				str							= [3, 3, 27, 3]
            SCREAMING_SNAKE_CASE_:				str							= [1_92, 3_84, 7_68, 15_36]
      if "huge" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				List[str]							= [3, 3, 27, 3]
            SCREAMING_SNAKE_CASE_:				Tuple							= [3_52, 7_04, 14_08, 28_16]

      SCREAMING_SNAKE_CASE_:				Optional[Any]							= 10_00
      SCREAMING_SNAKE_CASE_:				List[Any]							= "imagenet-1k-id2label.json"
      SCREAMING_SNAKE_CASE_:				str							= (1, 10_00)

      SCREAMING_SNAKE_CASE_:				Tuple							= "huggingface/label-files"
      SCREAMING_SNAKE_CASE_:				List[str]							= num_labels
      SCREAMING_SNAKE_CASE_:				Optional[Any]							= json.load(open(hf_hub_download(_UpperCAmelCase       ,			_UpperCAmelCase       ,			repo_type="dataset"					)       ,			"r"					)					)
      SCREAMING_SNAKE_CASE_:				Any							= {int(_UpperCAmelCase					): v for k, v in idalabel.items()}

      SCREAMING_SNAKE_CASE_:				List[Any]							= idalabel
      SCREAMING_SNAKE_CASE_:				Union[str, Any]							= {v: k for k, v in idalabel.items()}
      SCREAMING_SNAKE_CASE_:				Any							= hidden_sizes
      SCREAMING_SNAKE_CASE_:				Tuple							= depths

      return config, expected_shape







def     A_   (					_UpperCAmelCase					):
      if "downsample_layers.0.0" in name:
            SCREAMING_SNAKE_CASE_:				int							= name.replace("downsample_layers.0.0"       ,			"embeddings.patch_embeddings"					)
      if "downsample_layers.0.1" in name:
            SCREAMING_SNAKE_CASE_:				Optional[int]							= name.replace("downsample_layers.0.1"       ,			"embeddings.norm"					)  # we rename to layernorm later on
      if "downsample_layers.1.0" in name:
            SCREAMING_SNAKE_CASE_:				Optional[int]							= name.replace("downsample_layers.1.0"       ,			"stages.1.downsampling_layer.0"					)
      if "downsample_layers.1.1" in name:
            SCREAMING_SNAKE_CASE_:				Optional[Any]							= name.replace("downsample_layers.1.1"       ,			"stages.1.downsampling_layer.1"					)
      if "downsample_layers.2.0" in name:
            SCREAMING_SNAKE_CASE_:				Optional[int]							= name.replace("downsample_layers.2.0"       ,			"stages.2.downsampling_layer.0"					)
      if "downsample_layers.2.1" in name:
            SCREAMING_SNAKE_CASE_:				Optional[Any]							= name.replace("downsample_layers.2.1"       ,			"stages.2.downsampling_layer.1"					)
      if "downsample_layers.3.0" in name:
            SCREAMING_SNAKE_CASE_:				Dict							= name.replace("downsample_layers.3.0"       ,			"stages.3.downsampling_layer.0"					)
      if "downsample_layers.3.1" in name:
            SCREAMING_SNAKE_CASE_:				List[str]							= name.replace("downsample_layers.3.1"       ,			"stages.3.downsampling_layer.1"					)
      if "stages" in name and "downsampling_layer" not in name:
            # stages.0.0. for instance should be renamed to stages.0.layers.0.
            SCREAMING_SNAKE_CASE_:				List[Any]							= name[: len("stages.0"					)] + ".layers" + name[len("stages.0"					) :]
      if "gamma" in name:
            SCREAMING_SNAKE_CASE_:				int							= name.replace("gamma"       ,			"weight"					)
      if "beta" in name:
            SCREAMING_SNAKE_CASE_:				Any							= name.replace("beta"       ,			"bias"					)
      if "stages" in name:
            SCREAMING_SNAKE_CASE_:				Dict							= name.replace("stages"       ,			"encoder.stages"					)
      if "norm" in name:
            SCREAMING_SNAKE_CASE_:				List[str]							= name.replace("norm"       ,			"layernorm"					)
      if "head" in name:
            SCREAMING_SNAKE_CASE_:				Dict							= name.replace("head"       ,			"classifier"					)

      return name







def     A_   (					):
      SCREAMING_SNAKE_CASE_:				Optional[int]							= "http://images.cocodataset.org/val2017/000000039769.jpg"
      SCREAMING_SNAKE_CASE_:				Optional[int]							= Image.open(requests.get(_UpperCAmelCase       ,			stream=_UpperCAmelCase					).raw					)
      return im







def     A_   (					_UpperCAmelCase					):
      if "224" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				Tuple							= 2_24
            SCREAMING_SNAKE_CASE_:				Optional[int]							= 2_24 / 2_56
      elif "384" in checkpoint_url:
            SCREAMING_SNAKE_CASE_:				Any							= 3_84
            SCREAMING_SNAKE_CASE_:				Tuple							= None
      else:
            SCREAMING_SNAKE_CASE_:				Tuple							= 5_12
            SCREAMING_SNAKE_CASE_:				Optional[Any]							= None

      return ConvNextImageProcessor(
          size=_UpperCAmelCase       ,			crop_pct=_UpperCAmelCase       ,			image_mean=[0.4_8_5, 0.4_5_6, 0.4_0_6]       ,			image_std=[0.2_2_9, 0.2_2_4, 0.2_2_5]       ,			resample=PILImageResampling.BICUBIC       ,			)







@torch.no_grad()
def     A_   (					_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase					):
      print("Downloading original model from checkpoint..."					)
      # define ConvNeXTV2 configuration based on URL
      SCREAMING_SNAKE_CASE_:				Optional[int]							= get_convnextva_config(_UpperCAmelCase					)
      # load original state_dict from URL
      SCREAMING_SNAKE_CASE_:				List[str]							= torch.hub.load_state_dict_from_url(_UpperCAmelCase					)["model"]

      print("Converting model parameters..."					)
      # rename keys
      for key in state_dict.copy().keys():
            SCREAMING_SNAKE_CASE_:				List[str]							= state_dict.pop(_UpperCAmelCase					)
            SCREAMING_SNAKE_CASE_:				str							= val
      # add prefix to all keys expect classifier head
      for key in state_dict.copy().keys():
            SCREAMING_SNAKE_CASE_:				str							= state_dict.pop(_UpperCAmelCase					)
            if not key.startswith("classifier"					):
                  SCREAMING_SNAKE_CASE_:				Union[str, Any]							= "convnextv2." + key
            SCREAMING_SNAKE_CASE_:				Dict							= val

      # load HuggingFace model
      SCREAMING_SNAKE_CASE_:				Tuple							= ConvNextVaForImageClassification(_UpperCAmelCase					)
      model.load_state_dict(_UpperCAmelCase					)
      model.eval()

      # Check outputs on an image, prepared by ConvNextImageProcessor
      SCREAMING_SNAKE_CASE_:				Union[str, Any]							= convert_preprocessor(_UpperCAmelCase					)
      SCREAMING_SNAKE_CASE_:				int							= preprocessor(images=prepare_img()       ,			return_tensors="pt"					)
      SCREAMING_SNAKE_CASE_:				Optional[int]							= model(**_UpperCAmelCase					).logits

      # note: the logits below were obtained without center cropping
      if checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				int							= torch.tensor([-0.3_9_3_0, 0.1_7_4_7, -0.5_2_4_6, 0.4_1_7_7, 0.4_2_9_5]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_femto_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				List[str]							= torch.tensor([-0.1_7_2_7, -0.5_3_4_1, -0.7_8_1_8, -0.4_7_4_5, -0.6_5_6_6]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_pico_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				List[Any]							= torch.tensor([-0.0_3_3_3, 0.1_5_6_3, -0.9_1_3_7, 0.1_0_5_4, 0.0_3_8_1]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_nano_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				List[str]							= torch.tensor([-0.1_7_4_4, -0.1_5_5_5, -0.0_7_1_3, 0.0_9_5_0, -0.1_4_3_1]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_tiny_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				Optional[int]							= torch.tensor([0.9_9_9_6, 0.1_9_6_6, -0.4_3_8_6, -0.3_4_7_2, 0.6_6_6_1]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_base_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				Any							= torch.tensor([-0.2_5_5_3, -0.6_7_0_8, -0.1_3_5_9, 0.2_5_1_8, -0.2_4_8_8]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_large_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				Union[str, Any]							= torch.tensor([-0.0_6_7_3, -0.5_6_2_7, -0.3_7_5_3, -0.2_7_2_2, 0.0_1_7_8]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_huge_1k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				Optional[Any]							= torch.tensor([-0.6_3_7_7, -0.7_4_5_8, -0.2_1_5_0, 0.1_1_8_4, -0.0_5_9_7]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				int							= torch.tensor([1.0_7_9_9, 0.2_3_2_2, -0.8_8_6_0, 1.0_2_1_9, 0.6_2_3_1]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_nano_22k_384_ema.pt":
            SCREAMING_SNAKE_CASE_:				Any							= torch.tensor([0.3_7_6_6, 0.4_9_1_7, -1.1_4_2_6, 0.9_9_4_2, 0.6_0_2_4]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				int							= torch.tensor([0.4_2_2_0, -0.6_9_1_9, -0.4_3_1_7, -0.2_8_8_1, -0.6_6_0_9]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_tiny_22k_384_ema.pt":
            SCREAMING_SNAKE_CASE_:				Any							= torch.tensor([0.1_0_8_2, -0.8_2_8_6, -0.5_0_9_5, 0.4_6_8_1, -0.8_0_8_5]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				Optional[int]							= torch.tensor([-0.2_4_1_9, -0.6_2_2_1, 0.2_1_7_6, -0.0_9_8_0, -0.7_5_2_7]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_base_22k_384_ema.pt":
            SCREAMING_SNAKE_CASE_:				str							= torch.tensor([0.0_3_9_1, -0.4_3_7_1, 0.3_7_8_6, 0.1_2_5_1, -0.2_7_8_4]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_224_ema.pt":
            SCREAMING_SNAKE_CASE_:				Any							= torch.tensor([-0.0_5_0_4, 0.5_6_3_6, -0.1_7_2_9, -0.6_5_0_7, -0.3_9_4_9]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_large_22k_384_ema.pt":
            SCREAMING_SNAKE_CASE_:				Dict							= torch.tensor([0.3_5_6_0, 0.9_4_8_6, 0.3_1_4_9, -0.2_6_6_7, -0.5_1_3_8]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_384_ema.pt":
            SCREAMING_SNAKE_CASE_:				Tuple							= torch.tensor([-0.2_4_6_9, -0.4_5_5_0, -0.5_8_5_3, -0.0_8_1_0, 0.0_3_0_9]					)
      elif checkpoint_url == "https://dl.fbaipublicfiles.com/convnext/convnextv2/im22k/convnextv2_huge_22k_512_ema.pt":
            SCREAMING_SNAKE_CASE_:				str							= torch.tensor([-0.3_0_9_0, 0.0_8_0_2, -0.0_6_8_2, -0.1_9_7_9, -0.2_8_2_6]					)
      else:
            raise ValueError(f"Unknown URL: {checkpoint_url}"					)

      assert torch.allclose(logits[0, :5]       ,			_UpperCAmelCase       ,			atol=1e-3					)
      assert logits.shape == expected_shape
      print("Model outputs match the original results!"					)

      if save_model:
            print("Saving model to local..."					)
            # Create folder to save model
            if not os.path.isdir(_UpperCAmelCase					):
                  os.mkdir(_UpperCAmelCase					)

            model.save_pretrained(_UpperCAmelCase					)
            preprocessor.save_pretrained(_UpperCAmelCase					)

      SCREAMING_SNAKE_CASE_:				Optional[int]							= "convnextv2"
      if "atto" in checkpoint_url:
            model_name += "-atto"
      if "femto" in checkpoint_url:
            model_name += "-femto"
      if "pico" in checkpoint_url:
            model_name += "-pico"
      if "nano" in checkpoint_url:
            model_name += "-nano"
      elif "tiny" in checkpoint_url:
            model_name += "-tiny"
      elif "base" in checkpoint_url:
            model_name += "-base"
      elif "large" in checkpoint_url:
            model_name += "-large"
      elif "huge" in checkpoint_url:
            model_name += "-huge"
      if "22k" in checkpoint_url and "1k" not in checkpoint_url:
            model_name += "-22k"
      elif "22k" in checkpoint_url and "1k" in checkpoint_url:
            model_name += "-22k-1k"
      elif "1k" in checkpoint_url:
            model_name += "-1k"
      if "224" in checkpoint_url:
            model_name += "-224"
      elif "384" in checkpoint_url:
            model_name += "-384"
      elif "512" in checkpoint_url:
            model_name += "-512"

      if push_to_hub:
            print(f"Pushing {model_name} to the hub..."					)
            model.push_to_hub(_UpperCAmelCase					)
            preprocessor.push_to_hub(_UpperCAmelCase					)


if __name__ == "__main__":
      lowerCAmelCase							:						Tuple     			=		argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          """--checkpoint_url""",
          default="""https://dl.fbaipublicfiles.com/convnext/convnextv2/im1k/convnextv2_atto_1k_224_ema.pt""",
          type=str,
          help="""URL of the original ConvNeXTV2 checkpoint you'd like to convert.""",
      )
      parser.add_argument(
          """--pytorch_dump_folder_path""",
          default="""model""",
          type=str,
          help="""Path to the output PyTorch model directory.""",
      )
      parser.add_argument("""--save_model""", action="""store_true""", help="""Save model to local""")
      parser.add_argument("""--push_to_hub""", action="""store_true""", help="""Push model and image preprocessor to the hub""")

      lowerCAmelCase							:						Union[str, Any]     			=		parser.parse_args()
      convert_convnextva_checkpoint(
          args.checkpoint_url, args.pytorch_dump_folder_path, args.save_model, args.push_to_hub
      )









import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase							:						Tuple     			=		datasets.utils.logging.get_logger(__name__)



@dataclass
class 						__lowercase       (				datasets.BuilderConfig			):




			"""simple docstring"""







			_UpperCAmelCase     :							Optional[datasets.Features]    							=  None
			_UpperCAmelCase     :							str    							=  "utf-8"
			_UpperCAmelCase     :							Optional[str]    							=  None
			_UpperCAmelCase     :							int    							=  10 << 20  # 10MB
			_UpperCAmelCase     :							bool    							=  False
			_UpperCAmelCase     :							str    							=  "line"







class 						__lowercase       (				datasets.ArrowBasedBuilder			):




			"""simple docstring"""







			_UpperCAmelCase     :							List[Any]    							=  TextConfig



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[Any]):
									return datasets.DatasetInfo(features=self.config.features)



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[Any]     ,		lowerCAmelCase__						:  Tuple):
									if not self.config.data_files:
															raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
									SCREAMING_SNAKE_CASE_:				Dict							= dl_manager.download_and_extract(self.config.data_files)
									if isinstance(lowerCAmelCase__     ,		(str, list, tuple)):
															SCREAMING_SNAKE_CASE_:				Dict							= data_files
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				Dict							= [files]
															SCREAMING_SNAKE_CASE_:				List[str]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,		gen_kwargs={"files": files})]
									SCREAMING_SNAKE_CASE_:				Any							= []
									for split_name, files in data_files.items():
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				List[Any]							= [files]
															SCREAMING_SNAKE_CASE_:				Union[str, Any]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															splits.append(datasets.SplitGenerator(name=lowerCAmelCase__     ,		gen_kwargs={"files": files}))
									return splits



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]     ,		lowerCAmelCase__						:  pa.Table):
									if self.config.features is not None:
															SCREAMING_SNAKE_CASE_:				Optional[int]							= self.config.features.arrow_schema
															if all(not require_storage_cast(lowerCAmelCase__) for feature in self.config.features.values()):
																					# cheaper cast
																					SCREAMING_SNAKE_CASE_:				List[str]							= pa_table.cast(lowerCAmelCase__)
															else:
																					# more expensive cast; allows str <-> int/float or str to Audio for example
																					SCREAMING_SNAKE_CASE_:				List[Any]							= table_cast(lowerCAmelCase__     ,		lowerCAmelCase__)
															return pa_table
									else:
															return pa_table.cast(pa.schema({"text": pa.string()}))




			def  _SCREAMING_SNAKE_CASE    (				self						:  int     ,		lowerCAmelCase__						:  Dict):
									SCREAMING_SNAKE_CASE_:				str							= list(self.config.features) if self.config.features is not None else ["text"]
									for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__)):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(lowerCAmelCase__     ,		encoding=self.config.encoding     ,		errors=self.config.errors) as f:
																					if self.config.sample_by == "line":
																											SCREAMING_SNAKE_CASE_:				Any							= 0
																											while True:
																																	SCREAMING_SNAKE_CASE_:				str							= f.read(self.config.chunksize)
																																	if not batch:
																																							break
																																	batch += f.readline()  # finish current line
																																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= StringIO(lowerCAmelCase__).readlines()
																																	if not self.config.keep_linebreaks:
																																							SCREAMING_SNAKE_CASE_:				Optional[Any]							= [line.rstrip("\n") for line in batch]
																																	SCREAMING_SNAKE_CASE_:				Any							= pa.Table.from_arrays([pa.array(lowerCAmelCase__)]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																					elif self.config.sample_by == "paragraph":
																											SCREAMING_SNAKE_CASE_:				List[str]							= 0
																											SCREAMING_SNAKE_CASE_:				Tuple							= ""
																											while True:
																																	SCREAMING_SNAKE_CASE_:				Dict							= f.read(self.config.chunksize)
																																	if not new_batch:
																																							break
																																	batch += new_batch
																																	batch += f.readline()  # finish current line
																																	SCREAMING_SNAKE_CASE_:				List[str]							= batch.split("\n\n")
																																	SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays(
																																	    [pa.array([example for example in batch[:-1] if example])]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= batch[-1]
																											if batch:
																																	SCREAMING_SNAKE_CASE_:				int							= pa.Table.from_arrays([pa.array([batch])]     ,		names=lowerCAmelCase__)
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																					elif self.config.sample_by == "document":
																											SCREAMING_SNAKE_CASE_:				List[Any]							= f.read()
																											SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays([pa.array([text])]     ,		names=lowerCAmelCase__)
																											yield file_idx, self._cast_table(lowerCAmelCase__)






