

import unittest

import numpy as np
from datasets import load_dataset

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
						import torch

if is_vision_available():
						from PIL import Image

						from transformers import SegformerImageProcessor



class 						__lowercase       (				unittest.TestCase			):




			"""simple docstring"""







			def __init__(				self						:  str     ,		lowerCAmelCase__						:  Tuple     ,		lowerCAmelCase__						:  Optional[Any]=7     ,		lowerCAmelCase__						:  Dict=3     ,		lowerCAmelCase__						:  Optional[int]=30     ,		lowerCAmelCase__						:  Tuple=400     ,		lowerCAmelCase__						:  Optional[int]=True     ,		lowerCAmelCase__						:  Dict=None     ,		lowerCAmelCase__						:  str=True     ,		lowerCAmelCase__						:  List[Any]=[0.5, 0.5, 0.5]     ,		lowerCAmelCase__						:  List[Any]=[0.5, 0.5, 0.5]     ,		lowerCAmelCase__						:  Optional[int]=False     ,		):
									SCREAMING_SNAKE_CASE_:				Union[str, Any]							= size if size is not None else {"height": 30, "width": 30}
									SCREAMING_SNAKE_CASE_:				Any							= parent
									SCREAMING_SNAKE_CASE_:				Optional[int]							= batch_size
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= num_channels
									SCREAMING_SNAKE_CASE_:				Dict							= min_resolution
									SCREAMING_SNAKE_CASE_:				str							= max_resolution
									SCREAMING_SNAKE_CASE_:				List[str]							= do_resize
									SCREAMING_SNAKE_CASE_:				Union[str, Any]							= size
									SCREAMING_SNAKE_CASE_:				Tuple							= do_normalize
									SCREAMING_SNAKE_CASE_:				str							= image_mean
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= image_std
									SCREAMING_SNAKE_CASE_:				str							= do_reduce_labels




			def  _SCREAMING_SNAKE_CASE    (				self						:  str):
									return {
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "do_normalize": self.do_normalize,
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									    "do_reduce_labels": self.do_reduce_labels,
									}







def     A_   (					):
						SCREAMING_SNAKE_CASE_:				Tuple							= load_dataset("hf-internal-testing/fixtures_ade20k"       ,			split="test"					)

						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= Image.open(dataset[0]["file"]					)
						SCREAMING_SNAKE_CASE_:				Tuple							= Image.open(dataset[1]["file"]					)

						return image, map







def     A_   (					):
						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= load_dataset("hf-internal-testing/fixtures_ade20k"       ,			split="test"					)

						SCREAMING_SNAKE_CASE_:				Dict							= Image.open(dataset[0]["file"]					)
						SCREAMING_SNAKE_CASE_:				Tuple							= Image.open(dataset[1]["file"]					)
						SCREAMING_SNAKE_CASE_:				Dict							= Image.open(dataset[2]["file"]					)
						SCREAMING_SNAKE_CASE_:				Optional[int]							= Image.open(dataset[3]["file"]					)

						return [imagea, imagea], [mapa, mapa]



@require_torch
@require_vision
class 						__lowercase       (				UpperCAmelCase_	, unittest.TestCase			):




			"""simple docstring"""







			_UpperCAmelCase     :							Dict    							=  SegformerImageProcessor if is_vision_available() else None



			def  _SCREAMING_SNAKE_CASE    (				self						:  str):
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= SegformerImageProcessingTester(self)



			@property
			def  _SCREAMING_SNAKE_CASE    (				self						:  Any):
									return self.image_processor_tester.prepare_image_processor_dict()



			def  _SCREAMING_SNAKE_CASE    (				self						:  Any):
									SCREAMING_SNAKE_CASE_:				Optional[int]							= self.image_processing_class(**self.image_processor_dict)
									self.assertTrue(hasattr(lowerCAmelCase__     ,		"do_resize"))
									self.assertTrue(hasattr(lowerCAmelCase__     ,		"size"))
									self.assertTrue(hasattr(lowerCAmelCase__     ,		"do_normalize"))
									self.assertTrue(hasattr(lowerCAmelCase__     ,		"image_mean"))
									self.assertTrue(hasattr(lowerCAmelCase__     ,		"image_std"))
									self.assertTrue(hasattr(lowerCAmelCase__     ,		"do_reduce_labels"))



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[Any]):
									SCREAMING_SNAKE_CASE_:				Tuple							= self.image_processing_class.from_dict(self.image_processor_dict)
									self.assertEqual(image_processor.size     ,		{"height": 30, "width": 30})
									self.assertEqual(image_processor.do_reduce_labels     ,		lowerCAmelCase__)

									SCREAMING_SNAKE_CASE_:				Any							= self.image_processing_class.from_dict(self.image_processor_dict     ,		size=42     ,		reduce_labels=lowerCAmelCase__)
									self.assertEqual(image_processor.size     ,		{"height": 42, "width": 42})
									self.assertEqual(image_processor.do_reduce_labels     ,		lowerCAmelCase__)



			def  _SCREAMING_SNAKE_CASE    (				self						:  Tuple):
									pass



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[Any]):
									# Initialize image_processing
									SCREAMING_SNAKE_CASE_:				Union[str, Any]							= self.image_processing_class(**self.image_processor_dict)
									# create random PIL images
									SCREAMING_SNAKE_CASE_:				Dict							= prepare_image_inputs(self.image_processor_tester     ,		equal_resolution=lowerCAmelCase__)
									for image in image_inputs:
															self.assertIsInstance(lowerCAmelCase__     ,		Image.Image)

									# Test not batched input
									SCREAMING_SNAKE_CASE_:				List[Any]							= image_processing(image_inputs[0]     ,		return_tensors="pt").pixel_values
									self.assertEqual(
									    encoded_images.shape     ,		(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)

									# Test batched
									SCREAMING_SNAKE_CASE_:				int							= image_processing(lowerCAmelCase__     ,		return_tensors="pt").pixel_values
									self.assertEqual(
									    encoded_images.shape     ,		(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)



			def  _SCREAMING_SNAKE_CASE    (				self						:  int):
									# Initialize image_processing
									SCREAMING_SNAKE_CASE_:				Optional[int]							= self.image_processing_class(**self.image_processor_dict)
									# create random numpy tensors
									SCREAMING_SNAKE_CASE_:				Any							= prepare_image_inputs(self.image_processor_tester     ,		equal_resolution=lowerCAmelCase__     ,		numpify=lowerCAmelCase__)
									for image in image_inputs:
															self.assertIsInstance(lowerCAmelCase__     ,		np.ndarray)

									# Test not batched input
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= image_processing(image_inputs[0]     ,		return_tensors="pt").pixel_values
									self.assertEqual(
									    encoded_images.shape     ,		(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)

									# Test batched
									SCREAMING_SNAKE_CASE_:				Tuple							= image_processing(lowerCAmelCase__     ,		return_tensors="pt").pixel_values
									self.assertEqual(
									    encoded_images.shape     ,		(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)



			def  _SCREAMING_SNAKE_CASE    (				self						:  Dict):
									# Initialize image_processing
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= self.image_processing_class(**self.image_processor_dict)
									# create random PyTorch tensors
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= prepare_image_inputs(self.image_processor_tester     ,		equal_resolution=lowerCAmelCase__     ,		torchify=lowerCAmelCase__)
									for image in image_inputs:
															self.assertIsInstance(lowerCAmelCase__     ,		torch.Tensor)

									# Test not batched input
									SCREAMING_SNAKE_CASE_:				str							= image_processing(image_inputs[0]     ,		return_tensors="pt").pixel_values
									self.assertEqual(
									    encoded_images.shape     ,		(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)

									# Test batched
									SCREAMING_SNAKE_CASE_:				List[str]							= image_processing(lowerCAmelCase__     ,		return_tensors="pt").pixel_values
									self.assertEqual(
									    encoded_images.shape     ,		(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[str]):
									# Initialize image_processing
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= self.image_processing_class(**self.image_processor_dict)
									# create random PyTorch tensors
									SCREAMING_SNAKE_CASE_:				Union[str, Any]							= prepare_image_inputs(self.image_processor_tester     ,		equal_resolution=lowerCAmelCase__     ,		torchify=lowerCAmelCase__)
									SCREAMING_SNAKE_CASE_:				Dict							= []
									for image in image_inputs:
															self.assertIsInstance(lowerCAmelCase__     ,		torch.Tensor)
															maps.append(torch.zeros(image.shape[-2:]).long())

									# Test not batched input
									SCREAMING_SNAKE_CASE_:				Union[str, Any]							= image_processing(image_inputs[0]     ,		maps[0]     ,		return_tensors="pt")
									self.assertEqual(
									    encoding["pixel_values"].shape     ,		(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(
									    encoding["labels"].shape     ,		(
									        1,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(encoding["labels"].dtype     ,		torch.long)
									self.assertTrue(encoding["labels"].min().item() >= 0)
									self.assertTrue(encoding["labels"].max().item() <= 255)

									# Test batched
									SCREAMING_SNAKE_CASE_:				Optional[Any]							= image_processing(lowerCAmelCase__     ,		lowerCAmelCase__     ,		return_tensors="pt")
									self.assertEqual(
									    encoding["pixel_values"].shape     ,		(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(
									    encoding["labels"].shape     ,		(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(encoding["labels"].dtype     ,		torch.long)
									self.assertTrue(encoding["labels"].min().item() >= 0)
									self.assertTrue(encoding["labels"].max().item() <= 255)

									# Test not batched input (PIL images)
									SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_:				str							= prepare_semantic_single_inputs()

									SCREAMING_SNAKE_CASE_:				Union[str, Any]							= image_processing(lowerCAmelCase__     ,		lowerCAmelCase__     ,		return_tensors="pt")
									self.assertEqual(
									    encoding["pixel_values"].shape     ,		(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(
									    encoding["labels"].shape     ,		(
									        1,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(encoding["labels"].dtype     ,		torch.long)
									self.assertTrue(encoding["labels"].min().item() >= 0)
									self.assertTrue(encoding["labels"].max().item() <= 255)

									# Test batched input (PIL images)
									SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_:				str							= prepare_semantic_batch_inputs()

									SCREAMING_SNAKE_CASE_:				Optional[int]							= image_processing(lowerCAmelCase__     ,		lowerCAmelCase__     ,		return_tensors="pt")
									self.assertEqual(
									    encoding["pixel_values"].shape     ,		(
									        2,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(
									    encoding["labels"].shape     ,		(
									        2,
									        self.image_processor_tester.size["height"],
									        self.image_processor_tester.size["width"],
									    )     ,		)
									self.assertEqual(encoding["labels"].dtype     ,		torch.long)
									self.assertTrue(encoding["labels"].min().item() >= 0)
									self.assertTrue(encoding["labels"].max().item() <= 255)




			def  _SCREAMING_SNAKE_CASE    (				self						:  Tuple):
									# Initialize image_processing
									SCREAMING_SNAKE_CASE_:				Tuple							= self.image_processing_class(**self.image_processor_dict)

									# ADE20k has 150 classes, and the background is included, so labels should be between 0 and 150
									SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_:				Dict							= prepare_semantic_single_inputs()
									SCREAMING_SNAKE_CASE_:				List[str]							= image_processing(lowerCAmelCase__     ,		lowerCAmelCase__     ,		return_tensors="pt")
									self.assertTrue(encoding["labels"].min().item() >= 0)
									self.assertTrue(encoding["labels"].max().item() <= 150)

									SCREAMING_SNAKE_CASE_:				Optional[Any]							= True
									SCREAMING_SNAKE_CASE_:				Optional[int]							= image_processing(lowerCAmelCase__     ,		lowerCAmelCase__     ,		return_tensors="pt")
									self.assertTrue(encoding["labels"].min().item() >= 0)
									self.assertTrue(encoding["labels"].max().item() <= 255)









import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase							:						Tuple     			=		datasets.utils.logging.get_logger(__name__)



@dataclass
class 						__lowercase       (				datasets.BuilderConfig			):




			"""simple docstring"""







			_UpperCAmelCase     :							Optional[datasets.Features]    							=  None
			_UpperCAmelCase     :							str    							=  "utf-8"
			_UpperCAmelCase     :							Optional[str]    							=  None
			_UpperCAmelCase     :							int    							=  10 << 20  # 10MB
			_UpperCAmelCase     :							bool    							=  False
			_UpperCAmelCase     :							str    							=  "line"







class 						__lowercase       (				datasets.ArrowBasedBuilder			):




			"""simple docstring"""







			_UpperCAmelCase     :							List[Any]    							=  TextConfig



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[Any]):
									return datasets.DatasetInfo(features=self.config.features)



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[Any]     ,		lowerCAmelCase__						:  Tuple):
									if not self.config.data_files:
															raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
									SCREAMING_SNAKE_CASE_:				Dict							= dl_manager.download_and_extract(self.config.data_files)
									if isinstance(lowerCAmelCase__     ,		(str, list, tuple)):
															SCREAMING_SNAKE_CASE_:				Dict							= data_files
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				Dict							= [files]
															SCREAMING_SNAKE_CASE_:				List[str]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,		gen_kwargs={"files": files})]
									SCREAMING_SNAKE_CASE_:				Any							= []
									for split_name, files in data_files.items():
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				List[Any]							= [files]
															SCREAMING_SNAKE_CASE_:				Union[str, Any]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															splits.append(datasets.SplitGenerator(name=lowerCAmelCase__     ,		gen_kwargs={"files": files}))
									return splits



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]     ,		lowerCAmelCase__						:  pa.Table):
									if self.config.features is not None:
															SCREAMING_SNAKE_CASE_:				Optional[int]							= self.config.features.arrow_schema
															if all(not require_storage_cast(lowerCAmelCase__) for feature in self.config.features.values()):
																					# cheaper cast
																					SCREAMING_SNAKE_CASE_:				List[str]							= pa_table.cast(lowerCAmelCase__)
															else:
																					# more expensive cast; allows str <-> int/float or str to Audio for example
																					SCREAMING_SNAKE_CASE_:				List[Any]							= table_cast(lowerCAmelCase__     ,		lowerCAmelCase__)
															return pa_table
									else:
															return pa_table.cast(pa.schema({"text": pa.string()}))




			def  _SCREAMING_SNAKE_CASE    (				self						:  int     ,		lowerCAmelCase__						:  Dict):
									SCREAMING_SNAKE_CASE_:				str							= list(self.config.features) if self.config.features is not None else ["text"]
									for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__)):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(lowerCAmelCase__     ,		encoding=self.config.encoding     ,		errors=self.config.errors) as f:
																					if self.config.sample_by == "line":
																											SCREAMING_SNAKE_CASE_:				Any							= 0
																											while True:
																																	SCREAMING_SNAKE_CASE_:				str							= f.read(self.config.chunksize)
																																	if not batch:
																																							break
																																	batch += f.readline()  # finish current line
																																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= StringIO(lowerCAmelCase__).readlines()
																																	if not self.config.keep_linebreaks:
																																							SCREAMING_SNAKE_CASE_:				Optional[Any]							= [line.rstrip("\n") for line in batch]
																																	SCREAMING_SNAKE_CASE_:				Any							= pa.Table.from_arrays([pa.array(lowerCAmelCase__)]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																					elif self.config.sample_by == "paragraph":
																											SCREAMING_SNAKE_CASE_:				List[str]							= 0
																											SCREAMING_SNAKE_CASE_:				Tuple							= ""
																											while True:
																																	SCREAMING_SNAKE_CASE_:				Dict							= f.read(self.config.chunksize)
																																	if not new_batch:
																																							break
																																	batch += new_batch
																																	batch += f.readline()  # finish current line
																																	SCREAMING_SNAKE_CASE_:				List[str]							= batch.split("\n\n")
																																	SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays(
																																	    [pa.array([example for example in batch[:-1] if example])]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= batch[-1]
																											if batch:
																																	SCREAMING_SNAKE_CASE_:				int							= pa.Table.from_arrays([pa.array([batch])]     ,		names=lowerCAmelCase__)
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																					elif self.config.sample_by == "document":
																											SCREAMING_SNAKE_CASE_:				List[Any]							= f.read()
																											SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays([pa.array([text])]     ,		names=lowerCAmelCase__)
																											yield file_idx, self._cast_table(lowerCAmelCase__)






