code: 
  
  
  
  
  
  '''simple docstring'''
  
  
  import os
  import tempfile
  from pathlib import Path
  from unittest import TestCase
  
  import pyarrow as pa
  import pytest
  
  from datasets.arrow_dataset import Dataset
  from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
  from datasets.info import DatasetInfo
  from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo
  
  from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
  
  
  
  class 							__UpperCAmelCase  (       UpperCamelCase__						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def        A	(self						:      List[str]      ,      _lowerCAmelCase						:      int      ,      _lowerCAmelCase						:      Union[str, Any]=False      ):
  			A							  =      (
  			    filename_skip_take["filename"],
  			    filename_skip_take["skip"] if "skip" in filename_skip_take else None,
  			    filename_skip_take["take"] if "take" in filename_skip_take else None,
  			)
  			open(os.path.join(lowerCAmelCase__      )      ,      """wb"""      ).close()
  			A							  =      pa.Table.from_pydict({"""filename""": [Path(lowerCAmelCase__      ).name] * 100}      )
  			if take == -1:
  				A							  =      len(lowerCAmelCase__      ) - skip
  			if skip is not None and take is not None:
  				A							  =      pa_table.slice(lowerCAmelCase__      ,      lowerCAmelCase__      )
  			return pa_table
  
  
  
  
  
  
  class 							__UpperCAmelCase  (       UpperCamelCase__						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		def        A	(self						:      int      ):
  			A							  =      "my_name"
  			A							  =      SplitInfo(name="""train"""      ,      num_examples=100      )
  			A							  =      SplitInfo(name="""test"""      ,      num_examples=100      )
  			A							  =      [train_info, test_info]
  			A							  =      SplitDict()
  			split_dict.add(lowerCAmelCase__      )
  			split_dict.add(lowerCAmelCase__      )
  			A							  =      DatasetInfo(splits=lowerCAmelCase__      )
  
  			with tempfile.TemporaryDirectory() as tmp_dir:
  				A							  =      ReaderTest(lowerCAmelCase__      ,      lowerCAmelCase__      )
  
  				A							  =      "test[:33%]"
  				A							  =      Dataset(**reader.read(lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      )      )
  				self.assertEqual(dset["""filename"""][0]      ,      F"""{name}-test"""      )
  				self.assertEqual(dset.num_rows      ,      33      )
  				self.assertEqual(dset.num_columns      ,      1      )
  
  				A							  =      ["train", "test[:33%]"]
  				A							  =      [Split.TRAIN, ReadInstruction.from_spec("""test[:33%]"""      )]
  				for instructions in [instructionsa, instructionsa]:
  					A							  =      [reader.read(lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      ) for instr in instructions]
  					A							  =      (Dataset(**lowerCAmelCase__      ) for dataset_kwargs in datasets_kwargs)
  					self.assertEqual(train_dset["""filename"""][0]      ,      F"""{name}-train"""      )
  					self.assertEqual(train_dset.num_rows      ,      100      )
  					self.assertEqual(train_dset.num_columns      ,      1      )
  					self.assertIsInstance(train_dset.split      ,      lowerCAmelCase__      )
  					self.assertEqual(str(train_dset.split      )      ,      """train"""      )
  					self.assertEqual(test_dset["""filename"""][0]      ,      F"""{name}-test"""      )
  					self.assertEqual(test_dset.num_rows      ,      33      )
  					self.assertEqual(test_dset.num_columns      ,      1      )
  					self.assertIsInstance(test_dset.split      ,      lowerCAmelCase__      )
  					self.assertEqual(str(test_dset.split      )      ,      """test[:33%]"""      )
  					del train_dset, test_dset
  
  
  
  
  
  
  
  		def        A	(self						:      Tuple      ):
  			A							  =      "my_name"
  			A							  =      SplitInfo(name="""train"""      ,      num_examples=1000      ,      shard_lengths=[100] * 10      )
  			A							  =      [train_info]
  			A							  =      SplitDict()
  			split_dict.add(lowerCAmelCase__      )
  			A							  =      DatasetInfo(splits=lowerCAmelCase__      )
  
  			with tempfile.TemporaryDirectory() as tmp_dir:
  				A							  =      ReaderTest(lowerCAmelCase__      ,      lowerCAmelCase__      )
  
  				A							  =      "train[:33%]"
  				A							  =      Dataset(**reader.read(lowerCAmelCase__      ,      lowerCAmelCase__      ,      lowerCAmelCase__      )      )
  				self.assertEqual(dset["""filename"""][0]      ,      F"""{name}-train-00000-of-00010"""      )
  				self.assertEqual(dset["""filename"""][-1]      ,      F"""{name}-train-00003-of-00010"""      )
  				self.assertEqual(dset.num_rows      ,      330      )
  				self.assertEqual(dset.num_columns      ,      1      )
  
  
  
  
  
  
  
  		def        A	(self						:      int      ):
  			A							  =      SplitInfo(name="""train"""      ,      num_examples=100      )
  			A							  =      SplitInfo(name="""test"""      ,      num_examples=100      )
  			A							  =      SplitDict()
  			split_dict.add(lowerCAmelCase__      )
  			split_dict.add(lowerCAmelCase__      )
  			A							  =      DatasetInfo(splits=lowerCAmelCase__      )
  
  			with tempfile.TemporaryDirectory() as tmp_dir:
  				A							  =      ReaderTest(lowerCAmelCase__      ,      lowerCAmelCase__      )
  
  				A							  =      [
  				    {"filename": os.path.join(lowerCAmelCase__      ,      """train"""      )},
  				    {"filename": os.path.join(lowerCAmelCase__      ,      """test"""      ), "skip": 10, "take": 10},
  				]
  				A							  =      Dataset(**reader.read_files(lowerCAmelCase__      ,      original_instructions="""train+test[10:20]"""      )      )
  				self.assertEqual(dset.num_rows      ,      110      )
  				self.assertEqual(dset.num_columns      ,      1      )
  				del dset
  
  
  
  
  
  
  @pytest.mark.parametrize("""in_memory"""       ,			[False, True]    )
  def    __a (     UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase    )      ->Dict:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	A							  =      arrow_file
  	with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
  		A							  =      ArrowReader.read_table(UpperCAmelCase       ,			in_memory=UpperCAmelCase    )
  	assert table.shape == dataset.data.shape
  	assert set(table.column_names    ) == set(dataset.data.column_names    )
  	assert dict(table.to_pydict()    ) == dict(dataset.data.to_pydict()    )  # to_pydict returns OrderedDict
  
  
  
  
  @pytest.mark.parametrize("""in_memory"""       ,			[False, True]    )
  def    __a (     UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase    )      ->List[Any]:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	A							  =      arrow_file
  	A							  =      ArrowReader(""""""       ,			UpperCAmelCase    )
  	with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
  		A							  =      reader.read_files([{"""filename""": filename}]       ,			in_memory=UpperCAmelCase    )
  	assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
  	A							  =      dataset_kwargs["arrow_table"]
  	assert table.shape == dataset.data.shape
  	assert set(table.column_names    ) == set(dataset.data.column_names    )
  	assert dict(table.to_pydict()    ) == dict(dataset.data.to_pydict()    )  # to_pydict returns OrderedDict
  
  
  
  
  def    __a (     )      ->Any:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	assert ReadInstruction("""train"""       ,			to=10       ,			unit="""abs"""    ).to_spec() == "train[:10]"
  	assert ReadInstruction("""train"""       ,			from_=-80       ,			to=10       ,			unit="""%"""    ).to_spec() == "train[-80%:10%]"
  
  	A							  =      "train+test"
  	assert ReadInstruction.from_spec(UpperCAmelCase    ).to_spec() == spec_train_test
  
  	A							  =      "train[2:10]"
  	assert ReadInstruction.from_spec(UpperCAmelCase    ).to_spec() == spec_train_abs
  
  	A							  =      "train[15%:-20%]"
  	assert ReadInstruction.from_spec(UpperCAmelCase    ).to_spec() == spec_train_pct
  
  	A							  =      "train[:10%](closest)"
  	assert ReadInstruction.from_spec(UpperCAmelCase    ).to_spec() == "train[:10%]"
  
  	A							  =      "train[:10%](pct1_dropremainder)"
  	assert ReadInstruction.from_spec(UpperCAmelCase    ).to_spec() == spec_train_pct_rounding
  
  	A							  =      "train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)"
  	assert ReadInstruction.from_spec(UpperCAmelCase    ).to_spec() == spec_train_test_pct_rounding
  
  
  
  
  def    __a (     )      ->Dict:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	A							  =      "dummy"
  	A							  =      [SplitInfo(name="""train"""       ,			num_examples=100    )]
  	A							  =      "train[:33%]"
  	A							  =      "arrow"
  	A							  =      "prefix"
  
  	A							  =      make_file_instructions(UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase    )
  	assert isinstance(UpperCAmelCase       ,			UpperCAmelCase    )
  	assert file_instructions.num_examples == 33
  	assert file_instructions.file_instructions == [
  	    {"filename": os.path.join(UpperCAmelCase       ,			f"""{name}-train.arrow"""    ), "skip": 0, "take": 33}
  	]
  
  	A							  =      [SplitInfo(name="""train"""       ,			num_examples=100       ,			shard_lengths=[10] * 10    )]
  	A							  =      make_file_instructions(UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase    )
  	assert isinstance(UpperCAmelCase       ,			UpperCAmelCase    )
  	assert file_instructions.num_examples == 33
  	assert file_instructions.file_instructions == [
  	    {"filename": os.path.join(UpperCAmelCase       ,			f"""{name}-train-00000-of-00010.arrow"""    ), "skip": 0, "take": -1},
  	    {"filename": os.path.join(UpperCAmelCase       ,			f"""{name}-train-00001-of-00010.arrow"""    ), "skip": 0, "take": -1},
  	    {"filename": os.path.join(UpperCAmelCase       ,			f"""{name}-train-00002-of-00010.arrow"""    ), "skip": 0, "take": -1},
  	    {"filename": os.path.join(UpperCAmelCase       ,			f"""{name}-train-00003-of-00010.arrow"""    ), "skip": 0, "take": 3},
  	]
  
  
  
  
  @pytest.mark.parametrize("""name, expected_exception"""       ,			[(None, TypeError), ("""""", ValueError)]    )
  def    __a (     UpperCAmelCase       ,			UpperCAmelCase    )      ->Dict:
  
  
  
  	"""simple docstring"""
  
  
  
  
  
  
  
  	A							  =      [SplitInfo(name="""train"""       ,			num_examples=100    )]
  	A							  =      "train"
  	A							  =      "arrow"
  	A							  =      "prefix_path"
  	with pytest.raises(UpperCAmelCase    ):
  		A							  =      make_file_instructions(UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase       ,			UpperCAmelCase    )
  
  
style_context: 
  
  
  
  
  
  '''simple docstring'''
  
  
  import torch
  
  from diffusers import KDPMaAncestralDiscreteScheduler
  from diffusers.utils import torch_device
  
  from .test_schedulers import SchedulerCommonTest
  
  
  
  class 							__UpperCAmelCase  (       A__						):
  
  
  
  
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		__lowerCAmelCase 				=    (KDPMaAncestralDiscreteScheduler,)
  		__lowerCAmelCase 				=    10
  
  
  
  
  
  
  
  		def        A     (self						:      Any      ,      **_lowerCAmelCase						:      List[Any]      ):
  			A							  =      {
  			    """num_train_timesteps""": 1100,
  			    """beta_start""": 0.0_001,
  			    """beta_end""": 0.02,
  			    """beta_schedule""": """linear""",
  			}
  
  			config.update(**_lowerCAmelCase      )
  			return config
  
  
  
  
  
  
  
  		def        A     (self						:      Dict      ):
  			for timesteps in [10, 50, 100, 1000]:
  				self.check_over_configs(num_train_timesteps=_lowerCAmelCase      )
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ):
  			for beta_start, beta_end in zip([0.00_001, 0.0_001, 0.001]      ,      [0.0_002, 0.002, 0.02]      ):
  				self.check_over_configs(beta_start=_lowerCAmelCase      ,      beta_end=_lowerCAmelCase      )
  
  
  
  
  
  
  
  		def        A     (self						:      Tuple      ):
  			for schedule in ["linear", "scaled_linear"]:
  				self.check_over_configs(beta_schedule=_lowerCAmelCase      )
  
  
  
  
  
  
  
  		def        A     (self						:      Dict      ):
  			if torch_device == "mps":
  				return
  			A							  =      self.scheduler_classes[0]
  			A							  =      self.get_scheduler_config()
  			A							  =      scheduler_class(**_lowerCAmelCase      )
  
  			scheduler.set_timesteps(self.num_inference_steps      )
  
  			A							  =      torch.manual_seed(0      )
  
  			A							  =      self.dummy_model()
  			A							  =      self.dummy_sample_deter * scheduler.init_noise_sigma
  			A							  =      sample.to(_lowerCAmelCase      )
  
  			for i, t in enumerate(scheduler.timesteps      ):
  				A							  =      scheduler.scale_model_input(_lowerCAmelCase      ,      _lowerCAmelCase      )
  
  				A							  =      model(_lowerCAmelCase      ,      _lowerCAmelCase      )
  
  				A							  =      scheduler.step(_lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      ,      generator=_lowerCAmelCase      )
  				A							  =      output.prev_sample
  
  			A							  =      torch.sum(torch.abs(_lowerCAmelCase      )      )
  			A							  =      torch.mean(torch.abs(_lowerCAmelCase      )      )
  
  			assert abs(result_sum.item() - 13_849.3_877      ) < 1e-2
  			assert abs(result_mean.item() - 18.0_331      ) < 5e-3
  
  
  
  
  
  
  
  		def        A     (self						:      Union[str, Any]      ):
  			for prediction_type in ["epsilon", "v_prediction"]:
  				self.check_over_configs(prediction_type=_lowerCAmelCase      )
  
  
  
  
  
  
  
  		def        A     (self						:      Dict      ):
  			if torch_device == "mps":
  				return
  			A							  =      self.scheduler_classes[0]
  			A							  =      self.get_scheduler_config(prediction_type="""v_prediction"""      )
  			A							  =      scheduler_class(**_lowerCAmelCase      )
  
  			scheduler.set_timesteps(self.num_inference_steps      )
  
  			A							  =      self.dummy_model()
  			A							  =      self.dummy_sample_deter * scheduler.init_noise_sigma
  			A							  =      sample.to(_lowerCAmelCase      )
  
  			A							  =      torch.manual_seed(0      )
  
  			for i, t in enumerate(scheduler.timesteps      ):
  				A							  =      scheduler.scale_model_input(_lowerCAmelCase      ,      _lowerCAmelCase      )
  
  				A							  =      model(_lowerCAmelCase      ,      _lowerCAmelCase      )
  
  				A							  =      scheduler.step(_lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      ,      generator=_lowerCAmelCase      )
  				A							  =      output.prev_sample
  
  			A							  =      torch.sum(torch.abs(_lowerCAmelCase      )      )
  			A							  =      torch.mean(torch.abs(_lowerCAmelCase      )      )
  
  			assert abs(result_sum.item() - 328.9_970      ) < 1e-2
  			assert abs(result_mean.item() - 0.4_284      ) < 1e-3
  
  
  
  
  
  
  
  		def        A     (self						:      Optional[int]      ):
  			if torch_device == "mps":
  				return
  			A							  =      self.scheduler_classes[0]
  			A							  =      self.get_scheduler_config()
  			A							  =      scheduler_class(**_lowerCAmelCase      )
  
  			scheduler.set_timesteps(self.num_inference_steps      ,      device=_lowerCAmelCase      )
  			A							  =      torch.manual_seed(0      )
  
  			A							  =      self.dummy_model()
  			A							  =      self.dummy_sample_deter.to(_lowerCAmelCase      ) * scheduler.init_noise_sigma
  
  			for t in scheduler.timesteps:
  				A							  =      scheduler.scale_model_input(_lowerCAmelCase      ,      _lowerCAmelCase      )
  
  				A							  =      model(_lowerCAmelCase      ,      _lowerCAmelCase      )
  
  				A							  =      scheduler.step(_lowerCAmelCase      ,      _lowerCAmelCase      ,      _lowerCAmelCase      ,      generator=_lowerCAmelCase      )
  				A							  =      output.prev_sample
  
  			A							  =      torch.sum(torch.abs(_lowerCAmelCase      )      )
  			A							  =      torch.mean(torch.abs(_lowerCAmelCase      )      )
  
  			assert abs(result_sum.item() - 13_849.3_818      ) < 1e-1
  			assert abs(result_mean.item() - 18.0_331      ) < 1e-3
  
  