




import gc
import tempfile
import unittest

import numpy as np
import torch

from diffusers import VersatileDiffusionTextToImagePipeline
from diffusers.utils.testing_utils import nightly, require_torch_gpu, torch_device


__lowerCAmelCase		:				Optional[int]       			=		False




class     snake_case__							(unittest.TestCase   ):
				"""simple docstring"""



				pass




@nightly
@require_torch_gpu
class     snake_case__							(unittest.TestCase   ):
				"""simple docstring"""



				def 							__UpperCAmelCase							(      self				:     Optional[Any]							)					->     List[str]:
							# clean up the VRAM after each test
							super().tearDown()
							gc.collect()
							torch.cuda.empty_cache()





				def 							__UpperCAmelCase							(      self				:     Dict							)					->     int:
							a    =			VersatileDiffusionTextToImagePipeline.from_pretrained("shi-labs/versatile-diffusion"							)
							# remove text_unet
							pipe.remove_unused_weights()
							pipe.to(__lowerCamelCase							)
							pipe.set_progress_bar_config(disable=__lowerCamelCase							)

							a    =			"A painting of a squirrel eating a burger "
							a    =			torch.manual_seed(0							)
							a    =			pipe(
							    prompt=__lowerCamelCase					,      generator=__lowerCamelCase					,      guidance_scale=7.5					,      num_inference_steps=2					,      output_type="numpy"							).images

							with tempfile.TemporaryDirectory() as tmpdirname:
										pipe.save_pretrained(__lowerCamelCase							)
										a    =			VersatileDiffusionTextToImagePipeline.from_pretrained(__lowerCamelCase							)
							pipe.to(__lowerCamelCase							)
							pipe.set_progress_bar_config(disable=__lowerCamelCase							)

							a    =			generator.manual_seed(0							)
							a    =			pipe(
							    prompt=__lowerCamelCase					,      generator=__lowerCamelCase					,      guidance_scale=7.5					,      num_inference_steps=2					,      output_type="numpy"							).images

							assert np.abs(image - new_image							).sum() < 1e-5, "Models don't have the same forward pass"





				def 							__UpperCAmelCase							(      self				:     str							)					->     List[str]:
							a    =			VersatileDiffusionTextToImagePipeline.from_pretrained(
							    "shi-labs/versatile-diffusion"					,      torch_dtype=torch.floataa							)
							pipe.to(__lowerCamelCase							)
							pipe.set_progress_bar_config(disable=__lowerCamelCase							)

							a    =			"A painting of a squirrel eating a burger "
							a    =			torch.manual_seed(0							)
							a    =			pipe(
							    prompt=__lowerCamelCase					,      generator=__lowerCamelCase					,      guidance_scale=7.5					,      num_inference_steps=50					,      output_type="numpy"							).images

							a    =			image[0, 2_53:2_56, 2_53:2_56, -1]

							assert image.shape == (1, 5_12, 5_12, 3)
							a    =			np.array([0.3_367, 0.3_169, 0.2_656, 0.3_870, 0.4_790, 0.3_796, 0.4_009, 0.4_878, 0.4_778]							)

							assert np.abs(image_slice.flatten() - expected_slice							).max() < 1e-2







from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def  snake_case  (       snake_case__			:Union[str, Any]   ,	snake_case__			:Dict)						->   Any:
       _A        =  []
       for part_id in partition_order:
              _A        =  df.where(F'''SPARK_PARTITION_ID() = {part_id}''').collect()
              for row_idx, row in enumerate(snake_case__):
                     expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict()))
       return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def  snake_case  (       )						->   Optional[Any]:
       _A        =  pyspark.sql.SparkSession.builder.master("""local[*]""").appName("""pyspark""").getOrCreate()
       _A        =  spark.range(100).repartition(1)
       _A        =  Spark(snake_case__)
       # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
       # that each partition can hold 2 rows.
       spark_builder._repartition_df_if_needed(max_shard_size=16)
       # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
       assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def  snake_case  (       )						->   Union[str, Any]:
       _A        =  pyspark.sql.SparkSession.builder.master("""local[*]""").appName("""pyspark""").getOrCreate()
       _A        =  spark.range(10).repartition(2)
       _A        =  [1, 0]
       _A        =  _generate_iterable_examples(snake_case__   ,	snake_case__)  # Reverse the partitions.
       _A        =  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__   ,	snake_case__)

       for i, (row_id, row_dict) in enumerate(generate_fn()):
              _A   ,  _A        =  expected_row_ids_and_row_dicts[i]
              assert row_id == expected_row_id
              assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def  snake_case  (       )						->   int:
       _A        =  pyspark.sql.SparkSession.builder.master("""local[*]""").appName("""pyspark""").getOrCreate()
       _A        =  spark.range(10).repartition(1)
       _A        =  SparkExamplesIterable(snake_case__)
       assert it.n_shards == 1
       for i, (row_id, row_dict) in enumerate(snake_case__):
              assert row_id == F'''0_{i}'''
              assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def  snake_case  (       )						->   Union[str, Any]:
       _A        =  pyspark.sql.SparkSession.builder.master("""local[*]""").appName("""pyspark""").getOrCreate()
       _A        =  spark.range(30).repartition(3)
       # Mock the generator so that shuffle reverses the partition indices.
       with patch("""numpy.random.Generator""") as generator_mock:
              _A        =  lambda snake_case__: x.reverse()
              _A        =  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__   ,	[2, 1, 0])

              _A        =  SparkExamplesIterable(snake_case__).shuffle_data_sources(snake_case__)
              assert shuffled_it.n_shards == 3
              for i, (row_id, row_dict) in enumerate(snake_case__):
                     _A   ,  _A        =  expected_row_ids_and_row_dicts[i]
                     assert row_id == expected_row_id
                     assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def  snake_case  (       )						->   List[str]:
       _A        =  pyspark.sql.SparkSession.builder.master("""local[*]""").appName("""pyspark""").getOrCreate()
       _A        =  spark.range(20).repartition(4)

       # Partitions 0 and 2
       _A        =  SparkExamplesIterable(snake_case__).shard_data_sources(worker_id=0   ,	num_workers=2)
       assert shard_it_a.n_shards == 2
       _A        =  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__   ,	[0, 2])
       for i, (row_id, row_dict) in enumerate(snake_case__):
              _A   ,  _A        =  expected_row_ids_and_row_dicts_a[i]
              assert row_id == expected_row_id
              assert row_dict == expected_row_dict

       # Partitions 1 and 3
       _A        =  SparkExamplesIterable(snake_case__).shard_data_sources(worker_id=1   ,	num_workers=2)
       assert shard_it_a.n_shards == 2
       _A        =  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__   ,	[1, 3])
       for i, (row_id, row_dict) in enumerate(snake_case__):
              _A   ,  _A        =  expected_row_ids_and_row_dicts_a[i]
              assert row_id == expected_row_id
              assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def  snake_case  (       )						->   Tuple:
       _A        =  pyspark.sql.SparkSession.builder.master("""local[*]""").appName("""pyspark""").getOrCreate()
       _A        =  spark.range(100).repartition(1)
       _A        =  Spark(snake_case__)
       # Choose a small max_shard_size for maximum partitioning.
       spark_builder._repartition_df_if_needed(max_shard_size=1)
       # The new number of partitions should not be greater than the number of rows.
       assert spark_builder.df.rdd.getNumPartitions() == 100




