

from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def        UpperCamelCase_(							_snake_case   :  Optional[int]       ,     _snake_case   :  Optional[Any]	):




			"""simple docstring"""






			__a        =[]
			for part_id in partition_order:
						__a        =df.where(F'SPARK_PARTITION_ID() = {part_id}'	).collect()
						for row_idx, row in enumerate(_snake_case	):
									expected_row_ids_and_row_dicts.append((F'{part_id}_{row_idx}', row.asDict())	)
			return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def        UpperCamelCase_(							):




			"""simple docstring"""






			__a        =pyspark.sql.SparkSession.builder.master('local[*]'	).appName('pyspark'	).getOrCreate()
			__a        =spark.range(100	).repartition(1	)
			__a        =Spark(_snake_case	)
			# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
			# that each partition can hold 2 rows.
			spark_builder._repartition_df_if_needed(max_shard_size=16	)
			# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
			assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def        UpperCamelCase_(							):




			"""simple docstring"""






			__a        =pyspark.sql.SparkSession.builder.master('local[*]'	).appName('pyspark'	).getOrCreate()
			__a        =spark.range(10	).repartition(2	)
			__a        =[1, 0]
			__a        =_generate_iterable_examples(_snake_case       ,     _snake_case	)  # Reverse the partitions.
			__a        =_get_expected_row_ids_and_row_dicts_for_partition_order(_snake_case       ,     _snake_case	)

			for i, (row_id, row_dict) in enumerate(generate_fn()	):
						__a      ,    __a        =expected_row_ids_and_row_dicts[i]
						assert row_id == expected_row_id
						assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def        UpperCamelCase_(							):




			"""simple docstring"""






			__a        =pyspark.sql.SparkSession.builder.master('local[*]'	).appName('pyspark'	).getOrCreate()
			__a        =spark.range(10	).repartition(1	)
			__a        =SparkExamplesIterable(_snake_case	)
			assert it.n_shards == 1
			for i, (row_id, row_dict) in enumerate(_snake_case	):
						assert row_id == F'0_{i}'
						assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def        UpperCamelCase_(							):




			"""simple docstring"""






			__a        =pyspark.sql.SparkSession.builder.master('local[*]'	).appName('pyspark'	).getOrCreate()
			__a        =spark.range(30	).repartition(3	)
			# Mock the generator so that shuffle reverses the partition indices.
			with patch('numpy.random.Generator'	) as generator_mock:
						__a        =lambda _snake_case	: x.reverse()
						__a        =_get_expected_row_ids_and_row_dicts_for_partition_order(_snake_case       ,     [2, 1, 0]	)

						__a        =SparkExamplesIterable(_snake_case	).shuffle_data_sources(_snake_case	)
						assert shuffled_it.n_shards == 3
						for i, (row_id, row_dict) in enumerate(_snake_case	):
									__a      ,    __a        =expected_row_ids_and_row_dicts[i]
									assert row_id == expected_row_id
									assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def        UpperCamelCase_(							):




			"""simple docstring"""






			__a        =pyspark.sql.SparkSession.builder.master('local[*]'	).appName('pyspark'	).getOrCreate()
			__a        =spark.range(20	).repartition(4	)

			# Partitions 0 and 2
			__a        =SparkExamplesIterable(_snake_case	).shard_data_sources(worker_id=0       ,     num_workers=2	)
			assert shard_it_a.n_shards == 2
			__a        =_get_expected_row_ids_and_row_dicts_for_partition_order(_snake_case       ,     [0, 2]	)
			for i, (row_id, row_dict) in enumerate(_snake_case	):
						__a      ,    __a        =expected_row_ids_and_row_dicts_a[i]
						assert row_id == expected_row_id
						assert row_dict == expected_row_dict

			# Partitions 1 and 3
			__a        =SparkExamplesIterable(_snake_case	).shard_data_sources(worker_id=1       ,     num_workers=2	)
			assert shard_it_a.n_shards == 2
			__a        =_get_expected_row_ids_and_row_dicts_for_partition_order(_snake_case       ,     [1, 3]	)
			for i, (row_id, row_dict) in enumerate(_snake_case	):
						__a      ,    __a        =expected_row_ids_and_row_dicts_a[i]
						assert row_id == expected_row_id
						assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def        UpperCamelCase_(							):




			"""simple docstring"""






			__a        =pyspark.sql.SparkSession.builder.master('local[*]'	).appName('pyspark'	).getOrCreate()
			__a        =spark.range(100	).repartition(1	)
			__a        =Spark(_snake_case	)
			# Choose a small max_shard_size for maximum partitioning.
			spark_builder._repartition_df_if_needed(max_shard_size=1	)
			# The new number of partitions should not be greater than the number of rows.
			assert spark_builder.df.rdd.getNumPartitions() == 100









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_tokenizers_available,
    is_torch_available,
)


_lowerCAmelCase					:			Tuple  				=			{
    "configuration_llama": ["LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP", "LlamaConfig"],
}

try:
						if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowerCAmelCase					:			Optional[int]  				=			["LlamaTokenizer"]

try:
						if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowerCAmelCase					:			Optional[int]  				=			["LlamaTokenizerFast"]

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						_lowerCAmelCase					:			List[Any]  				=			[
						    "LlamaForCausalLM",
						    "LlamaModel",
						    "LlamaPreTrainedModel",
						    "LlamaForSequenceClassification",
						]


if TYPE_CHECKING:
						from .configuration_llama import LLAMA_PRETRAINED_CONFIG_ARCHIVE_MAP, LlamaConfig

						try:
												if not is_sentencepiece_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_llama import LlamaTokenizer

						try:
												if not is_tokenizers_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .tokenization_llama_fast import LlamaTokenizerFast

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_llama import LlamaForCausalLM, LlamaForSequenceClassification, LlamaModel, LlamaPreTrainedModel


else:
						import sys

						_lowerCAmelCase					:			int  				=			_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)






