





"""simple docstring"""



from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)







def 		__lowercase     (   snake_case_  :  Optional[int]				,snake_case_  :  Optional[int]						)   ->List[Any]:



    '''simple docstring'''







    __A    :						int						 =     []
    for part_id in partition_order:
        __A    :						Union[str, Any]						 =     df.where(F"""SPARK_PARTITION_ID() = {part_id}"""						).collect()
        for row_idx, row in enumerate(snake_case_						):
            expected_row_ids_and_row_dicts.append((F"""{part_id}_{row_idx}""", row.asDict())						)
    return expected_row_ids_and_row_dicts







@require_not_windows
@require_dill_gt_0_3_2
def 		__lowercase     (   )   ->Optional[int]:



    '''simple docstring'''







    __A    :						Union[str, Any]						 =     pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
    __A    :						Union[str, Any]						 =     spark.range(100						).repartition(1						)
    __A    :						Union[str, Any]						 =     Spark(snake_case_						)
    # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
    # that each partition can hold 2 rows.
    spark_builder._repartition_df_if_needed(max_shard_size=16						)
    # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
    assert spark_builder.df.rdd.getNumPartitions() == 50







@require_not_windows
@require_dill_gt_0_3_2
def 		__lowercase     (   )   ->Optional[Any]:



    '''simple docstring'''







    __A    :						List[Any]						 =     pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
    __A    :						List[str]						 =     spark.range(10						).repartition(2						)
    __A    :						List[str]						 =     [1, 0]
    __A    :						List[Any]						 =     _generate_iterable_examples(snake_case_				,snake_case_						)  # Reverse the partitions.
    __A    :						Tuple						 =     _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_				,snake_case_						)

    for i, (row_id, row_dict) in enumerate(generate_fn()						):
        __A ,       __A    :						Dict						 =     expected_row_ids_and_row_dicts[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		__lowercase     (   )   ->Any:



    '''simple docstring'''







    __A    :						str						 =     pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
    __A    :						Optional[Any]						 =     spark.range(10						).repartition(1						)
    __A    :						List[str]						 =     SparkExamplesIterable(snake_case_						)
    assert it.n_shards == 1
    for i, (row_id, row_dict) in enumerate(snake_case_						):
        assert row_id == F"""0_{i}"""
        assert row_dict == {"id": i}







@require_not_windows
@require_dill_gt_0_3_2
def 		__lowercase     (   )   ->Dict:



    '''simple docstring'''







    __A    :						Tuple						 =     pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
    __A    :						Dict						 =     spark.range(30						).repartition(3						)
    # Mock the generator so that shuffle reverses the partition indices.
    with patch('''numpy.random.Generator'''						) as generator_mock:
        __A    :						str						 =     lambda snake_case_						: x.reverse()
        __A    :						List[Any]						 =     _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_				,[2, 1, 0]						)

        __A    :						str						 =     SparkExamplesIterable(snake_case_						).shuffle_data_sources(snake_case_						)
        assert shuffled_it.n_shards == 3
        for i, (row_id, row_dict) in enumerate(snake_case_						):
            __A ,       __A    :						str						 =     expected_row_ids_and_row_dicts[i]
            assert row_id == expected_row_id
            assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		__lowercase     (   )   ->int:



    '''simple docstring'''







    __A    :						Dict						 =     pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
    __A    :						Dict						 =     spark.range(20						).repartition(4						)

    # Partitions 0 and 2
    __A    :						List[str]						 =     SparkExamplesIterable(snake_case_						).shard_data_sources(worker_id=0				,num_workers=2						)
    assert shard_it_a.n_shards == 2
    __A    :						List[Any]						 =     _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_				,[0, 2]						)
    for i, (row_id, row_dict) in enumerate(snake_case_						):
        __A ,       __A    :						Tuple						 =     expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict

    # Partitions 1 and 3
    __A    :						List[Any]						 =     SparkExamplesIterable(snake_case_						).shard_data_sources(worker_id=1				,num_workers=2						)
    assert shard_it_a.n_shards == 2
    __A    :						Dict						 =     _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case_				,[1, 3]						)
    for i, (row_id, row_dict) in enumerate(snake_case_						):
        __A ,       __A    :						List[Any]						 =     expected_row_ids_and_row_dicts_a[i]
        assert row_id == expected_row_id
        assert row_dict == expected_row_dict







@require_not_windows
@require_dill_gt_0_3_2
def 		__lowercase     (   )   ->List[Any]:



    '''simple docstring'''







    __A    :						List[str]						 =     pyspark.sql.SparkSession.builder.master('''local[*]'''						).appName('''pyspark'''						).getOrCreate()
    __A    :						Tuple						 =     spark.range(100						).repartition(1						)
    __A    :						Optional[int]						 =     Spark(snake_case_						)
    # Choose a small max_shard_size for maximum partitioning.
    spark_builder._repartition_df_if_needed(max_shard_size=1						)
    # The new number of partitions should not be greater than the number of rows.
    assert spark_builder.df.rdd.getNumPartitions() == 100











"""simple docstring"""



# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import torch

from ..models.auto import AutoModelForSequenceClassification, AutoTokenizer
from .base import PipelineTool






class 		__snake_case					(		SCREAMING_SNAKE_CASE__     ):





 """simple docstring"""


 _lowerCamelCase        =     """facebook/bart-large-mnli"""
 _lowerCamelCase        =     (
     """This is a tool that classifies an English text using provided labels. It takes two inputs: `text`, which """
     """should be the text to classify, and `labels`, which should be the list of labels to use for classification. """
     """It returns the most likely label in the list of provided `labels` for the input text."""
 )
 _lowerCamelCase        =     """text_classifier"""
 _lowerCamelCase        =     AutoTokenizer
 _lowerCamelCase        =     AutoModelForSequenceClassification

 _lowerCamelCase        =     ["""text""", ["""text"""]]
 _lowerCamelCase        =     ["""text"""]

 def 					UpperCamelCase__(    self  ):



     '''simple docstring'''




     super().setup()
     __A    :						List[str]						 =     self.model.config
     __A    :						int						 =     -1
     for idx, label in config.idalabel.items():
         if label.lower().startswith('''entail'''  ):
             __A    :						List[str]						 =     int(__lowerCamelCase  )
     if self.entailment_id == -1:
         raise ValueError('''Could not determine the entailment ID from the model config, please pass it at init.'''  )

 def 					UpperCamelCase__(    self					,						__lowerCamelCase					,						__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						Union[str, Any]						 =     labels
     return self.pre_processor(
         [text] * len(__lowerCamelCase  )					,						[F"""This example is {label}""" for label in labels]					,						return_tensors='''pt'''					,						padding='''max_length'''					,						)







 def 					UpperCamelCase__(    self					,						__lowerCamelCase  ):



     '''simple docstring'''




     __A    :						List[Any]						 =     outputs.logits
     __A    :						List[str]						 =     torch.argmax(logits[:, 2]  ).item()
     return self._labels[label_id]




