





'''simple docstring'''


import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


_SCREAMING_SNAKE_CASE		       =							logging.getLogger(__name__)

@dataclass(frozen=__magic_name__						)
class       lowerCAmelCase_						:
 __lowerCamelCase  :							Tuple				     =    42
 __lowerCamelCase  :							Optional[int]				     =    42
 __lowerCamelCase  :							Union[str, Any]				     =    None
 __lowerCamelCase  :							List[Any]				     =    None
 __lowerCamelCase  :							List[str]				     =    None







@dataclass(frozen=__magic_name__						)
class       lowerCAmelCase_						:
 __lowerCamelCase  :							Tuple				     =    42
 __lowerCamelCase  :							List[str]				     =    None
 __lowerCamelCase  :							Dict				     =    None
 __lowerCamelCase  :							List[str]				     =    None
 __lowerCamelCase  :							Optional[int]				     =    None


if is_torch_available():
       import torch
       from torch.utils.data import Dataset

       class       lowerCAmelCase_						(						__magic_name__						):
        __lowerCamelCase  :							Dict				     =    42




        def __init__(		self	,   _lowerCAmelCase	,   _lowerCAmelCase	,   _lowerCAmelCase	,   _lowerCAmelCase = None	,   _lowerCAmelCase=False	,   _lowerCAmelCase = False	,   )			->							Tuple:
            _lowerCAmelCase			=					hans_processors[task]()

            _lowerCAmelCase			=					os.path.join(
                __lowerCAmelCase	,   "cached_{}_{}_{}_{}".format(
                    "dev" if evaluate else "train"	,   tokenizer.__class__.__name__	,   str(__lowerCAmelCase						)	,   __lowerCAmelCase	,   )	,   )
            _lowerCAmelCase			=					processor.get_labels()
            if tokenizer.__class__ in (
                RobertaTokenizer,
                RobertaTokenizerFast,
                XLMRobertaTokenizer,
                BartTokenizer,
                BartTokenizerFast,
            ):
                # HACK(label indices are swapped in RoBERTa pretrained model)
                _lowerCAmelCase	,	_lowerCAmelCase			=					label_list[2], label_list[1]
            _lowerCAmelCase			=					label_list

            # Make sure only the first process in distributed training processes the dataset,
            # and the others will use the cache.
            _lowerCAmelCase			=					cached_features_file + ".lock"
            with FileLock(__lowerCAmelCase						):
                if os.path.exists(__lowerCAmelCase						) and not overwrite_cache:
                    logger.info(f'''Loading features from cached file {cached_features_file}'''						)
                    _lowerCAmelCase			=					torch.load(__lowerCAmelCase						)
                else:
                    logger.info(f'''Creating features from dataset file at {data_dir}'''						)

                    _lowerCAmelCase			=					(
                        processor.get_dev_examples(__lowerCAmelCase						) if evaluate else processor.get_train_examples(__lowerCAmelCase						)
                    )

                    logger.info("Training examples: %s"	,   len(__lowerCAmelCase						)						)
                    _lowerCAmelCase			=					hans_convert_examples_to_features(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase						)
                    logger.info("Saving features into cached file %s"	,   __lowerCAmelCase						)
                    torch.save(self.features	,   __lowerCAmelCase						)




        def __len__(		self						)			->							str:
            return len(self.features						)




        def __getitem__(		self	,   _lowerCAmelCase						)			->							Optional[Any]:
            return self.features[i]





        def 							_snake_case      (		self						)			->							Any:
            return self.label_list


if is_tf_available():
       import tensorflow as tf

       class       lowerCAmelCase_						:
        __lowerCamelCase  :							Union[str, Any]				     =    42




        def __init__(		self	,   _lowerCAmelCase	,   _lowerCAmelCase	,   _lowerCAmelCase	,   _lowerCAmelCase = 128	,   _lowerCAmelCase=False	,   _lowerCAmelCase = False	,   )			->							Any:
            _lowerCAmelCase			=					hans_processors[task]()
            _lowerCAmelCase			=					processor.get_labels()
            if tokenizer.__class__ in (
                RobertaTokenizer,
                RobertaTokenizerFast,
                XLMRobertaTokenizer,
                BartTokenizer,
                BartTokenizerFast,
            ):
                # HACK(label indices are swapped in RoBERTa pretrained model)
                _lowerCAmelCase	,	_lowerCAmelCase			=					label_list[2], label_list[1]
            _lowerCAmelCase			=					label_list

            _lowerCAmelCase			=					processor.get_dev_examples(__lowerCAmelCase						) if evaluate else processor.get_train_examples(__lowerCAmelCase						)
            _lowerCAmelCase			=					hans_convert_examples_to_features(__lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase	,   __lowerCAmelCase						)

            def gen():
                for ex_index, ex in tqdm.tqdm(enumerate(self.features						)	,   desc="convert examples to features"						):
                    if ex_index % 10000 == 0:
                        logger.info("Writing example %d of %d" % (ex_index, len(__lowerCAmelCase						))						)

                    yield (
                        {
                            "example_id": 0,
                            "input_ids": ex.input_ids,
                            "attention_mask": ex.attention_mask,
                            "token_type_ids": ex.token_type_ids,
                        },
                        ex.label,
                    )

            _lowerCAmelCase			=					tf.data.Dataset.from_generator(
                __lowerCAmelCase	,   (
                    {
                        "example_id": tf.intaa,
                        "input_ids": tf.intaa,
                        "attention_mask": tf.intaa,
                        "token_type_ids": tf.intaa,
                    },
                    tf.intaa,
                )	,   (
                    {
                        "example_id": tf.TensorShape([]						),
                        "input_ids": tf.TensorShape([None, None]						),
                        "attention_mask": tf.TensorShape([None, None]						),
                        "token_type_ids": tf.TensorShape([None, None]						),
                    },
                    tf.TensorShape([]						),
                )	,   )




        def 							_snake_case      (		self						)			->							List[Any]:
            return self.dataset




        def __len__(		self						)			->							str:
            return len(self.features						)




        def __getitem__(		self	,   _lowerCAmelCase						)			->							Union[str, Any]:
            return self.features[i]





        def 							_snake_case      (		self						)			->							Optional[int]:
            return self.label_list

class       lowerCAmelCase_						(						__magic_name__						):




 def 							_snake_case      (		self	,   _lowerCAmelCase						)			->							Optional[Any]:
     return self._create_examples(self._read_tsv(os.path.join(__lowerCAmelCase	,   "heuristics_train_set.txt"						)						)	,   "train"						)




 def 							_snake_case      (		self	,   _lowerCAmelCase						)			->							List[str]:
     return self._create_examples(self._read_tsv(os.path.join(__lowerCAmelCase	,   "heuristics_evaluation_set.txt"						)						)	,   "dev"						)




 def 							_snake_case      (		self						)			->							Optional[int]:
     return ["contradiction", "entailment", "neutral"]





 def 							_snake_case      (		self	,   _lowerCAmelCase	,   _lowerCAmelCase						)			->							int:
     _lowerCAmelCase			=					[]
     for i, line in enumerate(__lowerCAmelCase						):
         if i == 0:
             continue
         _lowerCAmelCase			=					"%s-%s" % (set_type, line[0])
         _lowerCAmelCase			=					line[5]
         _lowerCAmelCase			=					line[6]
         _lowerCAmelCase			=					line[7][2:] if line[7].startswith("ex"						) else line[7]
         _lowerCAmelCase			=					line[0]
         examples.append(InputExample(guid=__lowerCAmelCase	,   text_a=__lowerCAmelCase	,   text_b=__lowerCAmelCase	,   label=__lowerCAmelCase	,   pairID=__lowerCAmelCase						)						)
     return examples







def 		__a(SCREAMING_SNAKE_CASE_     :    Tuple						,				SCREAMING_SNAKE_CASE_     :    Any						,				SCREAMING_SNAKE_CASE_     :    int						,				SCREAMING_SNAKE_CASE_     :    Any						,				):



    '''simple docstring'''





    _lowerCAmelCase			=					{label: i for i, label in enumerate(__snake_case			)}

    _lowerCAmelCase			=					[]
    for ex_index, example in tqdm.tqdm(enumerate(__snake_case			)						,				desc="convert examples to features"			):
        if ex_index % 10000 == 0:
            logger.info("Writing example %d" % (ex_index)			)

        _lowerCAmelCase			=					tokenizer(
            example.text_a						,				example.text_b						,				add_special_tokens=__snake_case						,				max_length=__snake_case						,				padding="max_length"						,				truncation=__snake_case						,				return_overflowing_tokens=__snake_case						,				)

        _lowerCAmelCase			=					label_map[example.label] if example.label in label_map else 0

        _lowerCAmelCase			=					int(example.pairID			)

        features.append(InputFeatures(**__snake_case						,				label=__snake_case						,				pairID=__snake_case			)			)

    for i, example in enumerate(examples[:5]			):
        logger.info("*** Example ***"			)
        logger.info(F'''guid: {example}'''			)
        logger.info(F'''features: {features[i]}'''			)

    return features


_SCREAMING_SNAKE_CASE		       =							{
    "hans": 3,
}





_SCREAMING_SNAKE_CASE		       =							{
    "hans": HansProcessor,
}









import pytest

from datasets.parallel import ParallelBackendConfig, parallel_backend
from datasets.utils.py_utils import map_nested

from .utils import require_dill_gt_0_3_2, require_joblibspark, require_not_windows





def lowerCAmelCase__(__snake_case							)				->      int:  # picklable for multiprocessing







	'''simple docstring'''







	return i + 1





@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
def lowerCAmelCase__()				->      Any:







	'''simple docstring'''







	with parallel_backend('''spark'''							):
		assert ParallelBackendConfig.backend_name == "spark"

	lowerCamelCase__								=		[1, 2, 3]
	with pytest.raises(__snake_case							):
		with parallel_backend('''unsupported backend'''							):
			map_nested(__snake_case       ,__snake_case       ,num_proc=2							)

	with pytest.raises(__snake_case							):
		with parallel_backend('''unsupported backend'''							):
			map_nested(__snake_case       ,__snake_case       ,num_proc=-1							)





@require_dill_gt_0_3_2
@require_joblibspark
@require_not_windows
@pytest.mark.parametrize('''num_proc'''       ,[2, -1]							)
def lowerCAmelCase__(__snake_case							)				->      Tuple:







	'''simple docstring'''







	lowerCamelCase__								=		[1, 2]
	lowerCamelCase__								=		{'''a''': 1, '''b''': 2}
	lowerCamelCase__								=		{'''a''': [1, 2], '''b''': [3, 4]}
	lowerCamelCase__								=		{'''a''': {'''1''': 1}, '''b''': 2}
	lowerCamelCase__								=		{'''a''': 1, '''b''': 2, '''c''': 3, '''d''': 4}
	lowerCamelCase__								=		[2, 3]
	lowerCamelCase__								=		{'''a''': 2, '''b''': 3}
	lowerCamelCase__								=		{'''a''': [2, 3], '''b''': [4, 5]}
	lowerCamelCase__								=		{'''a''': {'''1''': 2}, '''b''': 3}
	lowerCamelCase__								=		{'''a''': 2, '''b''': 3, '''c''': 4, '''d''': 5}

	with parallel_backend('''spark'''							):
		assert map_nested(__snake_case       ,__snake_case       ,num_proc=__snake_case							) == expected_map_nested_sa
		assert map_nested(__snake_case       ,__snake_case       ,num_proc=__snake_case							) == expected_map_nested_sa
		assert map_nested(__snake_case       ,__snake_case       ,num_proc=__snake_case							) == expected_map_nested_sa
		assert map_nested(__snake_case       ,__snake_case       ,num_proc=__snake_case							) == expected_map_nested_sa
		assert map_nested(__snake_case       ,__snake_case       ,num_proc=__snake_case							) == expected_map_nested_sa



