
import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


_A   =     logging.getLogger(__name__)





@dataclass(frozen=__a       )
class 	_lowerCAmelCase					:
				_lowercase					=42
				_lowercase					=42
				_lowercase					=None
				_lowercase					=None
				_lowercase					=None





@dataclass(frozen=__a       )
class 	_lowerCAmelCase					:
				_lowercase					=42
				_lowercase					=None
				_lowercase					=None
				_lowercase					=None
				_lowercase					=None


if is_torch_available():
			import torch
			from torch.utils.data import Dataset





			class 	_lowerCAmelCase					(	__a       ):
							_lowercase					=42




							def __init__(					self				,				_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase = None				,				_UpperCamelCase=False				,				_UpperCamelCase = False				,				)				->      Tuple:
										lowerCAmelCase_						=	hans_processors[task]()

										lowerCAmelCase_						=	os.path.join(
										    _UpperCamelCase				,				"cached_{}_{}_{}_{}".format(
										        "dev" if evaluate else "train"				,				tokenizer.__class__.__name__				,				str(_UpperCamelCase     )				,				_UpperCamelCase				,				)				,				)
										lowerCAmelCase_						=	processor.get_labels()
										if tokenizer.__class__ in (
										    RobertaTokenizer,
										    RobertaTokenizerFast,
										    XLMRobertaTokenizer,
										    BartTokenizer,
										    BartTokenizerFast,
										):
													# HACK(label indices are swapped in RoBERTa pretrained model)
													lowerCAmelCase_	,  lowerCAmelCase_						=	label_list[2], label_list[1]
										lowerCAmelCase_						=	label_list

										# Make sure only the first process in distributed training processes the dataset,
										# and the others will use the cache.
										lowerCAmelCase_						=	cached_features_file + ".lock"
										with FileLock(_UpperCamelCase     ):
													if os.path.exists(_UpperCamelCase     ) and not overwrite_cache:
																logger.info(f"""Loading features from cached file {cached_features_file}"""     )
																lowerCAmelCase_						=	torch.load(_UpperCamelCase     )
													else:
																logger.info(f"""Creating features from dataset file at {data_dir}"""     )

																lowerCAmelCase_						=	(
																    processor.get_dev_examples(_UpperCamelCase     ) if evaluate else processor.get_train_examples(_UpperCamelCase     )
																)

																logger.info("Training examples: %s"				,				len(_UpperCamelCase     )     )
																lowerCAmelCase_						=	hans_convert_examples_to_features(_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase     )
																logger.info("Saving features into cached file %s"				,				_UpperCamelCase     )
																torch.save(self.features				,				_UpperCamelCase     )




							def __len__(					self     )				->      int:
										return len(self.features     )




							def __getitem__(					self				,				_UpperCamelCase     )				->      InputFeatures:
										return self.features[i]




							def      __a						(					self     )				->      Tuple:
										return self.label_list


if is_tf_available():
			import tensorflow as tf





			class 	_lowerCAmelCase					:
							_lowercase					=42




							def __init__(					self				,				_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase = 128				,				_UpperCamelCase=False				,				_UpperCamelCase = False				,				)				->      Optional[int]:
										lowerCAmelCase_						=	hans_processors[task]()
										lowerCAmelCase_						=	processor.get_labels()
										if tokenizer.__class__ in (
										    RobertaTokenizer,
										    RobertaTokenizerFast,
										    XLMRobertaTokenizer,
										    BartTokenizer,
										    BartTokenizerFast,
										):
													# HACK(label indices are swapped in RoBERTa pretrained model)
													lowerCAmelCase_	,  lowerCAmelCase_						=	label_list[2], label_list[1]
										lowerCAmelCase_						=	label_list

										lowerCAmelCase_						=	processor.get_dev_examples(_UpperCamelCase     ) if evaluate else processor.get_train_examples(_UpperCamelCase     )
										lowerCAmelCase_						=	hans_convert_examples_to_features(_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase				,				_UpperCamelCase     )

										def gen():
													for ex_index, ex in tqdm.tqdm(enumerate(self.features     )				,				desc="convert examples to features"     ):
																if ex_index % 10_000 == 0:
																			logger.info("Writing example %d of %d" % (ex_index, len(_UpperCamelCase     ))     )

																yield (
																    {
																        "example_id": 0,
																        "input_ids": ex.input_ids,
																        "attention_mask": ex.attention_mask,
																        "token_type_ids": ex.token_type_ids,
																    },
																    ex.label,
																)

										lowerCAmelCase_						=	tf.data.Dataset.from_generator(
										    _UpperCamelCase				,				(
										        {
										            "example_id": tf.intaa,
										            "input_ids": tf.intaa,
										            "attention_mask": tf.intaa,
										            "token_type_ids": tf.intaa,
										        },
										        tf.intaa,
										    )				,				(
										        {
										            "example_id": tf.TensorShape([]     ),
										            "input_ids": tf.TensorShape([None, None]     ),
										            "attention_mask": tf.TensorShape([None, None]     ),
										            "token_type_ids": tf.TensorShape([None, None]     ),
										        },
										        tf.TensorShape([]     ),
										    )				,				)




							def      __a						(					self     )				->      str:
										return self.dataset




							def __len__(					self     )				->      Optional[int]:
										return len(self.features     )




							def __getitem__(					self				,				_UpperCamelCase     )				->      InputFeatures:
										return self.features[i]




							def      __a						(					self     )				->      List[Any]:
										return self.label_list





class 	_lowerCAmelCase					(	__a       ):




				def      __a						(					self				,				_UpperCamelCase     )				->      Union[str, Any]:
							return self._create_examples(self._read_tsv(os.path.join(_UpperCamelCase				,				"heuristics_train_set.txt"     )     )				,				"train"     )




				def      __a						(					self				,				_UpperCamelCase     )				->      Dict:
							return self._create_examples(self._read_tsv(os.path.join(_UpperCamelCase				,				"heuristics_evaluation_set.txt"     )     )				,				"dev"     )




				def      __a						(					self     )				->      Optional[Any]:
							return ["contradiction", "entailment", "neutral"]




				def      __a						(					self				,				_UpperCamelCase				,				_UpperCamelCase     )				->      List[Any]:
							lowerCAmelCase_						=	[]
							for i, line in enumerate(_UpperCamelCase     ):
										if i == 0:
													continue
										lowerCAmelCase_						=	"%s-%s" % (set_type, line[0])
										lowerCAmelCase_						=	line[5]
										lowerCAmelCase_						=	line[6]
										lowerCAmelCase_						=	line[7][2:] if line[7].startswith("ex"     ) else line[7]
										lowerCAmelCase_						=	line[0]
										examples.append(InputExample(guid=_UpperCamelCase				,				text_a=_UpperCamelCase				,				text_b=_UpperCamelCase				,				label=_UpperCamelCase				,				pairID=_UpperCamelCase     )     )
							return examples






def    lowerCamelCase__				(				__lowerCAmelCase  :  List[InputExample]    ,							__lowerCAmelCase  :  List[str]    ,							__lowerCAmelCase  :  int    ,							__lowerCAmelCase  :  PreTrainedTokenizer    ,							):




			"""simple docstring"""


			lowerCAmelCase_						=	{label: i for i, label in enumerate(__lowerCAmelCase				)}

			lowerCAmelCase_						=	[]
			for ex_index, example in tqdm.tqdm(enumerate(__lowerCAmelCase				)    ,							desc="convert examples to features"				):
						if ex_index % 10000 == 0:
									logger.info("Writing example %d" % (ex_index)				)

						lowerCAmelCase_						=	tokenizer(
						    example.text_a    ,							example.text_b    ,							add_special_tokens=__lowerCAmelCase    ,							max_length=__lowerCAmelCase    ,							padding="max_length"    ,							truncation=__lowerCAmelCase    ,							return_overflowing_tokens=__lowerCAmelCase    ,							)

						lowerCAmelCase_						=	label_map[example.label] if example.label in label_map else 0

						lowerCAmelCase_						=	int(example.pairID				)

						features.append(InputFeatures(**__lowerCAmelCase    ,							label=__lowerCAmelCase    ,							pairID=__lowerCAmelCase				)				)

			for i, example in enumerate(examples[:5]				):
						logger.info("*** Example ***"				)
						logger.info(F"""guid: {example}"""				)
						logger.info(F"""features: {features[i]}"""				)

			return features


_A   =     {
    "hans": 3,
}
_A   =     {
    "hans": HansProcessor,
}





from typing import List, Optional, TypeVar

from .arrow_dataset import Dataset, _concatenate_map_style_datasets, _interleave_map_style_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .info import DatasetInfo
from .iterable_dataset import IterableDataset, _concatenate_iterable_datasets, _interleave_iterable_datasets
from .splits import NamedSplit
from .utils import logging
from .utils.py_utils import Literal


_A   =     logging.get_logger(__name__)


_A   =     TypeVar("DatasetType", Dataset, IterableDataset)






def    lowerCamelCase__				(				__lowerCAmelCase  :  List[DatasetType]    ,							__lowerCAmelCase  :  Optional[List[float]] = None    ,							__lowerCAmelCase  :  Optional[int] = None    ,							__lowerCAmelCase  :  Optional[DatasetInfo] = None    ,							__lowerCAmelCase  :  Optional[NamedSplit] = None    ,							__lowerCAmelCase  :  Literal["first_exhausted", "all_exhausted"] = "first_exhausted"    ,							):




			"""simple docstring"""


			from .arrow_dataset import Dataset
			from .iterable_dataset import IterableDataset

			if not datasets:
						raise ValueError("Unable to interleave an empty list of datasets."				)
			for i, dataset in enumerate(__lowerCAmelCase				):
						if not isinstance(__lowerCAmelCase    ,							(Dataset, IterableDataset)				):
									if isinstance(__lowerCAmelCase    ,							(DatasetDict, IterableDatasetDict)				):
												if not dataset:
															raise ValueError(
															    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} """
															    "is an empty dataset dictionary."				)
												raise ValueError(
												    F"""Dataset at position {i} has at least one split: {list(__lowerCAmelCase				)}\n"""
												    F"""Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__lowerCAmelCase				)				)}']"""				)
									raise ValueError(
									    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__lowerCAmelCase				).__name__}."""				)
						if i == 0:
									lowerCAmelCase_	,  lowerCAmelCase_						=	(
									    (Dataset, IterableDataset) if isinstance(__lowerCAmelCase    ,							__lowerCAmelCase				) else (IterableDataset, Dataset)
									)
						elif not isinstance(__lowerCAmelCase    ,							__lowerCAmelCase				):
									raise ValueError(
									    F"""Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."""				)
			if stopping_strategy not in ["first_exhausted", "all_exhausted"]:
						raise ValueError(F"""{stopping_strategy} is not supported. Please enter a valid stopping_strategy."""				)
			if dataset_type is Dataset:
						return _interleave_map_style_datasets(
						    __lowerCAmelCase    ,							__lowerCAmelCase    ,							__lowerCAmelCase    ,							info=__lowerCAmelCase    ,							split=__lowerCAmelCase    ,							stopping_strategy=__lowerCAmelCase				)
			else:
						return _interleave_iterable_datasets(
						    __lowerCAmelCase    ,							__lowerCAmelCase    ,							__lowerCAmelCase    ,							info=__lowerCAmelCase    ,							split=__lowerCAmelCase    ,							stopping_strategy=__lowerCAmelCase				)






def    lowerCamelCase__				(				__lowerCAmelCase  :  List[DatasetType]    ,							__lowerCAmelCase  :  Optional[DatasetInfo] = None    ,							__lowerCAmelCase  :  Optional[NamedSplit] = None    ,							__lowerCAmelCase  :  int = 0    ,							):




			"""simple docstring"""


			if not dsets:
						raise ValueError("Unable to concatenate an empty list of datasets."				)
			for i, dataset in enumerate(__lowerCAmelCase				):
						if not isinstance(__lowerCAmelCase    ,							(Dataset, IterableDataset)				):
									if isinstance(__lowerCAmelCase    ,							(DatasetDict, IterableDatasetDict)				):
												if not dataset:
															raise ValueError(
															    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} """
															    "is an empty dataset dictionary."				)
												raise ValueError(
												    F"""Dataset at position {i} has at least one split: {list(__lowerCAmelCase				)}\n"""
												    F"""Please pick one to interleave with the other datasets, for example: dataset['{next(iter(__lowerCAmelCase				)				)}']"""				)
									raise ValueError(
									    F"""Expected a list of Dataset objects or a list of IterableDataset objects, but element at position {i} is a {type(__lowerCAmelCase				).__name__}."""				)
						if i == 0:
									lowerCAmelCase_	,  lowerCAmelCase_						=	(
									    (Dataset, IterableDataset) if isinstance(__lowerCAmelCase    ,							__lowerCAmelCase				) else (IterableDataset, Dataset)
									)
						elif not isinstance(__lowerCAmelCase    ,							__lowerCAmelCase				):
									raise ValueError(
									    F"""Unable to interleave a {dataset_type.__name__} (at position 0) with a {other_type.__name__} (at position {i}). Expected a list of Dataset objects or a list of IterableDataset objects."""				)
			if dataset_type is Dataset:
						return _concatenate_map_style_datasets(__lowerCAmelCase    ,							info=__lowerCAmelCase    ,							split=__lowerCAmelCase    ,							axis=__lowerCAmelCase				)
			else:
						return _concatenate_iterable_datasets(__lowerCAmelCase    ,							info=__lowerCAmelCase    ,							split=__lowerCAmelCase    ,							axis=__lowerCAmelCase				)



