
import logging
import os
from dataclasses import dataclass, field
from typing import Dict, List, Optional

import numpy as np
import torch
from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import is_main_process


snake_case				:	List[Any]							   =						logging.getLogger(__name__)





@dataclass
class    _snake_case      :
			UpperCamelCase__					   =					field(
			    metadata={'help': 'Path to pretrained model or model identifier from huggingface.co/models'}       )
			UpperCamelCase__					   =					field(
			    default=snake_case       ,		metadata={'help': 'Pretrained config name or path if not the same as model_name'}       )
			UpperCamelCase__					   =					field(
			    default=snake_case       ,		metadata={'help': 'Pretrained tokenizer name or path if not the same as model_name'}       )
			UpperCamelCase__					   =					field(
			    default=snake_case       ,		metadata={'help': 'Where do you want to store the pretrained models downloaded from huggingface.co'}       ,		)







@dataclass
class    _snake_case      :
			UpperCamelCase__					   =					field(
			    metadata={'help': 'The name of the task to train selected in the list: ' + ', '.join(hans_processors.keys()       )}       )
			UpperCamelCase__					   =					field(
			    metadata={'help': 'The input data dir. Should contain the .tsv files (or other data files) for the task.'}       )
			UpperCamelCase__					   =					field(
			    default=128       ,		metadata={
			        'help': (
			            'The maximum total input sequence length after tokenization. Sequences longer '
			            'than this will be truncated, sequences shorter will be padded.'
			        )
			    }       ,		)
			UpperCamelCase__					   =					field(
			    default=snake_case       ,		metadata={'help': 'Overwrite the cached training and evaluation sets'}       )







def 						lowerCAmelCase_		(      _snake_case   :						List[InputFeatures]     )       ->	Dict[str, torch.Tensor]:







	'''simple docstring'''


	__magic_name__	:	Any			     =  default_data_collator(_snake_case     )
	__magic_name__	:	str			     =  batch.pop("pairID"   ,     _snake_case     )
	return batch






def 						lowerCAmelCase_		(      )       ->	Union[str, Any]:







	'''simple docstring'''


	__magic_name__	:	int			     =  HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )
	__magic_name__     ,					__magic_name__     ,					__magic_name__	:	List[Any]			     =  parser.parse_args_into_dataclasses()

	if (
	    os.path.exists(training_args.output_dir     )
	    and os.listdir(training_args.output_dir     )
	    and training_args.do_train
	    and not training_args.overwrite_output_dir
	):
		raise ValueError(
		    F'''Output directory ({training_args.output_dir}) already exists and is not empty. Use'''
		    " --overwrite_output_dir to overcome."     )

	# Setup logging
	logging.basicConfig(
	    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s"   ,     datefmt="%m/%d/%Y %H:%M:%S"   ,     level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN   ,     )
	logger.warning(
	    "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s"   ,     training_args.local_rank   ,     training_args.device   ,     training_args.n_gpu   ,     bool(training_args.local_rank != -1     )   ,     training_args.fpaa   ,     )
	# Set the verbosity to info of the Transformers logger (on main process only):
	if is_main_process(training_args.local_rank     ):
		transformers.utils.logging.set_verbosity_info()
		transformers.utils.logging.enable_default_handler()
		transformers.utils.logging.enable_explicit_format()
	logger.info("Training/evaluation parameters %s"   ,     _snake_case     )

	# Set seed
	set_seed(training_args.seed     )

	try:
		__magic_name__	:	Any			     =  hans_tasks_num_labels[data_args.task_name]
	except KeyError:
		raise ValueError("Task not found: %s" % (data_args.task_name)     )

	# Load pretrained model and tokenizer
	#
	# Distributed training:
	# The .from_pretrained methods guarantee that only one local process can concurrently
	# download model & vocab.

	__magic_name__	:	Dict			     =  AutoConfig.from_pretrained(
	    model_args.config_name if model_args.config_name else model_args.model_name_or_path   ,     num_labels=_snake_case   ,     finetuning_task=data_args.task_name   ,     cache_dir=model_args.cache_dir   ,     )
	__magic_name__	:	List[Any]			     =  AutoTokenizer.from_pretrained(
	    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path   ,     cache_dir=model_args.cache_dir   ,     )
	__magic_name__	:	Any			     =  AutoModelForSequenceClassification.from_pretrained(
	    model_args.model_name_or_path   ,     from_tf=bool(".ckpt" in model_args.model_name_or_path     )   ,     config=_snake_case   ,     cache_dir=model_args.cache_dir   ,     )

	# Get datasets
	__magic_name__	:	int			     =  (
	    HansDataset(
	        data_dir=data_args.data_dir   ,     tokenizer=_snake_case   ,     task=data_args.task_name   ,     max_seq_length=data_args.max_seq_length   ,     overwrite_cache=data_args.overwrite_cache   ,     )
	    if training_args.do_train
	    else None
	)
	__magic_name__	:	str			     =  (
	    HansDataset(
	        data_dir=data_args.data_dir   ,     tokenizer=_snake_case   ,     task=data_args.task_name   ,     max_seq_length=data_args.max_seq_length   ,     overwrite_cache=data_args.overwrite_cache   ,     evaluate=_snake_case   ,     )
	    if training_args.do_eval
	    else None
	)

	# Initialize our Trainer
	__magic_name__	:	str			     =  Trainer(
	    model=_snake_case   ,     args=_snake_case   ,     train_dataset=_snake_case   ,     eval_dataset=_snake_case   ,     data_collator=_snake_case   ,     )

	# Training
	if training_args.do_train:
		trainer.train(
		    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
		trainer.save_model()
		# For convenience, we also re-save the tokenizer to the same directory,
		# so that you can share your model easily on huggingface.co/models =)
		if trainer.is_world_master():
			tokenizer.save_pretrained(training_args.output_dir     )

    # Evaluation
	if training_args.do_eval:
		logger.info("*** Evaluate ***"     )

		__magic_name__	:	Union[str, Any]			     =  trainer.predict(_snake_case     )
		__magic_name__	:	Optional[Any]			     =  output.predictions
		__magic_name__	:	str			     =  np.argmax(_snake_case   ,     axis=1     )

		__magic_name__	:	Optional[int]			     =  [ex.pairID for ex in eval_dataset]
		__magic_name__	:	Dict			     =  os.path.join(training_args.output_dir   ,     "hans_predictions.txt"     )
		__magic_name__	:	Tuple			     =  eval_dataset.get_labels()
		if trainer.is_world_master():
			with open(_snake_case   ,     "w"     ) as writer:
				writer.write("pairID,gold_label\n"     )
				for pid, pred in zip(_snake_case   ,     _snake_case     ):
					writer.write("ex" + str(_snake_case     ) + "," + label_list[int(_snake_case     )] + "\n"     )

		trainer._log(output.metrics     )






def 						lowerCAmelCase_		(      _snake_case   :						Union[str, Any]     )       ->	Optional[int]:







	'''simple docstring'''


	main()


if __name__ == "__main__":
				main()


import os
import unittest

from transformers import ElectraTokenizerFast
from transformers.models.electra.tokenization_electra import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    ElectraTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english





@require_tokenizers
class    _snake_case      (  snake_case       ,		unittest.TestCase       ):
			UpperCamelCase__					   =					ElectraTokenizer
			UpperCamelCase__					   =					ElectraTokenizerFast
			UpperCamelCase__					   =					True
			UpperCamelCase__					   =					True
			UpperCamelCase__					   =					filter_non_english







			def     SCREAMING_SNAKE_CASE					(       self	):
				super().setUp()

				__magic_name__	:	Optional[Any]			     =  [
				    "[UNK]",
				    "[CLS]",
				    "[SEP]",
				    "[PAD]",
				    "[MASK]",
				    "want",
				    "##want",
				    "##ed",
				    "wa",
				    "un",
				    "runn",
				    "##ing",
				    ",",
				    "low",
				    "lowest",
				]
				__magic_name__	:	Dict			     =  os.path.join(self.tmpdirname		,     VOCAB_FILES_NAMES["vocab_file"]	)
				with open(self.vocab_file		,     "w"		,     encoding="utf-8"	) as vocab_writer:
					vocab_writer.write("".join([x + "\n" for x in vocab_tokens]	)	)







			def     SCREAMING_SNAKE_CASE					(       self		,     _a	):
				__magic_name__	:	Dict			     =  "UNwant\u00E9d,running"
				__magic_name__	:	int			     =  "unwanted, running"
				return input_text, output_text







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Union[str, Any]			     =  self.tokenizer_class(self.vocab_file	)

				__magic_name__	:	Optional[int]			     =  tokenizer.tokenize("UNwant\u00E9d,running"	)
				self.assertListEqual(_a		,     ["un", "##want", "##ed", ",", "runn", "##ing"]	)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_a	)		,     [9, 6, 7, 12, 10, 11]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				if not self.test_rust_tokenizer:
					return

				__magic_name__	:	str			     =  self.get_tokenizer()
				__magic_name__	:	Optional[int]			     =  self.get_rust_tokenizer()

				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"

				__magic_name__	:	List[Any]			     =  tokenizer.tokenize(_a	)
				__magic_name__	:	str			     =  rust_tokenizer.tokenize(_a	)
				self.assertListEqual(_a		,     _a	)

				__magic_name__	:	int			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
				__magic_name__	:	Union[str, Any]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
				self.assertListEqual(_a		,     _a	)

				__magic_name__	:	Tuple			     =  self.get_rust_tokenizer()
				__magic_name__	:	Union[str, Any]			     =  tokenizer.encode(_a	)
				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
				self.assertListEqual(_a		,     _a	)

				# With lower casing
				__magic_name__	:	Optional[Any]			     =  self.get_tokenizer(do_lower_case=_a	)
				__magic_name__	:	Union[str, Any]			     =  self.get_rust_tokenizer(do_lower_case=_a	)

				__magic_name__	:	Tuple			     =  "UNwant\u00E9d,running"

				__magic_name__	:	Optional[Any]			     =  tokenizer.tokenize(_a	)
				__magic_name__	:	List[Any]			     =  rust_tokenizer.tokenize(_a	)
				self.assertListEqual(_a		,     _a	)

				__magic_name__	:	Any			     =  tokenizer.encode(_a		,     add_special_tokens=_a	)
				__magic_name__	:	List[str]			     =  rust_tokenizer.encode(_a		,     add_special_tokens=_a	)
				self.assertListEqual(_a		,     _a	)

				__magic_name__	:	Dict			     =  self.get_rust_tokenizer()
				__magic_name__	:	Optional[int]			     =  tokenizer.encode(_a	)
				__magic_name__	:	Any			     =  rust_tokenizer.encode(_a	)
				self.assertListEqual(_a		,     _a	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Tuple			     =  BasicTokenizer()

				self.assertListEqual(tokenizer.tokenize("ah\u535A\u63A8zz"	)		,     ["ah", "\u535A", "\u63A8", "zz"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Optional[Any]			     =  BasicTokenizer(do_lower_case=_a	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["hello", "!", "how", "are", "you", "?"]	)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hällo", "!", "how", "are", "you", "?"]	)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["h\u00E9llo"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	str			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Optional[int]			     =  BasicTokenizer(do_lower_case=_a	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["hallo", "!", "how", "are", "you", "?"]	)
				self.assertListEqual(tokenizer.tokenize("H\u00E9llo"	)		,     ["hello"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Union[str, Any]			     =  BasicTokenizer(do_lower_case=_a	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU?  "	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Dict			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HäLLo", "!", "how", "Are", "yoU", "?"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     strip_accents=_a	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHäLLo!how  \n Are yoU?  "	)		,     ["HaLLo", "!", "how", "Are", "yoU", "?"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Any			     =  BasicTokenizer(do_lower_case=_a		,     never_split=["[UNK]"]	)

				self.assertListEqual(
				    tokenizer.tokenize(" \tHeLLo!how  \n Are yoU? [UNK]"	)		,     ["HeLLo", "!", "how", "Are", "yoU", "?", "[UNK]"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	Dict			     =  ["[UNK]", "[CLS]", "[SEP]", "want", "##want", "##ed", "wa", "un", "runn", "##ing"]

				__magic_name__	:	Optional[int]			     =  {}
				for i, token in enumerate(_a	):
					__magic_name__	:	Tuple			     =  i
				__magic_name__	:	Tuple			     =  WordpieceTokenizer(vocab=_a		,     unk_token="[UNK]"	)

				self.assertListEqual(tokenizer.tokenize(""	)		,     []	)

				self.assertListEqual(tokenizer.tokenize("unwanted running"	)		,     ["un", "##want", "##ed", "runn", "##ing"]	)

				self.assertListEqual(tokenizer.tokenize("unwantedX running"	)		,     ["[UNK]", "runn", "##ing"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				self.assertTrue(_is_whitespace(" "	)	)
				self.assertTrue(_is_whitespace("\t"	)	)
				self.assertTrue(_is_whitespace("\r"	)	)
				self.assertTrue(_is_whitespace("\n"	)	)
				self.assertTrue(_is_whitespace("\u00A0"	)	)

				self.assertFalse(_is_whitespace("A"	)	)
				self.assertFalse(_is_whitespace("-"	)	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				self.assertTrue(_is_control("\u0005"	)	)

				self.assertFalse(_is_control("A"	)	)
				self.assertFalse(_is_control(" "	)	)
				self.assertFalse(_is_control("\t"	)	)
				self.assertFalse(_is_control("\r"	)	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				self.assertTrue(_is_punctuation("-"	)	)
				self.assertTrue(_is_punctuation("$"	)	)
				self.assertTrue(_is_punctuation("`"	)	)
				self.assertTrue(_is_punctuation("."	)	)

				self.assertFalse(_is_punctuation("A"	)	)
				self.assertFalse(_is_punctuation(" "	)	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	List[Any]			     =  self.get_tokenizer()
				__magic_name__	:	List[str]			     =  self.get_rust_tokenizer()

				# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
				self.assertListEqual([tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)

				self.assertListEqual(
				    [rust_tokenizer.tokenize(_a	) for t in ["Test", "\xad", "test"]]		,     [["[UNK]"], [], ["[UNK]"]]	)







			@slow
			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	int			     =  self.tokenizer_class.from_pretrained("google/electra-base-discriminator"	)

				__magic_name__	:	List[Any]			     =  tokenizer.encode("sequence builders"		,     add_special_tokens=_a	)
				__magic_name__	:	List[Any]			     =  tokenizer.encode("multi-sequence build"		,     add_special_tokens=_a	)

				__magic_name__	:	Optional[int]			     =  tokenizer.build_inputs_with_special_tokens(_a	)
				__magic_name__	:	Tuple			     =  tokenizer.build_inputs_with_special_tokens(_a		,     _a	)

				assert encoded_sentence == [101] + text + [102]
				assert encoded_pair == [101] + text + [102] + text_a + [102]







			def     SCREAMING_SNAKE_CASE					(       self	):
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
						__magic_name__	:	str			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)

						__magic_name__	:	Optional[Any]			     =  f'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
						__magic_name__	:	Any			     =  tokenizer_r.encode_plus(
						    _a		,     return_attention_mask=_a		,     return_token_type_ids=_a		,     return_offsets_mapping=_a		,     add_special_tokens=_a		,     )

						__magic_name__	:	Any			     =  tokenizer_r.do_lower_case if hasattr(_a		,     "do_lower_case"	) else False
						__magic_name__	:	int			     =  (
						    [
						        ((0, 0), tokenizer_r.cls_token),
						        ((0, 1), "A"),
						        ((1, 2), ","),
						        ((3, 5), "na"),
						        ((5, 6), "##ï"),
						        ((6, 8), "##ve"),
						        ((9, 15), tokenizer_r.mask_token),
						        ((16, 21), "Allen"),
						        ((21, 23), "##NL"),
						        ((23, 24), "##P"),
						        ((25, 33), "sentence"),
						        ((33, 34), "."),
						        ((0, 0), tokenizer_r.sep_token),
						    ]
						    if not do_lower_case
						    else [
						        ((0, 0), tokenizer_r.cls_token),
						        ((0, 1), "a"),
						        ((1, 2), ","),
						        ((3, 8), "naive"),
						        ((9, 15), tokenizer_r.mask_token),
						        ((16, 21), "allen"),
						        ((21, 23), "##nl"),
						        ((23, 24), "##p"),
						        ((25, 33), "sentence"),
						        ((33, 34), "."),
						        ((0, 0), tokenizer_r.sep_token),
						    ]
						)

						self.assertEqual(
						    [e[1] for e in expected_results]		,     tokenizer_r.convert_ids_to_tokens(tokens["input_ids"]	)	)
						self.assertEqual([e[0] for e in expected_results]		,     tokens["offset_mapping"]	)







			def     SCREAMING_SNAKE_CASE					(       self	):
				__magic_name__	:	str			     =  ["的", "人", "有"]
				__magic_name__	:	List[str]			     =  "".join(_a	)
				for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
					with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''	):
						__magic_name__	:	Optional[int]			     =  True
						__magic_name__	:	Optional[int]			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)
						__magic_name__	:	Union[str, Any]			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)

						__magic_name__	:	Tuple			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)
						__magic_name__	:	List[Any]			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)

						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)

						# it is expected that each Chinese character is not preceded by "##"
						self.assertListEqual(_a		,     _a	)
						self.assertListEqual(_a		,     _a	)

						__magic_name__	:	Tuple			     =  False
						__magic_name__	:	Tuple			     =  self.rust_tokenizer_class.from_pretrained(_a		,     **_a	)
						__magic_name__	:	str			     =  self.tokenizer_class.from_pretrained(_a		,     **_a	)

						__magic_name__	:	Dict			     =  tokenizer_r.encode(_a		,     add_special_tokens=_a	)
						__magic_name__	:	Dict			     =  tokenizer_p.encode(_a		,     add_special_tokens=_a	)

						__magic_name__	:	List[str]			     =  tokenizer_r.convert_ids_to_tokens(_a	)
						__magic_name__	:	List[Any]			     =  tokenizer_p.convert_ids_to_tokens(_a	)

						# it is expected that only the first Chinese character is not preceded by "##".
						__magic_name__	:	Tuple			     =  [
						    f'''##{token}''' if idx != 0 else token for idx, token in enumerate(_a	)
						]
						self.assertListEqual(_a		,     _a	)
						self.assertListEqual(_a		,     _a	)
