
import argparse
import ast
import logging
import os
import sys

import pandas as pd
import torch
from tqdm import tqdm

from transformers import BartForConditionalGeneration, RagRetriever, RagSequenceForGeneration, RagTokenForGeneration
from transformers import logging as transformers_logging


sys.path.append(os.path.join(os.getcwd()))  # noqa: E402 # isort:skip
from utils_rag import exact_match_score, fa_score  # noqa: E402 # isort:skip


lowerCamelCase__          =		logging.getLogger(__name__)
logging.basicConfig(level=logging.INFO)

transformers_logging.set_verbosity_info()
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Any:
			if "token" in model_name_or_path:
						return "rag_token"
			if "sequence" in model_name_or_path:
						return "rag_sequence"
			if "bart" in model_name_or_path:
						return "bart"
			return None
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		List[Any]:
			return max(metric_fn(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ) for gt in ground_truths     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		str:
			lowerCAmelCase__      :      str         =		[line.strip() for line in open(SCREAMING_SNAKE_CASE_       ,							'r'     ).readlines()]
			lowerCAmelCase__      :      Union[str, Any]         =		[]

			if args.gold_data_mode == "qa":
						lowerCAmelCase__      :      str         =		pd.read_csv(SCREAMING_SNAKE_CASE_       ,							sep='\t'       ,							header=SCREAMING_SNAKE_CASE_     )
						for answer_list in data[1]:
									lowerCAmelCase__      :      Any         =		ast.literal_eval(SCREAMING_SNAKE_CASE_     )
									answers.append(SCREAMING_SNAKE_CASE_     )
			else:
						lowerCAmelCase__      :      Any         =		[line.strip() for line in open(SCREAMING_SNAKE_CASE_       ,							'r'     ).readlines()]
						lowerCAmelCase__      :      Any         =		[[reference] for reference in references]

			lowerCAmelCase__      :      Union[str, Any]         =		0
			for prediction, ground_truths in zip(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ):
						total += 1
						em += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
						fa += metric_max_over_ground_truths(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )

			lowerCAmelCase__      :      str         =		100.0 * em / total
			lowerCAmelCase__      :      Any         =		100.0 * fa / total

			logger.info(F'''F1: {fa:.2f}'''     )
			logger.info(F'''EM: {em:.2f}'''     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Any:
			lowerCAmelCase__      :      str         =		args.k
			lowerCAmelCase__      :      Tuple         =		[line.strip() for line in open(SCREAMING_SNAKE_CASE_       ,							'r'     ).readlines()]
			lowerCAmelCase__      :      Optional[int]         =		[line.strip() for line in open(SCREAMING_SNAKE_CASE_       ,							'r'     ).readlines()]

			lowerCAmelCase__      :      str         =		0
			for hypo, reference in zip(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ):
						lowerCAmelCase__      :      Optional[Any]         =		set(hypo.split('\t'     )[:k]     )
						lowerCAmelCase__      :      Tuple         =		set(reference.split('\t'     )     )
						total += 1
						em += len(hypo_provenance & ref_provenance     ) / k

			lowerCAmelCase__      :      Any         =		100.0 * em / total
			logger.info(F'''Precision@{k}: {em: .2f}'''     )
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		Optional[int]:
			def strip_title(SCREAMING_SNAKE_CASE_     ):
						if title.startswith('"'     ):
									lowerCAmelCase__      :      Union[str, Any]         =		title[1:]
						if title.endswith('"'     ):
									lowerCAmelCase__      :      int         =		title[:-1]
						return title

			lowerCAmelCase__      :      Union[str, Any]         =		rag_model.retriever.question_encoder_tokenizer.batch_encode_plus(
			    SCREAMING_SNAKE_CASE_       ,							return_tensors='pt'       ,							padding=SCREAMING_SNAKE_CASE_       ,							truncation=SCREAMING_SNAKE_CASE_       ,							)['input_ids'].to(args.device     )

			lowerCAmelCase__      :      Optional[int]         =		rag_model.rag.question_encoder(SCREAMING_SNAKE_CASE_     )
			lowerCAmelCase__      :      Optional[Any]         =		question_enc_outputs[0]

			lowerCAmelCase__      :      Union[str, Any]         =		rag_model.retriever(
			    SCREAMING_SNAKE_CASE_       ,							question_enc_pool_output.cpu().detach().to(torch.floataa     ).numpy()       ,							prefix=rag_model.rag.generator.config.prefix       ,							n_docs=rag_model.config.n_docs       ,							return_tensors='pt'       ,							)
			lowerCAmelCase__      :      Dict         =		rag_model.retriever.index.get_doc_dicts(result.doc_ids     )
			lowerCAmelCase__      :      List[str]         =		[]
			for docs in all_docs:
						lowerCAmelCase__      :      Any         =		[strip_title(SCREAMING_SNAKE_CASE_     ) for title in docs['title']]
						provenance_strings.append('\t'.join(SCREAMING_SNAKE_CASE_     )     )
			return provenance_strings
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )				->		List[str]:
			with torch.no_grad():
						lowerCAmelCase__      :      List[str]         =		rag_model.retriever.question_encoder_tokenizer.batch_encode_plus(
						    SCREAMING_SNAKE_CASE_       ,							return_tensors='pt'       ,							padding=SCREAMING_SNAKE_CASE_       ,							truncation=SCREAMING_SNAKE_CASE_     )

						lowerCAmelCase__      :      int         =		inputs_dict.input_ids.to(args.device     )
						lowerCAmelCase__      :      Union[str, Any]         =		inputs_dict.attention_mask.to(args.device     )
						lowerCAmelCase__      :      int         =		rag_model.generate(  # rag_model overwrites generate
						    SCREAMING_SNAKE_CASE_       ,							attention_mask=SCREAMING_SNAKE_CASE_       ,							num_beams=args.num_beams       ,							min_length=args.min_length       ,							max_length=args.max_length       ,							early_stopping=SCREAMING_SNAKE_CASE_       ,							num_return_sequences=1       ,							bad_words_ids=[[0, 0]]       ,							)
						lowerCAmelCase__      :      int         =		rag_model.retriever.generator_tokenizer.batch_decode(SCREAMING_SNAKE_CASE_       ,							skip_special_tokens=SCREAMING_SNAKE_CASE_     )

						if args.print_predictions:
									for q, a in zip(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     ):
												logger.info('Q: {} - A: {}'.format(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )     )

						return answers
def  lowerCAmelCase__			(  )				->		Any:
			lowerCAmelCase__      :      Optional[Any]         =		argparse.ArgumentParser()
			parser.add_argument(
			    '--model_type'       ,							choices=['rag_sequence', 'rag_token', 'bart']       ,							type=SCREAMING_SNAKE_CASE_       ,							help=(
			        'RAG model type: rag_sequence, rag_token or bart, if none specified, the type is inferred from the'
			        ' model_name_or_path'
			    )       ,							)
			parser.add_argument(
			    '--index_name'       ,							default=SCREAMING_SNAKE_CASE_       ,							choices=['exact', 'compressed', 'legacy']       ,							type=SCREAMING_SNAKE_CASE_       ,							help='RAG model retriever type'       ,							)
			parser.add_argument(
			    '--index_path'       ,							default=SCREAMING_SNAKE_CASE_       ,							type=SCREAMING_SNAKE_CASE_       ,							help='Path to the retrieval index'       ,							)
			parser.add_argument('--n_docs'       ,							default=5       ,							type=SCREAMING_SNAKE_CASE_       ,							help='Number of retrieved docs'     )
			parser.add_argument(
			    '--model_name_or_path'       ,							default=SCREAMING_SNAKE_CASE_       ,							type=SCREAMING_SNAKE_CASE_       ,							required=SCREAMING_SNAKE_CASE_       ,							help='Path to pretrained checkpoints or model identifier from huggingface.co/models'       ,							)
			parser.add_argument(
			    '--eval_mode'       ,							choices=['e2e', 'retrieval']       ,							default='e2e'       ,							type=SCREAMING_SNAKE_CASE_       ,							help=(
			        'Evaluation mode, e2e calculates exact match and F1 of the downstream task, retrieval calculates'
			        ' precision@k.'
			    )       ,							)
			parser.add_argument('--k'       ,							default=1       ,							type=SCREAMING_SNAKE_CASE_       ,							help='k for the precision@k calculation'     )
			parser.add_argument(
			    '--evaluation_set'       ,							default=SCREAMING_SNAKE_CASE_       ,							type=SCREAMING_SNAKE_CASE_       ,							required=SCREAMING_SNAKE_CASE_       ,							help='Path to a file containing evaluation samples'       ,							)
			parser.add_argument(
			    '--gold_data_path'       ,							default=SCREAMING_SNAKE_CASE_       ,							type=SCREAMING_SNAKE_CASE_       ,							required=SCREAMING_SNAKE_CASE_       ,							help='Path to a tab-separated file with gold samples'       ,							)
			parser.add_argument(
			    '--gold_data_mode'       ,							default='qa'       ,							type=SCREAMING_SNAKE_CASE_       ,							choices=['qa', 'ans']       ,							help=(
			        'Format of the gold data file'
			        'qa - a single line in the following format: question [tab] answer_list'
			        'ans - a single line of the gold file contains the expected answer string'
			    )       ,							)
			parser.add_argument(
			    '--predictions_path'       ,							type=SCREAMING_SNAKE_CASE_       ,							default='predictions.txt'       ,							help='Name of the predictions file, to be stored in the checkpoints directory'       ,							)
			parser.add_argument(
			    '--eval_all_checkpoints'       ,							action='store_true'       ,							help='Evaluate all checkpoints starting with the same prefix as model_name ending and ending with step number'       ,							)
			parser.add_argument(
			    '--eval_batch_size'       ,							default=8       ,							type=SCREAMING_SNAKE_CASE_       ,							help='Batch size per GPU/CPU for evaluation.'       ,							)
			parser.add_argument(
			    '--recalculate'       ,							help='Recalculate predictions even if the prediction file exists'       ,							action='store_true'       ,							)
			parser.add_argument(
			    '--num_beams'       ,							default=4       ,							type=SCREAMING_SNAKE_CASE_       ,							help='Number of beams to be used when generating answers'       ,							)
			parser.add_argument('--min_length'       ,							default=1       ,							type=SCREAMING_SNAKE_CASE_       ,							help='Min length of the generated answers'     )
			parser.add_argument('--max_length'       ,							default=50       ,							type=SCREAMING_SNAKE_CASE_       ,							help='Max length of the generated answers'     )

			parser.add_argument(
			    '--print_predictions'       ,							action='store_true'       ,							help='If True, prints predictions while evaluating.'       ,							)
			parser.add_argument(
			    '--print_docs'       ,							action='store_true'       ,							help='If True, prints docs retried while generating.'       ,							)
			lowerCAmelCase__      :      Union[str, Any]         =		parser.parse_args()
			lowerCAmelCase__      :      List[Any]         =		torch.device('cuda' if torch.cuda.is_available() else 'cpu'     )
			return args
def  lowerCAmelCase__			(  SCREAMING_SNAKE_CASE_     )				->		Dict:
			lowerCAmelCase__      :      int         =		{}
			if args.model_type is None:
						lowerCAmelCase__      :      Any         =		infer_model_type(args.model_name_or_path     )
						assert args.model_type is not None
			if args.model_type.startswith('rag'     ):
						lowerCAmelCase__      :      List[str]         =		RagTokenForGeneration if args.model_type == 'rag_token' else RagSequenceForGeneration
						lowerCAmelCase__      :      Any         =		args.n_docs
						if args.index_name is not None:
									lowerCAmelCase__      :      Tuple         =		args.index_name
						if args.index_path is not None:
									lowerCAmelCase__      :      Tuple         =		args.index_path
			else:
						lowerCAmelCase__      :      Tuple         =		BartForConditionalGeneration

			lowerCAmelCase__      :      int         =		(
			    [f.path for f in os.scandir(args.model_name_or_path     ) if f.is_dir()]
			    if args.eval_all_checkpoints
			    else [args.model_name_or_path]
			)

			logger.info('Evaluate the following checkpoints: %s'       ,							SCREAMING_SNAKE_CASE_     )

			lowerCAmelCase__      :      List[str]         =		get_scores if args.eval_mode == 'e2e' else get_precision_at_k
			lowerCAmelCase__      :      Any         =		evaluate_batch_eae if args.eval_mode == 'e2e' else evaluate_batch_retrieval

			for checkpoint in checkpoints:
						if os.path.exists(args.predictions_path     ) and (not args.recalculate):
									logger.info('Calculating metrics based on an existing predictions file: {}'.format(args.predictions_path     )     )
									score_fn(SCREAMING_SNAKE_CASE_       ,							args.predictions_path       ,							args.gold_data_path     )
									continue

						logger.info('***** Running evaluation for {} *****'.format(SCREAMING_SNAKE_CASE_     )     )
						logger.info('  Batch size = %d'       ,							args.eval_batch_size     )
						logger.info('  Predictions will be stored under {}'.format(args.predictions_path     )     )

						if args.model_type.startswith('rag'     ):
									lowerCAmelCase__      :      Any         =		RagRetriever.from_pretrained(SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_     )
									lowerCAmelCase__      :      Optional[int]         =		model_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,							retriever=SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_     )
									model.retriever.init_retrieval()
						else:
									lowerCAmelCase__      :      int         =		model_class.from_pretrained(SCREAMING_SNAKE_CASE_       ,							**SCREAMING_SNAKE_CASE_     )
						model.to(args.device     )

						with open(args.evaluation_set       ,							'r'     ) as eval_file, open(args.predictions_path       ,							'w'     ) as preds_file:
									lowerCAmelCase__      :      Any         =		[]
									for line in tqdm(SCREAMING_SNAKE_CASE_     ):
												questions.append(line.strip()     )
												if len(SCREAMING_SNAKE_CASE_     ) == args.eval_batch_size:
															lowerCAmelCase__      :      Tuple         =		evaluate_batch_fn(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
															preds_file.write('\n'.join(SCREAMING_SNAKE_CASE_     ) + '\n'     )
															preds_file.flush()
															lowerCAmelCase__      :      Dict         =		[]
									if len(SCREAMING_SNAKE_CASE_     ) > 0:
												lowerCAmelCase__      :      List[str]         =		evaluate_batch_fn(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_     )
												preds_file.write('\n'.join(SCREAMING_SNAKE_CASE_     )     )
												preds_file.flush()

									score_fn(SCREAMING_SNAKE_CASE_       ,							args.predictions_path       ,							args.gold_data_path     )


if __name__ == "__main__":
					lowerCamelCase__          =		get_args()
					main(args)

import os
import tempfile
import unittest

from transformers import NezhaConfig, is_torch_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, require_torch_gpu, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import (
					    MODEL_FOR_PRETRAINING_MAPPING,
					    NezhaForMaskedLM,
					    NezhaForMultipleChoice,
					    NezhaForNextSentencePrediction,
					    NezhaForPreTraining,
					    NezhaForQuestionAnswering,
					    NezhaForSequenceClassification,
					    NezhaForTokenClassification,
					    NezhaModel,
					)
					from transformers.models.nezha.modeling_nezha import NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST







class      A__							:






				def __init__(						self					:      str    ,     a					:      Optional[Any]    ,     a					:      int=13    ,     a					:      str=7    ,     a					:      str=True    ,     a					:      List[str]=True    ,     a					:      Optional[Any]=True    ,     a					:      int=True    ,     a					:      List[Any]=99    ,     a					:      List[Any]=32    ,     a					:      Tuple=5    ,     a					:      Any=4    ,     a					:      Optional[int]=37    ,     a					:      Tuple="gelu"    ,     a					:      Any=0.1    ,     a					:      int=0.1    ,     a					:      List[Any]=128    ,     a					:      Union[str, Any]=32    ,     a					:      Union[str, Any]=16    ,     a					:      Dict=2    ,     a					:      List[Any]=0.0_2    ,     a					:      Optional[Any]=3    ,     a					:      List[Any]=4    ,     a					:      Optional[int]=None    ,     ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		parent
							lowerCAmelCase__      :      Dict         =		batch_size
							lowerCAmelCase__      :      Optional[Any]         =		seq_length
							lowerCAmelCase__      :      Optional[Any]         =		is_training
							lowerCAmelCase__      :      Union[str, Any]         =		use_input_mask
							lowerCAmelCase__      :      List[Any]         =		use_token_type_ids
							lowerCAmelCase__      :      str         =		use_labels
							lowerCAmelCase__      :      Optional[Any]         =		vocab_size
							lowerCAmelCase__      :      Union[str, Any]         =		hidden_size
							lowerCAmelCase__      :      Optional[int]         =		num_hidden_layers
							lowerCAmelCase__      :      Optional[int]         =		num_attention_heads
							lowerCAmelCase__      :      List[Any]         =		intermediate_size
							lowerCAmelCase__      :      List[str]         =		hidden_act
							lowerCAmelCase__      :      List[Any]         =		hidden_dropout_prob
							lowerCAmelCase__      :      Optional[int]         =		attention_probs_dropout_prob
							lowerCAmelCase__      :      Dict         =		max_position_embeddings
							lowerCAmelCase__      :      Any         =		type_vocab_size
							lowerCAmelCase__      :      Any         =		type_sequence_label_size
							lowerCAmelCase__      :      List[Any]         =		initializer_range
							lowerCAmelCase__      :      Dict         =		num_labels
							lowerCAmelCase__      :      Any         =		num_choices
							lowerCAmelCase__      :      Union[str, Any]         =		scope






				def 							_lowerCamelCase		(						self					:      Dict   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		ids_tensor([self.batch_size, self.seq_length]    ,     self.vocab_size   )

							lowerCAmelCase__      :      Tuple         =		None
							if self.use_input_mask:
										lowerCAmelCase__      :      Tuple         =		random_attention_mask([self.batch_size, self.seq_length]   )

							lowerCAmelCase__      :      Tuple         =		None
							if self.use_token_type_ids:
										lowerCAmelCase__      :      List[Any]         =		ids_tensor([self.batch_size, self.seq_length]    ,     self.type_vocab_size   )

							lowerCAmelCase__      :      Optional[int]         =		None
							lowerCAmelCase__      :      Optional[Any]         =		None
							lowerCAmelCase__      :      Optional[int]         =		None
							if self.use_labels:
										lowerCAmelCase__      :      Any         =		ids_tensor([self.batch_size]    ,     self.type_sequence_label_size   )
										lowerCAmelCase__      :      Union[str, Any]         =		ids_tensor([self.batch_size, self.seq_length]    ,     self.num_labels   )
										lowerCAmelCase__      :      List[Any]         =		ids_tensor([self.batch_size]    ,     self.num_choices   )

							lowerCAmelCase__      :      Dict         =		self.get_config()

							return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels






				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							return NezhaConfig(
							    vocab_size=self.vocab_size    ,     hidden_size=self.hidden_size    ,     num_hidden_layers=self.num_hidden_layers    ,     num_attention_heads=self.num_attention_heads    ,     intermediate_size=self.intermediate_size    ,     hidden_act=self.hidden_act    ,     hidden_dropout_prob=self.hidden_dropout_prob    ,     attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,     max_position_embeddings=self.max_position_embeddings    ,     type_vocab_size=self.type_vocab_size    ,     is_decoder=a    ,     initializer_range=self.initializer_range    ,     )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							(
							    (
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				
							)      :      List[Any]         =		self.prepare_config_and_inputs()

							lowerCAmelCase__      :      List[Any]         =		True
							lowerCAmelCase__      :      Tuple         =		floats_tensor([self.batch_size, self.seq_length, self.hidden_size]   )
							lowerCAmelCase__      :      Tuple         =		ids_tensor([self.batch_size, self.seq_length]    ,     vocab_size=2   )

							return (
							    config,
							    input_ids,
							    token_type_ids,
							    input_mask,
							    sequence_labels,
							    token_labels,
							    choice_labels,
							    encoder_hidden_states,
							    encoder_attention_mask,
							)






				def 							_lowerCamelCase		(						self					:      Optional[Any]    ,     a					:      Optional[int]    ,     a					:      Tuple    ,     a					:      Optional[int]    ,     a					:      List[Any]    ,     a					:      Tuple    ,     a					:      List[str]    ,     a					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		NezhaModel(config=a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      Dict         =		model(a    ,     attention_mask=a    ,     token_type_ids=a   )
							lowerCAmelCase__      :      List[str]         =		model(a    ,     token_type_ids=a   )
							lowerCAmelCase__      :      Any         =		model(a   )
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.hidden_size)   )
							self.parent.assertEqual(result.pooler_output.shape    ,     (self.batch_size, self.hidden_size)   )






				def 							_lowerCamelCase		(						self					:      List[Any]    ,     a					:      Union[str, Any]    ,     a					:      Dict    ,     a					:      List[Any]    ,     a					:      Optional[Any]    ,     a					:      int    ,     a					:      Tuple    ,     a					:      List[Any]    ,     a					:      Tuple    ,     a					:      List[str]    ,     ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		True
							lowerCAmelCase__      :      Optional[int]         =		NezhaModel(a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      Any         =		model(
							    a    ,     attention_mask=a    ,     token_type_ids=a    ,     encoder_hidden_states=a    ,     encoder_attention_mask=a    ,     )
							lowerCAmelCase__      :      Dict         =		model(
							    a    ,     attention_mask=a    ,     token_type_ids=a    ,     encoder_hidden_states=a    ,     )
							lowerCAmelCase__      :      List[str]         =		model(a    ,     attention_mask=a    ,     token_type_ids=a   )
							self.parent.assertEqual(result.last_hidden_state.shape    ,     (self.batch_size, self.seq_length, self.hidden_size)   )
							self.parent.assertEqual(result.pooler_output.shape    ,     (self.batch_size, self.hidden_size)   )






				def 							_lowerCamelCase		(						self					:      Tuple    ,     a					:      Optional[Any]    ,     a					:      List[Any]    ,     a					:      str    ,     a					:      List[str]    ,     a					:      Tuple    ,     a					:      List[Any]    ,     a					:      Optional[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[Any]         =		NezhaForMaskedLM(config=a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      Dict         =		model(a    ,     attention_mask=a    ,     token_type_ids=a    ,     labels=a   )
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.vocab_size)   )






				def 							_lowerCamelCase		(						self					:      List[Any]    ,     a					:      Optional[int]    ,     a					:      List[Any]    ,     a					:      int    ,     a					:      List[str]    ,     a					:      Union[str, Any]    ,     a					:      int    ,     a					:      Any   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[Any]         =		NezhaForNextSentencePrediction(config=a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      str         =		model(
							    a    ,     attention_mask=a    ,     token_type_ids=a    ,     labels=a    ,     )
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, 2)   )






				def 							_lowerCamelCase		(						self					:      int    ,     a					:      Optional[int]    ,     a					:      str    ,     a					:      List[str]    ,     a					:      int    ,     a					:      Dict    ,     a					:      Optional[Any]    ,     a					:      Optional[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Tuple         =		NezhaForPreTraining(config=a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      Optional[int]         =		model(
							    a    ,     attention_mask=a    ,     token_type_ids=a    ,     labels=a    ,     next_sentence_label=a    ,     )
							self.parent.assertEqual(result.prediction_logits.shape    ,     (self.batch_size, self.seq_length, self.vocab_size)   )
							self.parent.assertEqual(result.seq_relationship_logits.shape    ,     (self.batch_size, 2)   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]    ,     a					:      Dict    ,     a					:      List[str]    ,     a					:      Any    ,     a					:      Any    ,     a					:      Union[str, Any]    ,     a					:      Tuple    ,     a					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		NezhaForQuestionAnswering(config=a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      Tuple         =		model(
							    a    ,     attention_mask=a    ,     token_type_ids=a    ,     start_positions=a    ,     end_positions=a    ,     )
							self.parent.assertEqual(result.start_logits.shape    ,     (self.batch_size, self.seq_length)   )
							self.parent.assertEqual(result.end_logits.shape    ,     (self.batch_size, self.seq_length)   )






				def 							_lowerCamelCase		(						self					:      Tuple    ,     a					:      str    ,     a					:      Union[str, Any]    ,     a					:      Tuple    ,     a					:      Optional[Any]    ,     a					:      Dict    ,     a					:      str    ,     a					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		self.num_labels
							lowerCAmelCase__      :      Optional[Any]         =		NezhaForSequenceClassification(a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      Tuple         =		model(a    ,     attention_mask=a    ,     token_type_ids=a    ,     labels=a   )
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_labels)   )






				def 							_lowerCamelCase		(						self					:      List[str]    ,     a					:      Dict    ,     a					:      str    ,     a					:      Optional[Any]    ,     a					:      Optional[int]    ,     a					:      List[str]    ,     a					:      Dict    ,     a					:      str   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Dict         =		self.num_labels
							lowerCAmelCase__      :      str         =		NezhaForTokenClassification(config=a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      Any         =		model(a    ,     attention_mask=a    ,     token_type_ids=a    ,     labels=a   )
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.seq_length, self.num_labels)   )






				def 							_lowerCamelCase		(						self					:      int    ,     a					:      Tuple    ,     a					:      List[Any]    ,     a					:      Tuple    ,     a					:      List[Any]    ,     a					:      Optional[int]    ,     a					:      Optional[int]    ,     a					:      Optional[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[Any]         =		self.num_choices
							lowerCAmelCase__      :      Any         =		NezhaForMultipleChoice(config=a   )
							model.to(a   )
							model.eval()
							lowerCAmelCase__      :      int         =		input_ids.unsqueeze(1   ).expand(-1    ,     self.num_choices    ,     -1   ).contiguous()
							lowerCAmelCase__      :      str         =		token_type_ids.unsqueeze(1   ).expand(-1    ,     self.num_choices    ,     -1   ).contiguous()
							lowerCAmelCase__      :      Optional[Any]         =		input_mask.unsqueeze(1   ).expand(-1    ,     self.num_choices    ,     -1   ).contiguous()
							lowerCAmelCase__      :      Any         =		model(
							    a    ,     attention_mask=a    ,     token_type_ids=a    ,     labels=a    ,     )
							self.parent.assertEqual(result.logits.shape    ,     (self.batch_size, self.num_choices)   )






				def 							_lowerCamelCase		(						self					:      Tuple   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		self.prepare_config_and_inputs()
							(
							    (
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				
							)      :      Any         =		config_and_inputs
							lowerCAmelCase__      :      str         =		{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
							return config, inputs_dict







@require_torch
class      A__							(     __magic_name__						,   __magic_name__						,   __magic_name__						,   unittest.TestCase	):
				lowercase			    =  (
				    (
				        NezhaModel,
				        NezhaForMaskedLM,
				        NezhaForMultipleChoice,
				        NezhaForNextSentencePrediction,
				        NezhaForPreTraining,
				        NezhaForQuestionAnswering,
				        NezhaForSequenceClassification,
				        NezhaForTokenClassification,
				    )
				    if is_torch_available()
				    else ()
				)
				lowercase			    =  (
				    {
				        'feature-extraction': NezhaModel,
				        'fill-mask': NezhaForMaskedLM,
				        'question-answering': NezhaForQuestionAnswering,
				        'text-classification': NezhaForSequenceClassification,
				        'token-classification': NezhaForTokenClassification,
				        'zero-shot': NezhaForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				lowercase			    =  True






				def 							_lowerCamelCase		(						self					:      str    ,     a					:      Tuple    ,     a					:      int    ,     a					:      Dict=False   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		super()._prepare_for_class(a    ,     a    ,     return_labels=a   )

							if return_labels:
										if model_class in get_values(a   ):
													lowerCAmelCase__      :      Tuple         =		torch.zeros(
													    (self.model_tester.batch_size, self.model_tester.seq_length)    ,     dtype=torch.long    ,     device=a   )
													lowerCAmelCase__      :      Optional[Any]         =		torch.zeros(
													    self.model_tester.batch_size    ,     dtype=torch.long    ,     device=a   )
							return inputs_dict






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[int]         =		NezhaModelTester(self   )
							lowerCAmelCase__      :      Optional[int]         =		ConfigTester(self    ,     config_class=a    ,     hidden_size=37   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							self.config_tester.run_common_tests()






				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*a   )






				def 							_lowerCamelCase		(						self					:      List[str]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		self.model_tester.prepare_config_and_inputs_for_decoder()
							self.model_tester.create_and_check_model_as_decoder(*a   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							(
							    (
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				(
							    lowerCAmelCase__
							)      ,				
							)      :      str         =		self.model_tester.prepare_config_and_inputs_for_decoder()

							lowerCAmelCase__      :      str         =		None

							self.model_tester.create_and_check_model_as_decoder(
							    a    ,     a    ,     a    ,     a    ,     a    ,     a    ,     a    ,     a    ,     a    ,     )






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_masked_lm(*a   )






				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Union[str, Any]         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_multiple_choice(*a   )






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[int]         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_next_sequence_prediction(*a   )






				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      List[str]         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_pretraining(*a   )






				def 							_lowerCamelCase		(						self					:      int   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Optional[int]         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*a   )






				def 							_lowerCamelCase		(						self					:      Optional[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      int         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_sequence_classification(*a   )






				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*a   )






				@slow
				def 							_lowerCamelCase		(						self					:      Union[str, Any]   ):






							'''simple docstring'''







							for model_name in NEZHA_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
										lowerCAmelCase__      :      Optional[Any]         =		NezhaModel.from_pretrained(a   )
										self.assertIsNotNone(a   )






				@slow
				@require_torch_gpu
				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      ,				lowerCAmelCase__      :      Optional[int]         =		self.model_tester.prepare_config_and_inputs_for_common()
							for model_class in self.all_model_classes:
										# NezhaForMultipleChoice behaves incorrectly in JIT environments.
										if model_class == NezhaForMultipleChoice:
													return

										lowerCAmelCase__      :      Dict         =		True
										lowerCAmelCase__      :      Any         =		model_class(config=a   )

										lowerCAmelCase__      :      Union[str, Any]         =		self._prepare_for_class(a    ,     a   )
										lowerCAmelCase__      :      int         =		torch.jit.trace(
										    a    ,     (inputs_dict['input_ids'].to('cpu'   ), inputs_dict['attention_mask'].to('cpu'   ))   )

										with tempfile.TemporaryDirectory() as tmp:
													torch.jit.save(a    ,     os.path.join(a    ,     'bert.pt'   )   )
													lowerCAmelCase__      :      Any         =		torch.jit.load(os.path.join(a    ,     'bert.pt'   )    ,     map_location=a   )
													loaded(inputs_dict['input_ids'].to(a   )    ,     inputs_dict['attention_mask'].to(a   )   )







@require_torch
class      A__							(     unittest.TestCase	):






				@slow
				def 							_lowerCamelCase		(						self					:      Optional[int]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      str         =		NezhaModel.from_pretrained('sijunhe/nezha-cn-base'   )
							lowerCAmelCase__      :      Any         =		torch.tensor([[0, 1, 2, 3, 4, 5]]   )
							lowerCAmelCase__      :      Union[str, Any]         =		torch.tensor([[0, 1, 1, 1, 1, 1]]   )
							with torch.no_grad():
										lowerCAmelCase__      :      Optional[int]         =		model(a    ,     attention_mask=a   )[0]
							lowerCAmelCase__      :      Union[str, Any]         =		torch.Size((1, 6, 768)   )
							self.assertEqual(output.shape    ,     a   )
							lowerCAmelCase__      :      Optional[int]         =		torch.tensor([[[0.0_6_8_5, 0.2_4_4_1, 0.1_1_0_2], [0.0_6_0_0, 0.1_9_0_6, 0.1_3_4_9], [0.0_2_2_1, 0.0_8_1_9, 0.0_5_8_6]]]   )

							self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,     a    ,     atol=1E-4   )   )






				@slow
				def 							_lowerCamelCase		(						self					:      List[Any]   ):






							'''simple docstring'''







							lowerCAmelCase__      :      Any         =		NezhaForMaskedLM.from_pretrained('sijunhe/nezha-cn-base'   )
							lowerCAmelCase__      :      Optional[int]         =		torch.tensor([[0, 1, 2, 3, 4, 5]]   )
							lowerCAmelCase__      :      Optional[int]         =		torch.tensor([[1, 1, 1, 1, 1, 1]]   )
							with torch.no_grad():
										lowerCAmelCase__      :      Optional[int]         =		model(a    ,     attention_mask=a   )[0]
							lowerCAmelCase__      :      int         =		torch.Size((1, 6, 21_128)   )
							self.assertEqual(output.shape    ,     a   )
							lowerCAmelCase__      :      List[Any]         =		torch.tensor(
							    [[-2.7_9_3_9, -1.7_9_0_2, -2.2_1_8_9], [-2.8_5_8_5, -1.8_9_0_8, -2.3_7_2_3], [-2.6_4_9_9, -1.7_7_5_0, -2.2_5_5_8]]   )

							self.assertTrue(torch.allclose(output[:, 1:4, 1:4]    ,     a    ,     atol=1E-4   )   )