
import unittest

from transformers import BertGenerationConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import BertGenerationDecoder, BertGenerationEncoder







class        A_   :




	'''simple docstring'''


	def __init__(       self		,    snake_case		,    snake_case=13		,    snake_case=7		,    snake_case=True		,    snake_case=True		,    snake_case=99		,    snake_case=32		,    snake_case=5		,    snake_case=4		,    snake_case=37		,    snake_case="gelu"		,    snake_case=0.1		,    snake_case=0.1		,    snake_case=50		,    snake_case=0.02		,    snake_case=True		,    snake_case=None		,    ):
		lowercase       =    parent
		lowercase       =    batch_size
		lowercase       =    seq_length
		lowercase       =    is_training
		lowercase       =    use_input_mask
		lowercase       =    vocab_size
		lowercase       =    hidden_size
		lowercase       =    num_hidden_layers
		lowercase       =    num_attention_heads
		lowercase       =    intermediate_size
		lowercase       =    hidden_act
		lowercase       =    hidden_dropout_prob
		lowercase       =    attention_probs_dropout_prob
		lowercase       =    max_position_embeddings
		lowercase       =    initializer_range
		lowercase       =    use_labels
		lowercase       =    scope





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    ids_tensor([self.batch_size, self.seq_length]		,    self.vocab_size       )

		lowercase       =    None
		if self.use_input_mask:
			lowercase       =    random_attention_mask([self.batch_size, self.seq_length]       )

		if self.use_labels:
			lowercase       =    ids_tensor([self.batch_size, self.seq_length]		,    self.vocab_size       )

		lowercase       =    self.get_config()

		return config, input_ids, input_mask, token_labels





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		return BertGenerationConfig(
		    vocab_size=self.vocab_size		,    hidden_size=self.hidden_size		,    num_hidden_layers=self.num_hidden_layers		,    num_attention_heads=self.num_attention_heads		,    intermediate_size=self.intermediate_size		,    hidden_act=self.hidden_act		,    hidden_dropout_prob=self.hidden_dropout_prob		,    attention_probs_dropout_prob=self.attention_probs_dropout_prob		,    max_position_embeddings=self.max_position_embeddings		,    is_decoder=snake_case		,    initializer_range=self.initializer_range		,    )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		(
		    (
		    lowercase
		)  ,					(
		    lowercase
		)  ,					(
		    lowercase
		)  ,					(
		    lowercase
		)  ,					
		)       =    self.prepare_config_and_inputs()

		lowercase       =    True
		lowercase       =    floats_tensor([self.batch_size, self.seq_length, self.hidden_size]       )
		lowercase       =    ids_tensor([self.batch_size, self.seq_length]		,    vocab_size=2       )

		return (
		    config,
		    input_ids,
		    input_mask,
		    token_labels,
		    encoder_hidden_states,
		    encoder_attention_mask,
		)





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    **snake_case		,    ):
		lowercase       =    BertGenerationEncoder(config=snake_case       )
		model.to(snake_case       )
		model.eval()
		lowercase       =    model(snake_case		,    attention_mask=snake_case       )
		lowercase       =    model(snake_case       )
		self.parent.assertEqual(result.last_hidden_state.shape		,    (self.batch_size, self.seq_length, self.hidden_size)       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    **snake_case		,    ):
		lowercase       =    True
		lowercase       =    BertGenerationEncoder(config=snake_case       )
		model.to(snake_case       )
		model.eval()
		lowercase       =    model(
		    snake_case		,    attention_mask=snake_case		,    encoder_hidden_states=snake_case		,    encoder_attention_mask=snake_case		,    )
		lowercase       =    model(
		    snake_case		,    attention_mask=snake_case		,    encoder_hidden_states=snake_case		,    )
		self.parent.assertEqual(result.last_hidden_state.shape		,    (self.batch_size, self.seq_length, self.hidden_size)       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    **snake_case		,    ):
		lowercase       =    True
		lowercase       =    True
		lowercase       =    BertGenerationDecoder(config=snake_case       ).to(snake_case       ).eval()

		# first forward pass
		lowercase       =    model(
		    snake_case		,    attention_mask=snake_case		,    encoder_hidden_states=snake_case		,    encoder_attention_mask=snake_case		,    use_cache=snake_case		,    )
		lowercase       =    outputs.past_key_values

		# create hypothetical multiple next token and extent to next_input_ids
		lowercase       =    ids_tensor((self.batch_size, 3)		,    config.vocab_size       )
		lowercase       =    ids_tensor((self.batch_size, 3)		,    vocab_size=2       )

		# append to next input_ids and
		lowercase       =    torch.cat([input_ids, next_tokens]		,    dim=-1       )
		lowercase       =    torch.cat([input_mask, next_mask]		,    dim=-1       )

		lowercase       =    model(
		    snake_case		,    attention_mask=snake_case		,    encoder_hidden_states=snake_case		,    encoder_attention_mask=snake_case		,    output_hidden_states=snake_case		,    )['hidden_states'][0]
		lowercase       =    model(
		    snake_case		,    attention_mask=snake_case		,    encoder_hidden_states=snake_case		,    encoder_attention_mask=snake_case		,    past_key_values=snake_case		,    output_hidden_states=snake_case		,    )['hidden_states'][0]

		# select random slice
		lowercase       =    ids_tensor((1,)		,    output_from_past.shape[-1]       ).item()
		lowercase       =    output_from_no_past[:, -3:, random_slice_idx].detach()
		lowercase       =    output_from_past[:, :, random_slice_idx].detach()

		self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]       )

		# test that outputs are equal for slice
		self.parent.assertTrue(torch.allclose(snake_case		,    snake_case		,    atol=1E-3       )       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    *snake_case		,    ):
		lowercase       =    BertGenerationDecoder(snake_case       )
		model.to(snake_case       )
		model.eval()
		lowercase       =    model(snake_case		,    attention_mask=snake_case		,    labels=snake_case       )
		self.parent.assertEqual(result.logits.shape		,    (self.batch_size, self.seq_length, self.vocab_size)       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase  ,					lowercase  ,					lowercase  ,					lowercase       =    self.prepare_config_and_inputs()
		lowercase       =    {'input_ids': input_ids, 'attention_mask': input_mask}
		return config, inputs_dict







@require_torch
class        A_   (			__lowerCamelCase      ,	__lowerCamelCase      ,	__lowerCamelCase      ,	unittest.TestCase   ):




	'''simple docstring'''


	_UpperCamelCase     :     Optional[int]							 =       (BertGenerationEncoder, BertGenerationDecoder) if is_torch_available() else ()
	_UpperCamelCase     :     Optional[Any]							 =       (BertGenerationDecoder,) if is_torch_available() else ()
	_UpperCamelCase     :     List[str]							 =       (
	    {"""feature-extraction""": BertGenerationEncoder, """text-generation""": BertGenerationDecoder}
	    if is_torch_available()
	    else {}
	)





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    BertGenerationEncoderTester(self       )
		lowercase       =    ConfigTester(self		,    config_class=snake_case		,    hidden_size=37       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		self.config_tester.run_common_tests()





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    self.model_tester.prepare_config_and_inputs()
		self.model_tester.create_and_check_model(*snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase  ,					lowercase  ,					lowercase  ,					lowercase       =    self.model_tester.prepare_config_and_inputs()
		lowercase       =    'bert'
		self.model_tester.create_and_check_model(snake_case		,    snake_case		,    snake_case		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    self.model_tester.prepare_config_and_inputs_for_decoder()
		self.model_tester.create_and_check_model_as_decoder(*snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    self.model_tester.prepare_config_and_inputs_for_decoder()
		self.model_tester.create_and_check_decoder_model_past_large_inputs(*snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		# This regression test was failing with PyTorch < 1.3
		(
		    (
		    lowercase
		)  ,					(
		    lowercase
		)  ,					(
		    lowercase
		)  ,					(
		    lowercase
		)  ,					(
		    lowercase
		)  ,					(
		    lowercase
		)  ,					
		)       =    self.model_tester.prepare_config_and_inputs_for_decoder()

		lowercase       =    None

		self.model_tester.create_and_check_model_as_decoder(
		    snake_case		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    snake_case		,    )





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    self.model_tester.prepare_config_and_inputs_for_decoder()
		self.model_tester.create_and_check_for_causal_lm(*snake_case       )





	@slow
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    BertGenerationEncoder.from_pretrained('google/bert_for_seq_generation_L-24_bbc_encoder'       )
		self.assertIsNotNone(snake_case       )







@require_torch
class        A_   (			unittest.TestCase   ):




	'''simple docstring'''


	@slow
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    BertGenerationEncoder.from_pretrained('google/bert_for_seq_generation_L-24_bbc_encoder'       )
		lowercase       =    torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 1_0140, 102]]       )
		with torch.no_grad():
			lowercase       =    model(snake_case       )[0]
		lowercase       =    torch.Size([1, 8, 1024]       )
		self.assertEqual(output.shape		,    snake_case       )
		lowercase       =    torch.tensor(
		    [[[0.1_775, 0.0_083, -0.0_321], [1.6_002, 0.1_287, 0.3_912], [2.1_473, 0.5_791, 0.6_066]]]       )
		self.assertTrue(torch.allclose(output[:, :3, :3]		,    snake_case		,    atol=1E-4       )       )







@require_torch
class        A_   (			unittest.TestCase   ):




	'''simple docstring'''


	@slow
	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    BertGenerationDecoder.from_pretrained('google/bert_for_seq_generation_L-24_bbc_encoder'       )
		lowercase       =    torch.tensor([[101, 7592, 1010, 2026, 3899, 2003, 1_0140, 102]]       )
		with torch.no_grad():
			lowercase       =    model(snake_case       )[0]
		lowercase       =    torch.Size([1, 8, 5_0358]       )
		self.assertEqual(output.shape		,    snake_case       )
		lowercase       =    torch.tensor(
		    [[[-0.5_788, -2.5_994, -3.7_054], [0.0_438, 4.7_997, 1.8_795], [1.5_862, 6.6_409, 4.4_638]]]       )
		self.assertTrue(torch.allclose(output[:, :3, :3]		,    snake_case		,    atol=1E-4       )       )



import argparse
import glob
import logging
import os
import time
from argparse import Namespace

import numpy as np
import torch
from lightning_base import BaseTransformer, add_generic_args, generic_train
from torch.utils.data import DataLoader, TensorDataset

from transformers import glue_compute_metrics as compute_metrics
from transformers import glue_convert_examples_to_features as convert_examples_to_features
from transformers import glue_output_modes, glue_tasks_num_labels
from transformers import glue_processors as processors


UpperCAmelCase	      = logging.getLogger(__name__)







class        A_   (			__lowerCamelCase   ):




	'''simple docstring'''


	_UpperCamelCase     :     Optional[int]							 =       """sequence-classification"""





	def __init__(       self		,    snake_case       ):
		if type(snake_case       ) == dict:
			lowercase       =    Namespace(**snake_case       )
		lowercase       =    glue_output_modes[hparams.task]
		lowercase       =    glue_tasks_num_labels[hparams.task]

		super().__init__(snake_case		,    snake_case		,    self.mode       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    **snake_case       ):
		return self.model(**snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case       ):
		lowercase       =    {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

		if self.config.model_type not in ["distilbert", "bart"]:
			lowercase       =    batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

		lowercase       =    self(**snake_case       )
		lowercase       =    outputs[0]

		lowercase       =    self.trainer.lr_schedulers[0]['scheduler']
		lowercase       =    {'loss': loss, 'rate': lr_scheduler.get_last_lr()[-1]}
		return {"loss": loss, "log": tensorboard_logs}





	def 	SCREAMING_SNAKE_CASE__						(       self       ):
		lowercase       =    self.hparams
		lowercase       =    processors[args.task]()
		lowercase       =    processor.get_labels()

		for mode in ["train", "dev"]:
			lowercase       =    self._feature_file(snake_case       )
			if os.path.exists(snake_case       ) and not args.overwrite_cache:
				logger.info('Loading features from cached file %s'		,    snake_case       )
			else:
				logger.info('Creating features from dataset file at %s'		,    args.data_dir       )
				lowercase       =    (
				    processor.get_dev_examples(args.data_dir       )
				    if mode == 'dev'
				    else processor.get_train_examples(args.data_dir       )
				)
				lowercase       =    convert_examples_to_features(
				    snake_case		,    self.tokenizer		,    max_length=args.max_seq_length		,    label_list=self.labels		,    output_mode=args.glue_output_mode		,    )
				logger.info('Saving features into cached file %s'		,    snake_case       )
				torch.save(snake_case		,    snake_case       )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case		,    snake_case = False       ):
		lowercase       =    'dev' if mode == 'test' else mode

		lowercase       =    self._feature_file(snake_case       )
		logger.info('Loading features from cached file %s'		,    snake_case       )
		lowercase       =    torch.load(snake_case       )
		lowercase       =    torch.tensor([f.input_ids for f in features]		,    dtype=torch.long       )
		lowercase       =    torch.tensor([f.attention_mask for f in features]		,    dtype=torch.long       )
		lowercase       =    torch.tensor([f.token_type_ids for f in features]		,    dtype=torch.long       )
		if self.hparams.glue_output_mode == "classification":
			lowercase       =    torch.tensor([f.label for f in features]		,    dtype=torch.long       )
		elif self.hparams.glue_output_mode == "regression":
			lowercase       =    torch.tensor([f.label for f in features]		,    dtype=torch.float       )

		return DataLoader(
		    TensorDataset(snake_case		,    snake_case		,    snake_case		,    snake_case       )		,    batch_size=snake_case		,    shuffle=snake_case		,    )





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case		,    snake_case       ):
		lowercase       =    {'input_ids': batch[0], 'attention_mask': batch[1], 'labels': batch[3]}

		if self.config.model_type not in ["distilbert", "bart"]:
			lowercase       =    batch[2] if self.config.model_type in ['bert', 'xlnet', 'albert'] else None

		lowercase       =    self(**snake_case       )
		lowercase  ,					lowercase       =    outputs[:2]
		lowercase       =    logits.detach().cpu().numpy()
		lowercase       =    inputs['labels'].detach().cpu().numpy()

		return {"val_loss": tmp_eval_loss.detach().cpu(), "pred": preds, "target": out_label_ids}





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase       =    torch.stack([x['val_loss'] for x in outputs]       ).mean().detach().cpu().item()
		lowercase       =    np.concatenate([x['pred'] for x in outputs]		,    axis=0       )

		if self.hparams.glue_output_mode == "classification":
			lowercase       =    np.argmax(snake_case		,    axis=1       )
		elif self.hparams.glue_output_mode == "regression":
			lowercase       =    np.squeeze(snake_case       )

		lowercase       =    np.concatenate([x['target'] for x in outputs]		,    axis=0       )
		lowercase       =    [[] for _ in range(out_label_ids.shape[0]       )]
		lowercase       =    [[] for _ in range(out_label_ids.shape[0]       )]

		lowercase       =    {**{'val_loss': val_loss_mean}, **compute_metrics(self.hparams.task		,    snake_case		,    snake_case       )}

		lowercase       =    dict(results.items()       )
		lowercase       =    results
		return ret, preds_list, out_label_list





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase  ,					lowercase  ,					lowercase       =    self._eval_end(snake_case       )
		lowercase       =    ret['log']
		return {"val_loss": logs["val_loss"], "log": logs, "progress_bar": logs}





	def 	SCREAMING_SNAKE_CASE__						(       self		,    snake_case       ):
		lowercase  ,					lowercase  ,					lowercase       =    self._eval_end(snake_case       )
		lowercase       =    ret['log']
		# `val_loss` is the key returned by `self._eval_end()` but actually refers to `test_loss`
		return {"avg_test_loss": logs["val_loss"], "log": logs, "progress_bar": logs}





	@staticmethod
	def 	SCREAMING_SNAKE_CASE__						(       snake_case		,    snake_case       ):
		BaseTransformer.add_model_specific_args(snake_case		,    snake_case       )
		parser.add_argument(
		    '--max_seq_length'		,    default=128		,    type=snake_case		,    help=(
		        'The maximum total input sequence length after tokenization. Sequences longer '
		        'than this will be truncated, sequences shorter will be padded.'
		    )		,    )

		parser.add_argument(
		    '--task'		,    default=''		,    type=snake_case		,    required=snake_case		,    help='The GLUE task to run'		,    )
		parser.add_argument(
		    '--gpus'		,    default=0		,    type=snake_case		,    help='The number of GPUs allocated for this, it is by default 0 meaning none'		,    )

		parser.add_argument(
		    '--overwrite_cache'		,    action='store_true'		,    help='Overwrite the cached training and evaluation sets'       )

		return parser






def 							UpperCAmelCase_		(							):
	lowercase       =    argparse.ArgumentParser()
	add_generic_args(__SCREAMING_SNAKE_CASE ,							os.getcwd()							)
	lowercase       =    GLUETransformer.add_model_specific_args(__SCREAMING_SNAKE_CASE ,							os.getcwd()							)
	lowercase       =    parser.parse_args()

	# If output_dir not provided, a folder will be generated in pwd
	if args.output_dir is None:
		lowercase       =    os.path.join(
		    './results' ,							F'''{args.task}_{time.strftime('%Y%m%d_%H%M%S'							)}''' ,							)
		os.makedirs(args.output_dir							)

	lowercase       =    GLUETransformer(__SCREAMING_SNAKE_CASE							)
	lowercase       =    generic_train(__SCREAMING_SNAKE_CASE ,							__SCREAMING_SNAKE_CASE							)

	# Optionally, predict on dev set and write to output_dir
	if args.do_predict:
		lowercase       =    sorted(glob.glob(os.path.join(args.output_dir ,							'checkpoint-epoch=*.ckpt'							) ,							recursive=__SCREAMING_SNAKE_CASE							)							)
		lowercase       =    model.load_from_checkpoint(checkpoints[-1]							)
		return trainer.test(__SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
	main()

