






'''simple docstring'''







import argparse

import fairseq
import torch
from torch import nn

from transformers import (
    MBartaaTokenizer,
    MBartConfig,
    MBartForCausalLM,
    SpeechEncoderDecoderConfig,
    SpeechEncoderDecoderModel,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaModel,
    logging,
)


logging.set_verbosity_info()
_A	:			str         =logging.get_logger(__name__)

_A	:			str         ={
    '''post_extract_proj''': '''feature_projection.projection''',
    '''encoder.pos_conv.0''': '''encoder.pos_conv_embed.conv''',
    '''self_attn.k_proj''': '''encoder.layers.*.attention.k_proj''',
    '''self_attn.v_proj''': '''encoder.layers.*.attention.v_proj''',
    '''self_attn.q_proj''': '''encoder.layers.*.attention.q_proj''',
    '''self_attn.out_proj''': '''encoder.layers.*.attention.out_proj''',
    '''self_attn_layer_norm''': '''encoder.layers.*.layer_norm''',
    '''fc1''': '''encoder.layers.*.feed_forward.intermediate_dense''',
    '''fc2''': '''encoder.layers.*.feed_forward.output_dense''',
    '''final_layer_norm''': '''encoder.layers.*.final_layer_norm''',
    '''encoder.layer_norm''': '''encoder.layer_norm''',
    '''w2v_model.layer_norm''': '''feature_projection.layer_norm''',
    '''quantizer.weight_proj''': '''quantizer.weight_proj''',
    '''quantizer.vars''': '''quantizer.codevectors''',
    '''project_q''': '''project_q''',
    '''final_proj''': '''project_hid''',
    '''w2v_encoder.proj''': '''lm_head''',
    '''mask_emb''': '''masked_spec_embed''',
}
_A	:			Dict         =[
    '''lm_head''',
    '''quantizer.weight_proj''',
    '''quantizer.codevectors''',
    '''project_q''',
    '''project_hid''',
]





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )      ->       List[Any]:
							for attribute in key.split("""."""      ):
														lowerCamelCase__					:	Union[str, Any]         =       getattr(UpperCamelCase				,			UpperCamelCase      )

							if weight_type is not None:
														lowerCamelCase__					:	Optional[int]         =       getattr(UpperCamelCase				,			UpperCamelCase      ).shape
							else:
														lowerCamelCase__					:	Optional[int]         =       hf_pointer.shape

							assert hf_shape == value.shape, (
							    f'''Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be'''
							    f''' {value.shape} for {full_name}'''
							)

							if weight_type == "weight":
														lowerCamelCase__					:	Optional[Any]         =       value
							elif weight_type == "weight_g":
														lowerCamelCase__					:	str         =       value
							elif weight_type == "weight_v":
														lowerCamelCase__					:	Any         =       value
							elif weight_type == "bias":
														lowerCamelCase__					:	str         =       value
							else:
														lowerCamelCase__					:	Any         =       value

							logger.info(f'''{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.'''      )





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase      )      ->       int:
							lowerCamelCase__					:	int         =       []
							lowerCamelCase__					:	Optional[Any]         =       fairseq_model.state_dict()

							lowerCamelCase__					:	Dict         =       hf_model.feature_extractor
							lowerCamelCase__					:	List[str]         =       hf_model.adapter

							for name, value in fairseq_dict.items():
														lowerCamelCase__					:	Optional[int]         =       False
														if "conv_layers" in name:
																					load_conv_layer(
																					    UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			hf_model.config.feat_extract_norm == """group"""				,			)
																					lowerCamelCase__					:	Tuple         =       True
														elif any(x in name for x in ["""adaptor""", """w2v_encoder.proj.""", """w2v_proj_ln."""]      ):
																					load_adapter(UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )
																					lowerCamelCase__					:	List[Any]         =       True
														else:
																					for key, mapped_key in MAPPING.items():
																												if key in name or key.split("""w2v_model."""      )[-1] == name.split("""."""      )[0]:
																																			lowerCamelCase__					:	Dict         =       True
																																			if "*" in mapped_key:
																																										lowerCamelCase__					:	Dict         =       name.split(UpperCamelCase      )[0].split("""."""      )[-2]
																																										lowerCamelCase__					:	Any         =       mapped_key.replace("""*"""				,			UpperCamelCase      )
																																			if "weight_g" in name:
																																										lowerCamelCase__					:	Optional[int]         =       """weight_g"""
																																			elif "weight_v" in name:
																																										lowerCamelCase__					:	Any         =       """weight_v"""
																																			elif "bias" in name:
																																										lowerCamelCase__					:	List[str]         =       """bias"""
																																			elif "weight" in name:
																																										lowerCamelCase__					:	int         =       """weight"""
																																			else:
																																										lowerCamelCase__					:	Optional[Any]         =       None
																																			set_recursively(UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )
																												continue
														if not is_used:
																					unused_weights.append(UpperCamelCase      )

							logger.warning(f'''Unused weights: {unused_weights}'''      )





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )      ->       List[str]:
							lowerCamelCase__					:	Any         =       full_name.split("""conv_layers."""      )[-1]
							lowerCamelCase__					:	int         =       name.split("""."""      )
							lowerCamelCase__					:	List[Any]         =       int(items[0]      )
							lowerCamelCase__					:	List[str]         =       int(items[1]      )

							if type_id == 0:
														if "bias" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
																					    f'''{full_name} has size {value.shape}, but'''
																					    f''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''
																					)
																					lowerCamelCase__					:	Union[str, Any]         =       value
																					logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''      )
														elif "weight" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
																					    f'''{full_name} has size {value.shape}, but'''
																					    f''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''
																					)
																					lowerCamelCase__					:	List[Any]         =       value
																					logger.info(f'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''      )
							elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
														if "bias" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
																					    f'''{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'''
																					    " found."
																					)
																					lowerCamelCase__					:	List[Any]         =       value
																					logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''      )
														elif "weight" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
																					    f'''{full_name} has size {value.shape}, but'''
																					    f''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''
																					)
																					lowerCamelCase__					:	Dict         =       value
																					logger.info(f'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''      )
							else:
														unused_weights.append(UpperCamelCase      )





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase      )      ->       List[str]:
							lowerCamelCase__					:	Union[str, Any]         =       full_name.split("""adaptor."""      )[-1]
							lowerCamelCase__					:	Optional[int]         =       name.split("""."""      )

							if items[1].isdigit():
														lowerCamelCase__					:	Dict         =       int(items[1]      )
							else:
														lowerCamelCase__					:	str         =       None

							if "adaptor" not in full_name:
														if "proj_ln" in full_name:
																					# has to be layer norm
																					if "bias" in name:
																												assert (
																												    value.shape == adapter.proj_layer_norm.bias.data.shape
																												), f'''{full_name} has size {value.shape}, but {adapter.proj_layer_norm.bias.data.shape} was found.'''
																												lowerCamelCase__					:	List[Any]         =       value
																												logger.info(f'''Adapter proj layer norm bias was initialized from {full_name}.'''      )
																					if "weight" in name:
																												assert (
																												    value.shape == adapter.proj_layer_norm.weight.data.shape
																												), f'''{full_name} has size {value.shape}, but {adapter.proj_layer_norm.weight.data.shape} was found.'''
																												lowerCamelCase__					:	Dict         =       value
														else:
																					# has to be projection layer
																					if "bias" in name:
																												assert (
																												    value.shape == adapter.proj.bias.data.shape
																												), f'''{full_name} has size {value.shape}, but {adapter.proj.bias.data.shape} was found.'''
																												lowerCamelCase__					:	Tuple         =       value
																												logger.info(f'''Adapter proj layer bias was initialized from {full_name}.'''      )
																					if "weight" in name:
																												assert (
																												    value.shape == adapter.proj.weight.data.shape
																												), f'''{full_name} has size {value.shape}, but {adapter.proj.weight.data.shape} was found.'''
																												lowerCamelCase__					:	Union[str, Any]         =       value
																												logger.info(f'''Adapter proj layer weight was initialized from {full_name}.'''      )
							elif isinstance(UpperCamelCase				,			UpperCamelCase      ):
														if "bias" in name:
																					assert (
																					    value.shape == adapter.layers[layer_id].conv.bias.data.shape
																					), f'''{full_name} has size {value.shape}, but {adapter.layers[layer_id].conv.bias.data.shape} was found.'''
																					lowerCamelCase__					:	Any         =       value
																					logger.info(f'''Adapter layer {layer_id} bias was initialized from {full_name}.'''      )
														elif "weight" in name:
																					assert (
																					    value.shape == adapter.layers[layer_id].conv.weight.data.shape
																					), f'''{full_name} has size {value.shape}, but {adapter.layers[layer_id].conv.weight.data.shape} was found.'''
																					lowerCamelCase__					:	str         =       value
																					logger.info(f'''Adapter layer {layer_id} bias was initialized from {full_name}.'''      )
							else:
														unused_weights.append(UpperCamelCase      )





def  SCREAMING_SNAKE_CASE_       (UpperCamelCase      )      ->       int:
							lowerCamelCase__  ,		lowerCamelCase__					:	Union[str, Any]         =       emb.weight.shape
							lowerCamelCase__					:	int         =       nn.Linear(UpperCamelCase				,			UpperCamelCase				,			bias=UpperCamelCase      )
							lowerCamelCase__					:	str         =       emb.weight.data
							return lin_layer





@torch.no_grad()
def  SCREAMING_SNAKE_CASE_       (UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			UpperCamelCase				,			)      ->       Union[str, Any]:
							lowerCamelCase__					:	Optional[Any]         =       WavaVecaConfig.from_pretrained(
							    UpperCamelCase				,			add_adapter=UpperCamelCase				,			adapter_stride=UpperCamelCase				,			adapter_kernel_size=UpperCamelCase				,			use_auth_token=UpperCamelCase				,			output_hidden_size=UpperCamelCase				,			)
							lowerCamelCase__					:	Dict         =       MBartConfig.from_pretrained(UpperCamelCase      )

							# load model
							lowerCamelCase__  ,		lowerCamelCase__  ,		lowerCamelCase__					:	int         =       fairseq.checkpoint_utils.load_model_ensemble_and_task(
							    [checkpoint_path]				,			arg_overrides={
							        """config_yaml""": config_yaml_path,
							        """data""": """/""".join(dict_path.split("""/"""      )[:-1]      ),
							        """w2v_path""": checkpoint_path,
							        """load_pretrained_decoder_from""": None,
							    }				,			)
							lowerCamelCase__					:	Union[str, Any]         =       model[0].eval()

							# load feature extractor
							lowerCamelCase__					:	Any         =       WavaVecaFeatureExtractor.from_pretrained(UpperCamelCase				,			use_auth_token=UpperCamelCase      )

							# set weights for wav2vec2 encoder
							lowerCamelCase__					:	int         =       WavaVecaModel(UpperCamelCase      )

							recursively_load_weights_wavaveca(model.encoder				,			UpperCamelCase      )

							# load decoder weights
							lowerCamelCase__					:	int         =       MBartForCausalLM(UpperCamelCase      )
							lowerCamelCase__  ,		lowerCamelCase__					:	Dict         =       hf_decoder.model.decoder.load_state_dict(model.decoder.state_dict()				,			strict=UpperCamelCase      )
							logger.warning(f'''The following keys are missing when loading the decoder weights: {missing_keys}'''      )
							logger.warning(f'''The following keys are unexpected when loading the decoder weights: {unexpected_keys}'''      )

							lowerCamelCase__					:	List[str]         =       SpeechEncoderDecoderModel(encoder=UpperCamelCase				,			decoder=UpperCamelCase      )
							lowerCamelCase__					:	Any         =       False

							lowerCamelCase__					:	Any         =       MBartaaTokenizer(UpperCamelCase      )
							tokenizer.save_pretrained(UpperCamelCase      )

							lowerCamelCase__					:	List[str]         =       hf_wavavec.config.to_dict()
							lowerCamelCase__					:	Union[str, Any]         =       tokenizer.pad_token_id
							lowerCamelCase__					:	Dict         =       tokenizer.bos_token_id
							lowerCamelCase__					:	List[str]         =       tokenizer.eos_token_id
							lowerCamelCase__					:	Tuple         =       """mbart50"""
							lowerCamelCase__					:	int         =       """wav2vec2"""

							lowerCamelCase__					:	Any         =       tokenizer.eos_token_id
							lowerCamelCase__					:	List[Any]         =       250004
							lowerCamelCase__					:	Dict         =       tokenizer.eos_token_id

							lowerCamelCase__					:	Tuple         =       SpeechEncoderDecoderConfig.from_dict(UpperCamelCase      )

							hf_wavavec.save_pretrained(UpperCamelCase      )
							feature_extractor.save_pretrained(UpperCamelCase      )


if __name__ == "__main__":
			_A	:			Optional[Any]         =argparse.ArgumentParser()
			parser.add_argument('''--pytorch_dump_folder_path''', default=None, type=str, help='''Path to the output PyTorch model.''')
			parser.add_argument('''--checkpoint_path''', default=None, type=str, help='''Path to fairseq checkpoint''')
			parser.add_argument('''--dict_path''', default=None, type=str, help='''Path to dict of fine-tuned model''')
			parser.add_argument('''--config_yaml_path''', default=None, type=str, help='''Path to yaml file of fine-tuned model''')
			parser.add_argument(
			    '''--encoder_config_path''',
			    default='''facebook/wav2vec2-xls-r-1b''',
			    type=str,
			    help='''Path to hf encoder wav2vec2 checkpoint config''',
			)
			parser.add_argument(
			    '''--decoder_config_path''',
			    default='''facebook/mbart-large-50-one-to-many-mmt''',
			    type=str,
			    help='''Path to hf decoder checkpoint config''',
			)
			parser.add_argument('''--add_adapter''', default=True, type=bool, help='''whethere to add model adapter layers''')
			parser.add_argument('''--adapter_stride''', default=2, type=int, help='''stride of adapter layers''')
			parser.add_argument('''--adapter_kernel_size''', default=3, type=int, help='''kernel size of adapter layers''')
			parser.add_argument('''--encoder_output_dim''', default=1_024, type=int, help='''encoder output dim''')
			parser.add_argument('''--start_token_id''', default=250_004, type=int, help='''`decoder_start_token_id` of model config''')

			_A	:			Optional[Any]         =parser.parse_args()
			convert_wavaveca_checkpoint(
			    args.checkpoint_path,
			    args.pytorch_dump_folder_path,
			    args.dict_path,
			    args.config_yaml_path,
			    encoder_config_path=args.encoder_config_path,
			    decoder_config_path=args.decoder_config_path,
			    add_adapter=args.add_adapter,
			    adapter_kernel_size=args.adapter_kernel_size,
			    adapter_stride=args.adapter_stride,
			    decoder_start_token_id=args.start_token_id,
			    encoder_output_dim=args.encoder_output_dim,
			)




import inspect
import tempfile
import unittest

from huggingface_hub import hf_hub_download

from transformers import is_torch_available
from transformers.testing_utils import is_flaky, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


lowercase_									=   1e-4

if is_torch_available():
  import torch

  from transformers import AutoformerConfig, AutoformerForPrediction, AutoformerModel
  from transformers.models.autoformer.modeling_autoformer import AutoformerDecoder, AutoformerEncoder





@require_torch
class  __lowerCAmelCase     :




  def __init__(				self							,   lowerCAmelCase							,   lowerCAmelCase=16							,   lowerCAmelCase=13							,   lowerCAmelCase=7							,   lowerCAmelCase=14							,   lowerCAmelCase=10							,   lowerCAmelCase=19							,   lowerCAmelCase=5							,   lowerCAmelCase=4							,   lowerCAmelCase=True							,   lowerCAmelCase=16							,   lowerCAmelCase=2							,   lowerCAmelCase=4							,   lowerCAmelCase=4							,   lowerCAmelCase="gelu"							,   lowerCAmelCase=0.1							,   lowerCAmelCase=0.1							,   lowerCAmelCase=[1, 2, 3, 4, 5]							,   lowerCAmelCase=25							,   lowerCAmelCase=5							,   )  -> Optional[Any]:
        '''simple docstring'''






        _lowercase	       =d_model
        _lowercase	       =parent
        _lowercase	       =batch_size
        _lowercase	       =prediction_length
        _lowercase	       =context_length
        _lowercase	       =cardinality
        _lowercase	       =num_time_features
        _lowercase	       =lags_sequence
        _lowercase	       =embedding_dimension
        _lowercase	       =is_training
        _lowercase	       =hidden_size
        _lowercase	       =num_hidden_layers
        _lowercase	       =num_attention_heads
        _lowercase	       =intermediate_size
        _lowercase	       =hidden_act
        _lowercase	       =hidden_dropout_prob
        _lowercase	       =attention_probs_dropout_prob

        _lowercase	       =context_length
        _lowercase	       =prediction_length + label_length
        _lowercase	       =label_length

        _lowercase	       =moving_average
        _lowercase	       =autocorrelation_factor




  def   A__     (				self					)  -> Optional[Any]:
        '''simple docstring'''






        return AutoformerConfig(
            d_model=self.d_model							,   encoder_layers=self.num_hidden_layers							,   decoder_layers=self.num_hidden_layers							,   encoder_attention_heads=self.num_attention_heads							,   decoder_attention_heads=self.num_attention_heads							,   encoder_ffn_dim=self.intermediate_size							,   decoder_ffn_dim=self.intermediate_size							,   dropout=self.hidden_dropout_prob							,   attention_dropout=self.attention_probs_dropout_prob							,   prediction_length=self.prediction_length							,   context_length=self.context_length							,   label_length=self.label_length							,   lags_sequence=self.lags_sequence							,   num_time_features=self.num_time_features							,   num_static_categorical_features=1							,   cardinality=[self.cardinality]							,   embedding_dimension=[self.embedding_dimension]							,   moving_average=self.moving_average							,   )




  def   A__     (				self							,   lowerCAmelCase					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =config.context_length + max(config.lags_sequence					)

        _lowercase	       =ids_tensor([self.batch_size, 1]							,   config.cardinality[0]					)
        _lowercase	       =floats_tensor([self.batch_size, _past_length, config.num_time_features]					)
        _lowercase	       =floats_tensor([self.batch_size, _past_length]					)
        _lowercase	       =floats_tensor([self.batch_size, _past_length]					) > 0.5

        # decoder inputs
        _lowercase	       =floats_tensor([self.batch_size, config.prediction_length, config.num_time_features]					)
        _lowercase	       =floats_tensor([self.batch_size, config.prediction_length]					)

        _lowercase	       ={
            'past_values': past_values,
            'static_categorical_features': static_categorical_features,
            'past_time_features': past_time_features,
            'past_observed_mask': past_observed_mask,
            'future_time_features': future_time_features,
            'future_values': future_values,
        }
        return inputs_dict




  def   A__     (				self					)  -> List[str]:
        '''simple docstring'''






        _lowercase	       =self.get_config()
        _lowercase	       =self.prepare_autoformer_inputs_dict(lowerCAmelCase					)
        return config, inputs_dict




  def   A__     (				self					)  -> List[str]:
        '''simple docstring'''






        _lowercase       ,       _lowercase	       =self.prepare_config_and_inputs()
        return config, inputs_dict




  def   A__     (				self							,   lowerCAmelCase							,   lowerCAmelCase					)  -> Tuple:
        '''simple docstring'''






        _lowercase	       =AutoformerModel(config=lowerCAmelCase					).to(lowerCAmelCase					).eval()
        _lowercase	       =model(**lowerCAmelCase					)

        _lowercase	       =outputs.encoder_last_hidden_state
        _lowercase	       =outputs.last_hidden_state

        with tempfile.TemporaryDirectory() as tmpdirname:
              _lowercase	       =model.get_encoder()
              encoder.save_pretrained(lowerCAmelCase					)
              _lowercase	       =AutoformerEncoder.from_pretrained(lowerCAmelCase					).to(lowerCAmelCase					)

        _lowercase       ,       _lowercase       ,       _lowercase       ,       _lowercase       ,       _lowercase	       =model.create_network_inputs(**lowerCAmelCase					)
        _lowercase       ,       _lowercase	       =model.decomposition_layer(transformer_inputs[:, : config.context_length, ...]					)

        _lowercase	       =torch.cat(
            (transformer_inputs[:, : config.context_length, ...], feature[:, : config.context_length, ...])							,   dim=-1							,   )
        _lowercase	       =encoder(inputs_embeds=lowerCAmelCase					)[0]
        self.parent.assertTrue((encoder_last_hidden_state_a - encoder_last_hidden_state).abs().max().item() < 1e-3					)

        _lowercase	       =(
            torch.mean(transformer_inputs[:, : config.context_length, ...]							,   dim=1					)
            .unsqueeze(1					)
            .repeat(1							,   config.prediction_length							,   1					)
        )
        _lowercase	       =torch.zeros(
            [transformer_inputs.shape[0], config.prediction_length, transformer_inputs.shape[2]]							,   device=enc_input.device							,   )

        _lowercase	       =torch.cat(
            (
                torch.cat((seasonal_input[:, -config.label_length :, ...], zeros)							,   dim=1					),
                feature[:, config.context_length - config.label_length :, ...],
            )							,   dim=-1							,   )
        _lowercase	       =torch.cat(
            (
                torch.cat((trend_input[:, -config.label_length :, ...], mean)							,   dim=1					),
                feature[:, config.context_length - config.label_length :, ...],
            )							,   dim=-1							,   )

        with tempfile.TemporaryDirectory() as tmpdirname:
              _lowercase	       =model.get_decoder()
              decoder.save_pretrained(lowerCAmelCase					)
              _lowercase	       =AutoformerDecoder.from_pretrained(lowerCAmelCase					).to(lowerCAmelCase					)

        _lowercase	       =decoder(
            trend=lowerCAmelCase							,   inputs_embeds=lowerCAmelCase							,   encoder_hidden_states=lowerCAmelCase							,   )[0]

        self.parent.assertTrue((last_hidden_state_a - last_hidden_state).abs().max().item() < 1e-3					)





@require_torch
class  __lowerCAmelCase     (					SCREAMING_SNAKE_CASE			,			SCREAMING_SNAKE_CASE			,			unittest.TestCase							):
  _a									=  (AutoformerModel, AutoformerForPrediction) if is_torch_available() else ()
  _a									=  (AutoformerForPrediction,) if is_torch_available() else ()
  _a									=  {"""feature-extraction""": AutoformerModel} if is_torch_available() else {}
  _a									=  False
  _a									=  False
  _a									=  False
  _a									=  False
  _a									=  False
  _a									=  False




  def   A__     (				self					)  -> str:
        '''simple docstring'''






        _lowercase	       =AutoformerModelTester(self					)
        _lowercase	       =ConfigTester(self							,   config_class=lowerCAmelCase							,   has_text_modality=lowerCAmelCase					)




  def   A__     (				self					)  -> Optional[Any]:
        '''simple docstring'''






        self.config_tester.run_common_tests()




  def   A__     (				self					)  -> Optional[Any]:
        '''simple docstring'''






        _lowercase       ,       _lowercase	       =self.model_tester.prepare_config_and_inputs()
        for model_class in self.all_model_classes:
              _lowercase	       =model_class(lowerCAmelCase					)

              with tempfile.TemporaryDirectory() as tmpdirname:
                    model.save_pretrained(lowerCAmelCase					)
                    _lowercase       ,       _lowercase	       =model_class.from_pretrained(lowerCAmelCase							,   output_loading_info=lowerCAmelCase					)
              self.assertEqual(info['missing_keys']							,   []					)




  def   A__     (				self					)  -> List[str]:
        '''simple docstring'''






        _lowercase	       =self.model_tester.prepare_config_and_inputs_for_common()
        self.model_tester.check_encoder_decoder_model_standalone(*lowerCAmelCase					)




  @unittest.skip(reason='Model has no tokens embeddings'					)
  def   A__     (				self					)  -> int:
        '''simple docstring'''






        pass




  def   A__     (				self					)  -> Dict:
        '''simple docstring'''






        _lowercase	       =inspect.signature(getattr(lowerCAmelCase							,   'forward'					)					)
        # The main input is the name of the argument after `self`
        _lowercase	       =list(model_signature.parameters.keys()					)[1]
        self.assertEqual(AutoformerModel.main_input_name							,   lowerCAmelCase					)




  def   A__     (				self					)  -> List[Any]:
        '''simple docstring'''






        _lowercase       ,       _lowercase	       =self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
              _lowercase	       =model_class(lowerCAmelCase					)
              _lowercase	       =inspect.signature(model.forward					)
              # signature.parameters is an OrderedDict => so arg_names order is deterministic
              _lowercase	       =[*signature.parameters.keys()]

              _lowercase	       =[
                  'past_values',
                  'past_time_features',
                  'past_observed_mask',
                  'static_categorical_features',
                  'static_real_features',
                  'future_values',
                  'future_time_features',
              ]

              if model.__class__.__name__ in ["AutoformerForPrediction"]:
                    expected_arg_names.append('future_observed_mask'					)

              expected_arg_names.extend(
                  [
                      'decoder_attention_mask',
                      'head_mask',
                      'decoder_head_mask',
                      'cross_attn_head_mask',
                      'encoder_outputs',
                      'past_key_values',
                      'output_hidden_states',
                      'output_attentions',
                      'use_cache',
                      'return_dict',
                  ]					)

              self.assertListEqual(arg_names[: len(lowerCAmelCase					)]							,   lowerCAmelCase					)




  def   A__     (				self					)  -> int:
        '''simple docstring'''






        _lowercase       ,       _lowercase	       =self.model_tester.prepare_config_and_inputs_for_common()
        _lowercase	       =True

        _lowercase	       =getattr(self.model_tester							,   'seq_length'							,   lowerCAmelCase					)
        _lowercase	       =getattr(self.model_tester							,   'decoder_seq_length'							,   lowerCAmelCase					)
        _lowercase	       =getattr(self.model_tester							,   'encoder_seq_length'							,   lowerCAmelCase					)
        _lowercase	       =getattr(self.model_tester							,   'd_model'							,   lowerCAmelCase					)
        _lowercase	       =getattr(self.model_tester							,   'num_attention_heads'							,   lowerCAmelCase					)
        _lowercase	       =d_model // num_attention_heads

        for model_class in self.all_model_classes:
              _lowercase	       =True
              _lowercase	       =False
              _lowercase	       =True
              _lowercase	       =model_class(lowerCAmelCase					)
              model.to(lowerCAmelCase					)
              model.eval()
              with torch.no_grad():
                    _lowercase	       =model(**self._prepare_for_class(lowerCAmelCase							,   lowerCAmelCase					)					)
              _lowercase	       =outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions
              self.assertEqual(len(lowerCAmelCase					)							,   self.model_tester.num_hidden_layers					)

              # check that output_attentions also work using config
              del inputs_dict["output_attentions"]
              _lowercase	       =True
              _lowercase	       =model_class(lowerCAmelCase					)
              model.to(lowerCAmelCase					)
              model.eval()
              with torch.no_grad():
                    _lowercase	       =model(**self._prepare_for_class(lowerCAmelCase							,   lowerCAmelCase					)					)
              _lowercase	       =outputs.encoder_attentions
              self.assertEqual(len(lowerCAmelCase					)							,   self.model_tester.num_hidden_layers					)

              self.assertListEqual(
                  list(attentions[0].shape[-3:]					)							,   [self.model_tester.num_attention_heads, encoder_seq_length, dim]							,   )
              _lowercase	       =len(lowerCAmelCase					)

              _lowercase	       =7

              if "last_hidden_state" in outputs:
                    correct_outlen += 1

              if "trend" in outputs:
                    correct_outlen += 1

              if "past_key_values" in outputs:
                    correct_outlen += 1  # past_key_values have been returned

              if "loss" in outputs:
                    correct_outlen += 1

              if "params" in outputs:
                    correct_outlen += 1

              self.assertEqual(lowerCAmelCase							,   lowerCAmelCase					)

              # decoder attentions
              _lowercase	       =outputs.decoder_attentions
              self.assertIsInstance(lowerCAmelCase							,   (list, tuple)					)
              self.assertEqual(len(lowerCAmelCase					)							,   self.model_tester.num_hidden_layers					)
              self.assertListEqual(
                  list(decoder_attentions[0].shape[-3:]					)							,   [self.model_tester.num_attention_heads, decoder_seq_length, dim]							,   )

              # cross attentions
              _lowercase	       =outputs.cross_attentions
              self.assertIsInstance(lowerCAmelCase							,   (list, tuple)					)
              self.assertEqual(len(lowerCAmelCase					)							,   self.model_tester.num_hidden_layers					)
              self.assertListEqual(
                  list(cross_attentions[0].shape[-3:]					)							,   [self.model_tester.num_attention_heads, decoder_seq_length, dim]							,   )

        # Check attention is always last and order is fine
        _lowercase	       =True
        _lowercase	       =True
        _lowercase	       =model_class(lowerCAmelCase					)
        model.to(lowerCAmelCase					)
        model.eval()
        with torch.no_grad():
              _lowercase	       =model(**self._prepare_for_class(lowerCAmelCase							,   lowerCAmelCase					)					)

        self.assertEqual(out_len + 2							,   len(lowerCAmelCase					)					)

        _lowercase	       =outputs.encoder_attentions if config.is_encoder_decoder else outputs.attentions

        self.assertEqual(len(lowerCAmelCase					)							,   self.model_tester.num_hidden_layers					)
        self.assertListEqual(
            list(self_attentions[0].shape[-3:]					)							,   [self.model_tester.num_attention_heads, encoder_seq_length, dim]							,   )




  @is_flaky()
  def   A__     (				self					)  -> Dict:
        '''simple docstring'''






        super().test_retain_grad_hidden_states_attentions()
def      a      (			A__	:   List[str]="train-batch.pt"     )    ->      str:





      """simple docstring"""



      _lowercase	       =hf_hub_download(repo_id='hf-internal-testing/tourism-monthly-batch'      ,   filename=A__      ,   repo_type='dataset'     )
      _lowercase	       =torch.load(A__      ,   map_location=A__     )
      return batch







@require_torch
@slow
class  __lowerCAmelCase     (					unittest.TestCase							):




  def   A__     (				self					)  -> int:
        '''simple docstring'''






        _lowercase	       =AutoformerModel.from_pretrained('huggingface/autoformer-tourism-monthly'					).to(lowerCAmelCase					)
        _lowercase	       =prepare_batch()

        with torch.no_grad():
              _lowercase	       =model(
                  past_values=batch['past_values']							,   past_time_features=batch['past_time_features']							,   past_observed_mask=batch['past_observed_mask']							,   static_categorical_features=batch['static_categorical_features']							,   future_values=batch['future_values']							,   future_time_features=batch['future_time_features']							,   )[0]

        _lowercase	       =torch.Size(
            (64, model.config.prediction_length + model.config.label_length, model.config.feature_size)					)
        self.assertEqual(output.shape							,   lowerCAmelCase					)

        _lowercase	       =torch.tensor(
            [[0.3593, -1.3398, 0.6330], [0.2279, 1.5396, -0.1792], [0.0450, 1.3225, -0.2335]]							,   device=lowerCAmelCase					)
        self.assertTrue(torch.allclose(output[0, :3, :3]							,   lowerCAmelCase							,   atol=lowerCAmelCase					)					)




  def   A__     (				self					)  -> str:
        '''simple docstring'''






        _lowercase	       =AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'					).to(lowerCAmelCase					)
        _lowercase	       =prepare_batch('val-batch.pt'					)
        with torch.no_grad():
              _lowercase	       =model(
                  past_values=batch['past_values']							,   past_time_features=batch['past_time_features']							,   past_observed_mask=batch['past_observed_mask']							,   static_categorical_features=batch['static_categorical_features']							,   ).encoder_last_hidden_state
        _lowercase	       =torch.Size((64, model.config.context_length, model.config.d_model)					)
        self.assertEqual(output.shape							,   lowerCAmelCase					)

        _lowercase	       =torch.tensor(
            [[-0.0734, -0.9036, 0.8358], [4.7186, 2.4113, 1.9581], [1.7953, 2.3558, 1.2970]]							,   device=lowerCAmelCase					)
        self.assertTrue(torch.allclose(output[0, :3, :3]							,   lowerCAmelCase							,   atol=lowerCAmelCase					)					)




  def   A__     (				self					)  -> Optional[int]:
        '''simple docstring'''






        _lowercase	       =AutoformerForPrediction.from_pretrained('huggingface/autoformer-tourism-monthly'					).to(lowerCAmelCase					)
        _lowercase	       =prepare_batch('val-batch.pt'					)
        with torch.no_grad():
              _lowercase	       =model.generate(
                  static_categorical_features=batch['static_categorical_features']							,   past_time_features=batch['past_time_features']							,   past_values=batch['past_values']							,   future_time_features=batch['future_time_features']							,   past_observed_mask=batch['past_observed_mask']							,   )
        _lowercase	       =torch.Size((64, model.config.num_parallel_samples, model.config.prediction_length)					)
        self.assertEqual(outputs.sequences.shape							,   lowerCAmelCase					)

        _lowercase	       =torch.tensor([3130.6763, 4056.5293, 7053.0786]							,   device=lowerCAmelCase					)
        _lowercase	       =outputs.sequences.mean(dim=1					)
        self.assertTrue(torch.allclose(mean_prediction[0, -3:]							,   lowerCAmelCase							,   rtol=1e-1					)					)


