



"""simple docstring"""


import logging
import os
from dataclasses import dataclass, field
from typing import Dict, List, Optional

import numpy as np
import torch
from utils_hans import HansDataset, InputFeatures, hans_processors, hans_tasks_num_labels

import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import is_main_process


snake_case_								=			logging.getLogger(__name__)







@dataclass
class  A_     :



						"""simple docstring"""
						__UpperCamelCase													=				field(
						    metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}							)
						__UpperCamelCase													=				field(
						    default=SCREAMING_SNAKE_CASE_      ,  metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}							)
						__UpperCamelCase													=				field(
						    default=SCREAMING_SNAKE_CASE_      ,  metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}							)
						__UpperCamelCase													=				field(
						    default=SCREAMING_SNAKE_CASE_      ,  metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}      ,  )







@dataclass
class  A_     :



						"""simple docstring"""
						__UpperCamelCase													=				field(
						    metadata={"""help""": """The name of the task to train selected in the list: """ + """, """.join(hans_processors.keys()							)}							)
						__UpperCamelCase													=				field(
						    metadata={"""help""": """The input data dir. Should contain the .tsv files (or other data files) for the task."""}							)
						__UpperCamelCase													=				field(
						    default=1_28      ,  metadata={
						        """help""": (
						            """The maximum total input sequence length after tokenization. Sequences longer """
						            """than this will be truncated, sequences shorter will be padded."""
						        )
						    }      ,  )
						__UpperCamelCase													=				field(
						    default=SCREAMING_SNAKE_CASE_      ,  metadata={"""help""": """Overwrite the cached training and evaluation sets"""}							)


def 					_lowerCAmelCase (  lowercase_		):
						UpperCAmelCase			 =					default_data_collator(lowercase_		)
						UpperCAmelCase			 =					batch.pop('pairID'						,      lowercase_		)
						return batch

def 					_lowerCAmelCase (  ):
						# See all possible arguments in src/transformers/training_args.py
						# or by passing the --help flag to this script.
						# We now keep distinct sets of args, for a cleaner separation of concerns.

						UpperCAmelCase			 =					HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)		)
						UpperCAmelCase ,       UpperCAmelCase ,       UpperCAmelCase			 =					parser.parse_args_into_dataclasses()

						if (
						    os.path.exists(training_args.output_dir		)
						    and os.listdir(training_args.output_dir		)
						    and training_args.do_train
						    and not training_args.overwrite_output_dir
						):
												raise ValueError(
												    F"""Output directory ({training_args.output_dir}) already exists and is not empty. Use"""
												    ' --overwrite_output_dir to overcome.'		)

						# Setup logging
						logging.basicConfig(
						    format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'						,      datefmt='%m/%d/%Y %H:%M:%S'						,      level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN						,      )
						logger.warning(
						    'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'						,      training_args.local_rank						,      training_args.device						,      training_args.n_gpu						,      bool(training_args.local_rank != -1		)						,      training_args.fpaa						,      )
						# Set the verbosity to info of the Transformers logger (on main process only):
						if is_main_process(training_args.local_rank		):
												transformers.utils.logging.set_verbosity_info()
												transformers.utils.logging.enable_default_handler()
												transformers.utils.logging.enable_explicit_format()
						logger.info('Training/evaluation parameters %s'						,      lowercase_		)

						# Set seed
						set_seed(training_args.seed		)

						try:
												UpperCAmelCase			 =					hans_tasks_num_labels[data_args.task_name]
						except KeyError:
												raise ValueError('Task not found: %s' % (data_args.task_name)		)

						# Load pretrained model and tokenizer
						#
						# Distributed training:
						# The .from_pretrained methods guarantee that only one local process can concurrently
						# download model & vocab.

						UpperCAmelCase			 =					AutoConfig.from_pretrained(
						    model_args.config_name if model_args.config_name else model_args.model_name_or_path						,      num_labels=lowercase_						,      finetuning_task=data_args.task_name						,      cache_dir=model_args.cache_dir						,      )
						UpperCAmelCase			 =					AutoTokenizer.from_pretrained(
						    model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path						,      cache_dir=model_args.cache_dir						,      )
						UpperCAmelCase			 =					AutoModelForSequenceClassification.from_pretrained(
						    model_args.model_name_or_path						,      from_tf=bool('.ckpt' in model_args.model_name_or_path		)						,      config=lowercase_						,      cache_dir=model_args.cache_dir						,      )

						# Get datasets
						UpperCAmelCase			 =					(
						    HansDataset(
						        data_dir=data_args.data_dir						,      tokenizer=lowercase_						,      task=data_args.task_name						,      max_seq_length=data_args.max_seq_length						,      overwrite_cache=data_args.overwrite_cache						,      )
						    if training_args.do_train
						    else None
						)
						UpperCAmelCase			 =					(
						    HansDataset(
						        data_dir=data_args.data_dir						,      tokenizer=lowercase_						,      task=data_args.task_name						,      max_seq_length=data_args.max_seq_length						,      overwrite_cache=data_args.overwrite_cache						,      evaluate=lowercase_						,      )
						    if training_args.do_eval
						    else None
						)

						# Initialize our Trainer
						UpperCAmelCase			 =					Trainer(
						    model=lowercase_						,      args=lowercase_						,      train_dataset=lowercase_						,      eval_dataset=lowercase_						,      data_collator=lowercase_						,      )

						# Training
						if training_args.do_train:
												trainer.train(
												    model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path		) else None		)
												trainer.save_model()
												# For convenience, we also re-save the tokenizer to the same directory,
												# so that you can share your model easily on huggingface.co/models =)
												if trainer.is_world_master():
																		tokenizer.save_pretrained(training_args.output_dir		)

    # Evaluation
						if training_args.do_eval:
												logger.info('*** Evaluate ***'		)

												UpperCAmelCase			 =					trainer.predict(lowercase_		)
												UpperCAmelCase			 =					output.predictions
												UpperCAmelCase			 =					np.argmax(lowercase_						,      axis=1		)

												UpperCAmelCase			 =					[ex.pairID for ex in eval_dataset]
												UpperCAmelCase			 =					os.path.join(training_args.output_dir						,      'hans_predictions.txt'		)
												UpperCAmelCase			 =					eval_dataset.get_labels()
												if trainer.is_world_master():
																		with open(lowercase_						,      'w'		) as writer:
																								writer.write('pairID,gold_label\n'		)
																								for pid, pred in zip(lowercase_						,      lowercase_		):
																														writer.write('ex' + str(lowercase_		) + ',' + label_list[int(lowercase_		)] + '\n'		)

												trainer._log(output.metrics		)

def 					_lowerCAmelCase (  lowercase_		):
						# For xla_spawn (TPUs)
						main()


if __name__ == "__main__":
				main()









"""simple docstring"""






import argparse
import os
from functools import reduce

import fairseq
import torch
from datasets import load_dataset

from transformers import WavaVecaProcessor, logging
from transformers.models.dataavec.configuration_dataavec_audio import DataaVecAudioConfig

# Copied from https://github.com/pytorch/fairseq/blob/main/examples/data2vec/models/data2vec_audio.py
from transformers.models.dataavec.dataavec_audio import DataaVecAudioModel as Dummy  # noqa: F401
from transformers.models.dataavec.modeling_dataavec_audio import DataaVecAudioForCTC, DataaVecAudioModel


logging.set_verbosity_info()
lowercase  		=			logging.get_logger(__name__)

lowercase  		=			{
    """post_extract_proj""": """feature_projection.projection""",
    """models.0.layer_norm""": """feature_projection.layer_norm""",
    """self_attn.k_proj""": """encoder.layers.*.attention.k_proj""",
    """self_attn.v_proj""": """encoder.layers.*.attention.v_proj""",
    """self_attn.q_proj""": """encoder.layers.*.attention.q_proj""",
    """self_attn.out_proj""": """encoder.layers.*.attention.out_proj""",
    """self_attn_layer_norm""": """encoder.layers.*.layer_norm""",
    """fc1""": """encoder.layers.*.feed_forward.intermediate_dense""",
    """fc2""": """encoder.layers.*.feed_forward.output_dense""",
    """final_layer_norm""": """encoder.layers.*.final_layer_norm""",
    """encoder.layer_norm""": """encoder.layer_norm""",
    """w2v_model.layer_norm""": """feature_projection.layer_norm""",
    """w2v_encoder.proj""": """lm_head""",
    """mask_emb""": """masked_spec_embed""",
}
lowercase  		=			[
    """lm_head""",
]



def 			A__		(					_UpperCAmelCase    :      str				,    _UpperCAmelCase    :      List[Any]				,    _UpperCAmelCase    :      List[str]				,    _UpperCAmelCase    :      Optional[int]				,    _UpperCAmelCase    :      Tuple	)							->	Union[str, Any]:

	'''simple docstring'''


	for attribute in key.split("."	):
		snake_case__  :						Dict		=      getattr(_UpperCAmelCase				,    _UpperCAmelCase	)

	if weight_type is not None:
		snake_case__  :						Optional[Any]		=      getattr(_UpperCAmelCase				,    _UpperCAmelCase	).shape
	else:
		snake_case__  :						int		=      hf_pointer.shape

	if hf_shape != value.shape:
		raise ValueError(
		    F"""Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be"""
		    F""" {value.shape} for {full_name}"""	)

	if weight_type == "weight":
		snake_case__  :						Tuple		=      value
	elif weight_type == "weight_g":
		snake_case__  :						List[Any]		=      value
	elif weight_type == "weight_v":
		snake_case__  :						int		=      value
	elif weight_type == "bias":
		snake_case__  :						List[Any]		=      value
	else:
		snake_case__  :						Tuple		=      value

	logger.info(F"""{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}."""	)



def 			A__		(					_UpperCAmelCase    :      List[Any]				,    _UpperCAmelCase    :      Dict				,    _UpperCAmelCase    :      Tuple	)							->	List[Any]:

	'''simple docstring'''


	snake_case__  :						Optional[Any]		=      []
	snake_case__  :						Any		=      fairseq_model.state_dict()

	if not is_headless:
		snake_case__  :						int		=      hf_model.dataavec_audio.feature_extractor
		snake_case__  :						Optional[Any]		=      hf_model.dataavec_audio.encoder.pos_conv_embed

	else:
		snake_case__  :						Union[str, Any]		=      hf_model.feature_extractor
		snake_case__  :						str		=      hf_model.encoder.pos_conv_embed

	for name, value in fairseq_dict.items():
		snake_case__  :						Optional[Any]		=      False
		if "conv_layers" in name:
			load_conv_layer(
			    _UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    )
			snake_case__  :						str		=      True
		elif "pos_conv" in name:
			load_pos_conv_layer(
			    _UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    )
			snake_case__  :						Union[str, Any]		=      True
		else:
			for key, mapped_key in MAPPING.items():
				if not is_headless:
					snake_case__  :						Tuple		=      "data2vec_audio." + mapped_key if mapped_key not in TOP_LEVEL_KEYS else mapped_key
				if key in name or key.split("w2v_model."	)[-1] == name.split("."	)[0]:
					snake_case__  :						Dict		=      True
					if "*" in mapped_key:
						snake_case__  :						List[str]		=      name.split(_UpperCAmelCase	)[0].split("."	)[-2]
						snake_case__  :						Dict		=      mapped_key.replace("*"				,    _UpperCAmelCase	)
					if "weight_g" in name:
						snake_case__  :						str		=      "weight_g"
					elif "weight_v" in name:
						snake_case__  :						Union[str, Any]		=      "weight_v"
					elif "bias" in name:
						snake_case__  :						Tuple		=      "bias"
					elif "weight" in name:
						# TODO: don't match quantizer.weight_proj
						snake_case__  :						List[Any]		=      "weight"
					else:
						snake_case__  :						Any		=      None
					set_recursively(_UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase	)
				continue
		if not is_used:
			unused_weights.append(_UpperCAmelCase	)

	logger.warning(F"""Unused weights: {unused_weights}"""	)



def 			A__		(					_UpperCAmelCase    :      Optional[int]				,    _UpperCAmelCase    :      int	)							->	Tuple:

	'''simple docstring'''


	snake_case__  :						Optional[Any]		=      path.split("."	)
	return reduce(_UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase	)



def 			A__		(					_UpperCAmelCase    :      List[str]				,    _UpperCAmelCase    :      str				,    _UpperCAmelCase    :      Dict				,    _UpperCAmelCase    :      str	)							->	Tuple:

	'''simple docstring'''


	snake_case__  :						List[str]		=      access_by_string(_UpperCAmelCase				,    _UpperCAmelCase	)
	snake_case__  :						Optional[Any]		=      hf_weight.data

	if fsq_value.shape != hf_value.shape:
		raise ValueError(F"""{full_name} has size {fsq_value.shape}, but {hf_value.shape} was found."""	)
	snake_case__  :						Tuple		=      fsq_value
	logger.info(F"""{full_name} was correctly initialized from {hf_weight_path}."""	)



def 			A__		(					_UpperCAmelCase    :      Any				,    _UpperCAmelCase    :      Any				,    _UpperCAmelCase    :      List[str]				,    _UpperCAmelCase    :      Tuple	)							->	Dict:

	'''simple docstring'''


	snake_case__  :						List[str]		=      full_name.split("conv_layers."	)[-1]
	snake_case__  :						Tuple		=      name.split("."	)
	snake_case__  :						Any		=      int(items[0]	)
	snake_case__  :						List[Any]		=      int(items[1]	)

	snake_case__  :						Tuple		=      name.split("."	)[-1]
	if type_id == 0:
		snake_case__  :						Dict		=      "conv"
	elif type_id == 2:
		snake_case__  :						Dict		=      "layer_norm"
	else:
		unused_weights.append(_UpperCAmelCase	)
		return

	set_weights(_UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    F"""conv_layers.{layer_id}.{layer_type}.{weight_type}"""	)



def 			A__		(					_UpperCAmelCase    :      Any				,    _UpperCAmelCase    :      Dict				,    _UpperCAmelCase    :      Dict				,    _UpperCAmelCase    :      str	)							->	Tuple:

	'''simple docstring'''


	snake_case__  :						Optional[Any]		=      full_name.split("pos_conv."	)[-1]
	snake_case__  :						Optional[Any]		=      name.split("."	)
	snake_case__  :						Dict		=      int(items[0]	)
	snake_case__  :						Optional[Any]		=      int(items[1]	)

	snake_case__  :						int		=      name.split("."	)[-1]
	if type_id != 0:
		unused_weights.append(_UpperCAmelCase	)
		return
	else:
		snake_case__  :						Optional[int]		=      "conv"

	set_weights(_UpperCAmelCase				,    _UpperCAmelCase				,    _UpperCAmelCase				,    F"""layers.{layer_id}.{layer_type}.{weight_type}"""	)



@torch.no_grad()
def 			A__		(					_UpperCAmelCase    :      Dict				,    _UpperCAmelCase    :      Union[str, Any]				,    _UpperCAmelCase    :      List[str]=None				,    _UpperCAmelCase    :      Optional[Any]=None				,    _UpperCAmelCase    :      int=True	)							->	Dict:

	'''simple docstring'''


	if config_path is not None:
		snake_case__  :						List[str]		=      DataaVecAudioConfig.from_pretrained(_UpperCAmelCase	)
	else:
		snake_case__  :						int		=      DataaVecAudioConfig()

	if not is_finetuned:
		# Modify final_proj layer name
		snake_case__  :						int		=      DataaVecAudioModel(_UpperCAmelCase	)
		snake_case__  :						Union[str, Any]		=      os.path.dirname(_UpperCAmelCase	)

		snake_case__  :						Optional[Any]		=      torch.load(_UpperCAmelCase	)
		snake_case__  :						str		=      state_dict["model"].pop("final_proj.0.weight"	)
		snake_case__  :						List[str]		=      state_dict["model"].pop("final_proj.0.bias"	)
		snake_case__  :						Optional[Any]		=      os.path.join(_UpperCAmelCase				,    "converted.pt"	)
		torch.save(_UpperCAmelCase				,    _UpperCAmelCase	)
	else:
		snake_case__  :						Optional[Any]		=      DataaVecAudioForCTC(_UpperCAmelCase	)
		snake_case__  :						int		=      checkpoint_path

	def load_dataavec(_UpperCAmelCase    :      Union[str, Any]	):
		snake_case__,					snake_case__,					snake_case__  :						Dict		=      fairseq.checkpoint_utils.load_model_ensemble_and_task([path]	)
		return model[0].eval()

	snake_case__  :						Optional[int]		=      load_dataavec(_UpperCAmelCase	)

	recursively_load_weights(_UpperCAmelCase				,    _UpperCAmelCase				,    not is_finetuned	)

	snake_case__  :						Any		=      WavaVecaProcessor.from_pretrained("facebook/wav2vec2-large-lv60"	)

	snake_case__  :						Dict		=      load_dataset("patrickvonplaten/librispeech_asr_dummy"				,    "clean"				,    split="validation"	)
	snake_case__  :						Tuple		=      [x["array"] for x in ds[:4]["audio"]]

	snake_case__  :						Union[str, Any]		=      processor(_UpperCAmelCase				,    return_tensors="pt"				,    padding=_UpperCAmelCase	)

	snake_case__  :						Any		=      inputs.input_values
	snake_case__  :						List[str]		=      inputs.attention_mask
	#    input_values = inputs.input_values[:, :-1]
	#    attention_mask = inputs.attention_mask[:, :-1]

	hf_wavavec.eval()
	model.eval()
	if is_finetuned:
		snake_case__  :						Optional[Any]		=      model(source=_UpperCAmelCase				,    padding_mask=(1 - attention_mask)				,    mask=_UpperCAmelCase				,    features_only=_UpperCAmelCase	)[
		    "encoder_out"
		].transpose(0				,    1	)
		snake_case__  :						Union[str, Any]		=      hf_wavavec(_UpperCAmelCase				,    attention_mask=_UpperCAmelCase	)["logits"]

		snake_case__  :						Optional[int]		=      torch.argmax(_UpperCAmelCase				,    dim=-1	)
		snake_case__  :						Union[str, Any]		=      processor.batch_decode(_UpperCAmelCase	)

		print(F"""Expected Output: {ds[:4]['text']}, Pred: {output_string}"""	)
	else:
		snake_case__  :						Dict		=      model(source=_UpperCAmelCase				,    padding_mask=(1 - attention_mask)				,    mask=_UpperCAmelCase				,    features_only=_UpperCAmelCase	)[
		    "layer_results"
		][-1][0].transpose(0				,    1	)
		snake_case__  :						str		=      hf_wavavec(_UpperCAmelCase				,    attention_mask=_UpperCAmelCase	)["last_hidden_state"]

	print(our_output.shape				,    their_output.shape	)
	snake_case__  :						Union[str, Any]		=      torch.max(torch.abs(our_output - their_output	)	).item()
	print(F"""max_absolute_diff = {max_absolute_diff}"""	)  # ~ 1e-7
	snake_case__  :						str		=      torch.allclose(_UpperCAmelCase				,    _UpperCAmelCase				,    atol=1e-3	)
	print("Do both models output the same tensors?"				,    "🔥" if success else "💩"	)
	if not success:
		raise Exception("Something went wRoNg"	)

	hf_wavavec.save_pretrained(_UpperCAmelCase	)

	if is_finetuned:
		processor.save_pretrained(_UpperCAmelCase	)
	else:
		processor.feature_extractor.save_pretrained(_UpperCAmelCase	)


if __name__ == "__main__":
					lowercase  		=			argparse.ArgumentParser()
					parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
					parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
					parser.add_argument("""--dict_path""", default=None, type=str, help="""Path to dict of fine-tuned model""")
					parser.add_argument("""--config_path""", default=None, type=str, help="""Path to hf config.json of model to convert""")
					parser.add_argument(
					    """--not_finetuned""", action="""store_true""", help="""Whether the model to convert is a fine-tuned model or not"""
					)
					lowercase  		=			parser.parse_args()
					convert_wavaveca_checkpoint(
					    args.checkpoint_path, args.pytorch_dump_folder_path, args.config_path, args.dict_path, not args.not_finetuned
					)





