



'''simple docstring'''




import argparse
import json
import os

import fairseq
import torch
from torch import nn

from transformers import (
    SpeechaTextaConfig,
    SpeechaTextaForCausalLM,
    SpeechaTextaTokenizer,
    SpeechEncoderDecoderConfig,
    SpeechEncoderDecoderModel,
    WavaVecaConfig,
    WavaVecaFeatureExtractor,
    WavaVecaModel,
    logging,
)


logging.set_verbosity_info()
a   :			Union[str, Any]					  =    logging.get_logger(__name__)

a   :			int					  =    {
    """post_extract_proj""": """feature_projection.projection""",
    """encoder.pos_conv.0""": """encoder.pos_conv_embed.conv""",
    """self_attn.k_proj""": """encoder.layers.*.attention.k_proj""",
    """self_attn.v_proj""": """encoder.layers.*.attention.v_proj""",
    """self_attn.q_proj""": """encoder.layers.*.attention.q_proj""",
    """self_attn.out_proj""": """encoder.layers.*.attention.out_proj""",
    """self_attn_layer_norm""": """encoder.layers.*.layer_norm""",
    """fc1""": """encoder.layers.*.feed_forward.intermediate_dense""",
    """fc2""": """encoder.layers.*.feed_forward.output_dense""",
    """final_layer_norm""": """encoder.layers.*.final_layer_norm""",
    """encoder.layer_norm""": """encoder.layer_norm""",
    """w2v_model.layer_norm""": """feature_projection.layer_norm""",
    """quantizer.weight_proj""": """quantizer.weight_proj""",
    """quantizer.vars""": """quantizer.codevectors""",
    """project_q""": """project_q""",
    """final_proj""": """project_hid""",
    """w2v_encoder.proj""": """lm_head""",
    """mask_emb""": """masked_spec_embed""",
}
a   :			Tuple					  =    [
    """lm_head""",
    """quantizer.weight_proj""",
    """quantizer.codevectors""",
    """project_q""",
    """project_hid""",
]



def       __lowerCamelCase  (					_lowercase	,				_lowercase	,				_lowercase	,				_lowercase	,				_lowercase		)      ->			Union[str, Any]:

							for attribute in key.split("""."""		):
														UpperCAmelCase     :		str							     =		getattr(__lowerCAmelCase	,				__lowerCAmelCase		)

							if weight_type is not None:
														UpperCAmelCase     :		int							     =		getattr(__lowerCAmelCase	,				__lowerCAmelCase		).shape
							else:
														UpperCAmelCase     :		Optional[int]							     =		hf_pointer.shape

							assert hf_shape == value.shape, (
							    F'''Shape of hf {key + '.' + weight_type if weight_type is not None else ''} is {hf_shape}, but should be'''
							    F''' {value.shape} for {full_name}'''
							)

							if weight_type == "weight":
														UpperCAmelCase     :		Dict							     =		value
							elif weight_type == "weight_g":
														UpperCAmelCase     :		List[Any]							     =		value
							elif weight_type == "weight_v":
														UpperCAmelCase     :		int							     =		value
							elif weight_type == "bias":
														UpperCAmelCase     :		Union[str, Any]							     =		value
							else:
														UpperCAmelCase     :		List[Any]							     =		value

							logger.info(F'''{key + '.' + weight_type if weight_type is not None else ''} was initialized from {full_name}.'''		)



def       __lowerCamelCase  (					_lowercase	,				_lowercase		)      ->			Optional[int]:
							UpperCAmelCase     :		Union[str, Any]							     =		[]
							UpperCAmelCase     :		Any							     =		fairseq_model.state_dict()

							UpperCAmelCase     :		Union[str, Any]							     =		hf_model.feature_extractor

							# if encoder has different dim to decoder -> use proj_weight
							UpperCAmelCase     :		Union[str, Any]							     =		None

							for name, value in fairseq_dict.items():
														UpperCAmelCase     :		Optional[int]							     =		False
														if "conv_layers" in name:
																					load_conv_layer(
																					    __lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase	,				hf_model.config.feat_extract_norm == """group"""	,				)
																					UpperCAmelCase     :		List[str]							     =		True
														elif name.split("""."""		)[0] == "proj":
																					UpperCAmelCase     :		str							     =		fairseq_model.proj
																					UpperCAmelCase     :		List[Any]							     =		True
														else:
																					for key, mapped_key in MAPPING.items():
																												if key in name or key.split("""w2v_model."""		)[-1] == name.split("""."""		)[0]:
																																			UpperCAmelCase     :		Dict							     =		True
																																			if "*" in mapped_key:
																																										UpperCAmelCase     :		str							     =		name.split(__lowerCAmelCase		)[0].split("""."""		)[-2]
																																										UpperCAmelCase     :		Optional[int]							     =		mapped_key.replace("""*"""	,				__lowerCAmelCase		)
																																			if "weight_g" in name:
																																										UpperCAmelCase     :		Any							     =		'''weight_g'''
																																			elif "weight_v" in name:
																																										UpperCAmelCase     :		Any							     =		'''weight_v'''
																																			elif "bias" in name:
																																										UpperCAmelCase     :		List[Any]							     =		'''bias'''
																																			elif "weight" in name:
																																										UpperCAmelCase     :		Optional[Any]							     =		'''weight'''
																																			else:
																																										UpperCAmelCase     :		Any							     =		None
																																			set_recursively(__lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase	,				__lowerCAmelCase		)
																												continue
														if not is_used:
																					unused_weights.append(__lowerCAmelCase		)

							logger.warning(F'''Unused weights: {unused_weights}'''		)

							return proj_weight



def       __lowerCamelCase  (					_lowercase	,				_lowercase	,				_lowercase	,				_lowercase	,				_lowercase		)      ->			str:
							UpperCAmelCase     :		List[Any]							     =		full_name.split("""conv_layers."""		)[-1]
							UpperCAmelCase     :		str							     =		name.split("""."""		)
							UpperCAmelCase     :		List[Any]							     =		int(items[0]		)
							UpperCAmelCase     :		List[str]							     =		int(items[1]		)

							if type_id == 0:
														if "bias" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].conv.bias.data.shape, (
																					    F'''{full_name} has size {value.shape}, but'''
																					    F''' {feature_extractor.conv_layers[layer_id].conv.bias.data.shape} was found.'''
																					)
																					UpperCAmelCase     :		Optional[Any]							     =		value
																					logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''		)
														elif "weight" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].conv.weight.data.shape, (
																					    F'''{full_name} has size {value.shape}, but'''
																					    F''' {feature_extractor.conv_layers[layer_id].conv.weight.data.shape} was found.'''
																					)
																					UpperCAmelCase     :		Any							     =		value
																					logger.info(F'''Feat extract conv layer {layer_id} was initialized from {full_name}.'''		)
							elif (type_id == 2 and not use_group_norm) or (type_id == 2 and layer_id == 0 and use_group_norm):
														if "bias" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.bias.data.shape, (
																					    F'''{full_name} has size {value.shape}, but {feature_extractor[layer_id].layer_norm.bias.data.shape} was'''
																					    " found."
																					)
																					UpperCAmelCase     :		str							     =		value
																					logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''		)
														elif "weight" in name:
																					assert value.shape == feature_extractor.conv_layers[layer_id].layer_norm.weight.data.shape, (
																					    F'''{full_name} has size {value.shape}, but'''
																					    F''' {feature_extractor[layer_id].layer_norm.weight.data.shape} was found.'''
																					)
																					UpperCAmelCase     :		str							     =		value
																					logger.info(F'''Feat extract layer norm weight of layer {layer_id} was initialized from {full_name}.'''		)
							else:
														unused_weights.append(__lowerCAmelCase		)



def       __lowerCamelCase  (					_lowercase		)      ->			str:
							UpperCAmelCase     :		List[str]							     =		emb.weight.shape
							UpperCAmelCase     :		List[Any]							     =		nn.Linear(__lowerCAmelCase	,				__lowerCAmelCase	,				bias=__lowerCAmelCase		)
							UpperCAmelCase     :		str							     =		emb.weight.data
							return lin_layer



def       __lowerCamelCase  (					_lowercase		)      ->			Dict:

							with open(__lowerCAmelCase	,				"""r"""	,				encoding="""utf-8"""		) as f:
														UpperCAmelCase     :		Union[str, Any]							     =		f.readlines()
														UpperCAmelCase     :		List[str]							     =		[line.split(""" """		)[0] for line in lines]

							UpperCAmelCase     :		Optional[Any]							     =		len(__lowerCAmelCase		)

							UpperCAmelCase     :		Dict							     =		{
							    '''<s>''': 0,
							    '''<pad>''': 1,
							    '''</s>''': 2,
							    '''<unk>''': 3,
							}

							vocab_dict.update(dict(zip(__lowerCAmelCase	,				range(4	,				num_words + 4		)		)		)		)
							return vocab_dict



@torch.no_grad()
def       __lowerCamelCase  (					_lowercase	,				_lowercase	,				_lowercase	,				_lowercase	,				_lowercase	,				_lowercase	,				_lowercase	,				)      ->			List[Any]:
							UpperCAmelCase     :		List[Any]							     =		WavaVecaConfig.from_pretrained(__lowerCAmelCase		)
							UpperCAmelCase     :		Optional[int]							     =		SpeechaTextaConfig.from_pretrained(
							    __lowerCAmelCase	,				vocab_size=__lowerCAmelCase	,				decoder_layers=__lowerCAmelCase	,				do_stable_layer_norm=__lowerCAmelCase		)

							UpperCAmelCase     :		int							     =		WavaVecaFeatureExtractor(
							    feature_size=1	,				sampling_rate=1_6_0_0_0	,				padding_value=0	,				do_normalize=__lowerCAmelCase	,				return_attention_mask=__lowerCAmelCase	,				)

							UpperCAmelCase     :		List[str]							     =		fairseq.checkpoint_utils.load_model_ensemble_and_task(
							    [checkpoint_path]	,				arg_overrides={"""data""": """/""".join(dict_path.split("""/"""		)[:-1]		)}		)
							UpperCAmelCase     :		int							     =		model[0].eval()

							# set weights for wav2vec2 encoder
							UpperCAmelCase     :		Union[str, Any]							     =		WavaVecaModel(__lowerCAmelCase		)
							UpperCAmelCase     :		Tuple							     =		recursively_load_weights_wavaveca(model.encoder	,				__lowerCAmelCase		)

							UpperCAmelCase     :		Any							     =		SpeechaTextaForCausalLM(__lowerCAmelCase		)
							UpperCAmelCase     :		List[Any]							     =		hf_decoder.model.decoder.load_state_dict(model.decoder.state_dict()	,				strict=__lowerCAmelCase		)

							# set output linear layer
							unexpected_keys.remove("""embed_out"""		)
							UpperCAmelCase     :		str							     =		nn.Parameter(model.decoder.embed_out.detach()		)

							# layer norm is init to identity matrix so leaving it is fine
							logger.warning(F'''The following keys are missing when loading the decoder weights: {missing_keys}'''		)
							logger.warning(F'''The following keys are unexpected when loading the decoder weights: {unexpected_keys}'''		)

							UpperCAmelCase     :		Union[str, Any]							     =		SpeechEncoderDecoderModel(encoder=__lowerCAmelCase	,				decoder=__lowerCAmelCase		)
							UpperCAmelCase     :		List[str]							     =		False

							# add projection layer
							UpperCAmelCase     :		Dict							     =		nn.Parameter(projection_layer.weight		)
							UpperCAmelCase     :		str							     =		nn.Parameter(projection_layer.bias		)

							UpperCAmelCase     :		List[str]							     =		create_vocab_dict(__lowerCAmelCase		)

							with open(os.path.join(__lowerCAmelCase	,				"""vocab.json"""		)	,				"""w"""		) as fp:
														json.dump(__lowerCAmelCase	,				__lowerCAmelCase		)

							UpperCAmelCase     :		str							     =		SpeechaTextaTokenizer(os.path.join(__lowerCAmelCase	,				"""vocab.json"""		)		)
							tokenizer.save_pretrained(__lowerCAmelCase		)

							UpperCAmelCase     :		Optional[Any]							     =		hf_wavavec.config.to_dict()
							UpperCAmelCase     :		Optional[Any]							     =		tokenizer.pad_token_id
							UpperCAmelCase     :		Optional[int]							     =		tokenizer.bos_token_id
							UpperCAmelCase     :		Optional[int]							     =		tokenizer.eos_token_id
							UpperCAmelCase     :		Union[str, Any]							     =		'''speech_to_text_2'''
							UpperCAmelCase     :		Any							     =		'''wav2vec2'''

							UpperCAmelCase     :		Optional[Any]							     =		SpeechEncoderDecoderConfig.from_dict(__lowerCAmelCase		)

							hf_wavavec.save_pretrained(__lowerCAmelCase		)
							feature_extractor.save_pretrained(__lowerCAmelCase		)


if __name__ == "__main__":
							a   :			Dict					  =    argparse.ArgumentParser()
							parser.add_argument("""--pytorch_dump_folder_path""", default=None, type=str, help="""Path to the output PyTorch model.""")
							parser.add_argument("""--checkpoint_path""", default=None, type=str, help="""Path to fairseq checkpoint""")
							parser.add_argument("""--dict_path""", default=None, type=str, help="""Path to dict of fine-tuned model""")
							parser.add_argument(
							    """--encoder_config_path""",
							    default="""facebook/wav2vec2-large-lv60""",
							    type=str,
							    help="""Path to hf encoder wav2vec2 checkpoint config""",
							)
							parser.add_argument(
							    """--decoder_config_path""",
							    default="""facebook/s2t-small-mustc-en-fr-st""",
							    type=str,
							    help="""Path to hf decoder s2t checkpoint config""",
							)
							parser.add_argument("""--vocab_size""", default=1_0_2_2_4, type=int, help="""Vocab size of decoder""")
							parser.add_argument("""--num_decoder_layers""", default=7, type=int, help="""Number of decoder layers""")

							a   :			Tuple					  =    parser.parse_args()
							convert_wavaveca_checkpoint(
							    args.checkpoint_path,
							    args.pytorch_dump_folder_path,
							    args.dict_path,
							    encoder_config_path=args.encoder_config_path,
							    decoder_config_path=args.decoder_config_path,
							    vocab_size=args.vocab_size,
							    num_decoder_layers=args.num_decoder_layers,
							)






import json
import multiprocessing
import os
import re
from collections import defaultdict

import torch
from accelerate import Accelerator
from accelerate.utils import set_seed
from arguments import HumanEvalArguments
from datasets import load_dataset, load_metric
from torch.utils.data import IterableDataset
from torch.utils.data.dataloader import DataLoader
from tqdm import tqdm

import transformers
from transformers import AutoModelForCausalLM, AutoTokenizer, HfArgumentParser, StoppingCriteria, StoppingCriteriaList


A__								=						['''\nclass''', '''\ndef''', '''\n#''', '''\n@''', '''\nprint''', '''\nif''']


class       a							(			__lowerCamelCase				):



		def __init__(     self     :str   ,__lowercase     :Dict   ,__lowercase     :str   ,__lowercase     :Any=None   ,__lowercase     :Union[str, Any]=1  ):
						snake_case__	:      Optional[int]     		=	tokenizer
						snake_case__	:      List[Any]     		=	dataset
						snake_case__	:      Any     		=	len(__lowercase  ) if n_tasks is None else n_tasks
						snake_case__	:      List[Any]     		=	n_copies






		def __iter__(     self     :Tuple  ):
						snake_case__	:      Union[str, Any]     		=	[]
						for task in range(self.n_tasks  ):
										# without strip, the model generate commented codes ...
										prompts.append(self.tokenizer.eos_token + self.dataset[task]['''prompt'''].strip()  )
						snake_case__	:      Optional[int]     		=	self.tokenizer(__lowercase   ,padding=__lowercase   ,return_tensors='''pt'''  )
						for task in range(self.n_tasks  ):
										for _ in range(self.n_copies  ):
														yield {
														    "ids": outputs.input_ids[task],
														    "task_id": task,
														    "input_len": outputs.attention_mask[task].sum(),
														}


class       a							(			__lowerCamelCase				):



		def __init__(     self     :Union[str, Any]   ,__lowercase     :List[Any]   ,__lowercase     :int   ,__lowercase     :List[str]  ):
						snake_case__	:      List[str]     		=	start_length
						snake_case__	:      Any     		=	eof_strings
						snake_case__	:      Optional[Any]     		=	tokenizer






		def __call__(     self     :str   ,__lowercase     :str   ,__lowercase     :Any   ,**__lowercase     :Optional[int]  ):
						snake_case__	:      Tuple     		=	self.tokenizer.batch_decode(input_ids[:, self.start_length :]  )
						snake_case__	:      Union[str, Any]     		=	[]
						for decoded_generation in decoded_generations:
										done.append(any(stop_string in decoded_generation for stop_string in self.eof_strings  )  )
						return all(__lowercase  )



def 			_lowerCAmelCase       (					__lowerCAmelCase      )   ->					List[Any]:


				"""simple docstring"""

				snake_case__	:      List[Any]     		=	re.split('''(%s)''' % '''|'''.join(__lowerCAmelCase      )  ,  __lowerCAmelCase      )
				# last string should be ""
				return "".join(string_list[:-2]      )



def 			_lowerCAmelCase       (					__lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase=20  ,  **__lowerCAmelCase      )   ->					List[Any]:


				"""simple docstring"""

				snake_case__	:      Optional[int]     		=	defaultdict(__lowerCAmelCase      )  # dict of list of generated tokens
				for step, batch in tqdm(enumerate(__lowerCAmelCase      )      ):
								with torch.no_grad():
												snake_case__	:      List[Any]     		=	batch['''ids'''].shape[-1]
												snake_case__	:      str     		=	accelerator.unwrap_model(__lowerCAmelCase      ).generate(
												    input_ids=batch['''ids'''][:, : batch['''input_len''']]  ,  num_return_sequences=__lowerCAmelCase  ,  **__lowerCAmelCase      )
												# each task is generated batch_size times
												snake_case__	:      str     		=	batch['''task_id'''].repeat(__lowerCAmelCase      )
												snake_case__	:      Optional[int]     		=	accelerator.pad_across_processes(
												    __lowerCAmelCase  ,  dim=1  ,  pad_index=tokenizer.pad_token_id      )

												snake_case__		, snake_case__	:      Any     		=	accelerator.gather((generated_tokens, generated_tasks)      )
												snake_case__	:      Union[str, Any]     		=	generated_tokens.cpu().numpy()
												snake_case__	:      Optional[Any]     		=	generated_tasks.cpu().numpy()

												for task, generated_tokens in zip(__lowerCAmelCase  ,  __lowerCAmelCase      ):
																gen_token_dict[task].append(__lowerCAmelCase      )

				snake_case__	:      List[Any]     		=	[[] for _ in range(__lowerCAmelCase      )]
				for task, generated_tokens in gen_token_dict.items():
								for s in generated_tokens:
												snake_case__	:      int     		=	tokenizer.decode(__lowerCAmelCase  ,  skip_special_tokens=__lowerCAmelCase  ,  clean_up_tokenization_spaces=__lowerCAmelCase      )
												code_gens[task].append(remove_last_block(__lowerCAmelCase      )      )
				return code_gens



def 			_lowerCAmelCase       (					)   ->					Optional[int]:


				"""simple docstring"""

				snake_case__	:      List[str]     		=	HfArgumentParser(__lowerCAmelCase      )
				snake_case__	:      Optional[Any]     		=	parser.parse_args()

				transformers.logging.set_verbosity_error()
				# enables code execution in code_eval metric
				snake_case__	:      Union[str, Any]     		=	args.HF_ALLOW_CODE_EVAL
				# make sure tokenizer plays nice with multiprocessing
				snake_case__	:      Union[str, Any]     		=	'''false'''

				if args.num_workers is None:
								snake_case__	:      Union[str, Any]     		=	multiprocessing.cpu_count()

				# Use dataset load to feed to accelerate
				snake_case__	:      Dict     		=	Accelerator()
				set_seed(args.seed  ,  device_specific=__lowerCAmelCase      )

				# Load model and tokenizer
				snake_case__	:      str     		=	AutoTokenizer.from_pretrained(args.model_ckpt      )
				snake_case__	:      List[str]     		=	tokenizer.eos_token
				snake_case__	:      Optional[Any]     		=	AutoModelForCausalLM.from_pretrained(args.model_ckpt      )

				# Generation settings
				snake_case__	:      List[Any]     		=	{
				    '''do_sample''': args.do_sample,
				    '''temperature''': args.temperature,
				    '''max_new_tokens''': args.max_new_tokens,
				    '''top_p''': args.top_p,
				    '''top_k''': args.top_k,
				    '''stopping_criteria''': StoppingCriteriaList([EndOfFunctionCriteria(0  ,  __lowerCAmelCase  ,  __lowerCAmelCase      )]      ),
				}

				# Load evaluation dataset and metric
				snake_case__	:      List[str]     		=	load_dataset('''openai_humaneval'''      )
				snake_case__	:      str     		=	load_metric('''code_eval'''      )

				snake_case__	:      List[Any]     		=	args.num_tasks if args.num_tasks is not None else len(human_eval['''test''']      )
				snake_case__	:      str     		=	args.n_samples // args.batch_size

				snake_case__	:      Tuple     		=	TokenizedDataset(__lowerCAmelCase  ,  human_eval['''test''']  ,  n_copies=__lowerCAmelCase  ,  n_tasks=__lowerCAmelCase      )
				# do not confuse args.batch_size, which is actually the num_return_sequences
				snake_case__	:      Tuple     		=	DataLoader(__lowerCAmelCase  ,  batch_size=1      )

				# Run a quick test to see if code evaluation is enabled
				try:
								snake_case__	:      Optional[Any]     		=	code_eval_metric.compute(references=['''''']  ,  predictions=[['''''']]      )
				except ValueError as exception:
								print(
								    '''Code evaluation not enabled. Read the warning below carefully and then use `--HF_ALLOW_CODE_EVAL="1"`'''
								    ''' flag to enable code evaluation.'''      )
								raise exception

				snake_case__		, snake_case__	:      Optional[Any]     		=	accelerator.prepare(__lowerCAmelCase  ,  __lowerCAmelCase      )

				snake_case__	:      Dict     		=	complete_code(
				    __lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase  ,  __lowerCAmelCase  ,  n_tasks=__lowerCAmelCase  ,  batch_size=args.batch_size  ,  **__lowerCAmelCase  ,  )

				if accelerator.is_main_process:
								snake_case__	:      Optional[Any]     		=	[]

								for task in tqdm(range(__lowerCAmelCase      )      ):
												snake_case__	:      str     		=	human_eval['''test'''][task]['''test''']
												snake_case__	:      int     		=	f"""check({human_eval['test'][task]['entry_point']})"""
												references.append('''\n''' + test_func + '''\n''' + entry_point      )

								# Evaluate completions with "code_eval" metric
								snake_case__		, snake_case__	:      Any     		=	code_eval_metric.compute(
								    references=__lowerCAmelCase  ,  predictions=__lowerCAmelCase  ,  num_workers=args.num_workers      )
								print(f"""Results: {pass_at_k}"""      )

								# Save results to json file
								with open(args.output_file  ,  '''w'''      ) as fp:
												json.dump(__lowerCAmelCase  ,  __lowerCAmelCase      )


# For some reason the folliwng seems to be necessary sometimes for code_eval to work nice with multiprocessing
# https://stackoverflow.com/questions/60804599/python-multiprocessing-keeps-spawning-the-whole-script
if __name__ == "__main__":
				main()

