code: 
  
  
  
  import argparse
  import os
  import pathlib
  
  import fairseq
  import torch
  from fairseq.modules import TransformerSentenceEncoderLayer
  from packaging import version
  
  from transformers import (
      DataaVecTextConfig,
      DataaVecTextForMaskedLM,
      DataaVecTextForSequenceClassification,
      DataaVecTextModel,
  )
  from transformers.models.bert.modeling_bert import (
      BertIntermediate,
      BertLayer,
      BertOutput,
      BertSelfAttention,
      BertSelfOutput,
  )
  
  # IMPORTANT: In order for this script to run, please make sure to download the dictionary: `dict.txt` from wget https://dl.fbaipublicfiles.com/fairseq/models/roberta.large.tar.gz
  # File copied from https://github.com/pytorch/fairseq/blob/main/examples/data2vec/models/data2vec_text.py
  from transformers.utils import logging
  
  
  if version.parse(fairseq.__version__) < version.parse('''0.9.0'''):
  			raise Exception('''requires fairseq >= 0.9.0''')
  
  
  logging.set_verbosity_info()
  lowerCAmelCase__	:  Union[str, Any] 		=logging.get_logger(__name__)
  
  lowerCAmelCase__	:  Tuple 		='''Hello world! cécé herlolip'''
  
  
  
  
  
  
  def 			__lowercase     (				a__							,     a__							,     a__     )       ->				int:
  							__SCREAMING_SNAKE_CASE							, __SCREAMING_SNAKE_CASE								=      os.path.split(a__     )
  							__SCREAMING_SNAKE_CASE								=      DataaVecTextModel.from_pretrained(
  							    a__							,     checkpoint_file=a__     )
  							dataavec.eval()  # disable dropout
  							__SCREAMING_SNAKE_CASE								=      dataavec.models[0]
  							__SCREAMING_SNAKE_CASE								=      dataavec_model.encoder.sentence_encoder
  							__SCREAMING_SNAKE_CASE								=      DataaVecTextConfig(
  							    vocab_size=dataavec_sent_encoder.embed_tokens.num_embeddings							,     hidden_size=dataavec_model.args.encoder_embed_dim							,     num_hidden_layers=dataavec_model.args.encoder_layers							,     num_attention_heads=dataavec_model.args.encoder_attention_heads							,     intermediate_size=dataavec_model.args.encoder_ffn_embed_dim							,     max_position_embeddings=5_14							,     type_vocab_size=1							,     layer_norm_eps=1E-5							,     )
  							if classification_head:
  														__SCREAMING_SNAKE_CASE								=      dataavec.model.classification_heads['mnli'].out_proj.weight.shape[0]
  							print('Our BERT config:'							,     a__     )
  
  							__SCREAMING_SNAKE_CASE								=      DataaVecTextForSequenceClassification(a__     ) if classification_head else DataaVecTextForMaskedLM(a__     )
  							model.eval()
  
  							# Now let's copy all the weights.
  							# Embeddings
  							__SCREAMING_SNAKE_CASE								=      dataavec_sent_encoder.embed_tokens.weight
  							__SCREAMING_SNAKE_CASE								=      dataavec_sent_encoder.embed_positions.weight
  							__SCREAMING_SNAKE_CASE								=      torch.zeros_like(
  							    model.dataavec_text.embeddings.token_type_embeddings.weight     )  # just zero them out b/c data2vec doesn't use them.
  							__SCREAMING_SNAKE_CASE								=      dataavec_sent_encoder.layernorm_embedding.weight
  							__SCREAMING_SNAKE_CASE								=      dataavec_sent_encoder.layernorm_embedding.bias
  
  							for i in range(config.num_hidden_layers     ):
  														# Encoder: start of layer
  														__SCREAMING_SNAKE_CASE								=      model.dataavec_text.encoder.layer[i]
  														__SCREAMING_SNAKE_CASE								=      dataavec_sent_encoder.layers[i]
  
  														# self attention
  														__SCREAMING_SNAKE_CASE								=      layer.attention.self
  														assert dataavec_layer.self_attn.k_proj.weight.data.shape == torch.Size(
  														    (config.hidden_size, config.hidden_size)     ), (
  														    "Shape for data2vec_layer.self_attn.k_proj.weight.data should be"
  														    f""" {torch.Size((config.hidden_size, config.hidden_size)     )}"""
  														)
  														assert dataavec_layer.self_attn.q_proj.weight.data.shape == torch.Size(
  														    (config.hidden_size, config.hidden_size)     ), (
  														    "Shape for data2vec_layer.self_attn.q_proj.weight.data should be"
  														    f""" {torch.Size((config.hidden_size, config.hidden_size)     )}"""
  														)
  														assert dataavec_layer.self_attn.v_proj.weight.data.shape == torch.Size(
  														    (config.hidden_size, config.hidden_size)     ), (
  														    "Shape for data2vec_layer.self_attn.v_proj.weight.data should be"
  														    f""" {torch.Size((config.hidden_size, config.hidden_size)     )}"""
  														)
  
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.q_proj.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.q_proj.bias
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.k_proj.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.k_proj.bias
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.v_proj.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.v_proj.bias
  
  														# self-attention output
  														__SCREAMING_SNAKE_CASE								=      layer.attention.output
  														assert (
  														    self_output.dense.weight.shape == dataavec_layer.self_attn.out_proj.weight.shape
  														), f"""Shape for self_output.dense.weight should be {dataavec_layer.self_attn.out_proj.weight.shape}"""
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.out_proj.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn.out_proj.bias
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn_layer_norm.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.self_attn_layer_norm.bias
  
  														# intermediate
  														__SCREAMING_SNAKE_CASE								=      layer.intermediate
  														assert (
  														    intermediate.dense.weight.shape == dataavec_layer.fca.weight.shape
  														), f"""Shape for intermediate.dense.weight should be {dataavec_layer.fca.weight.shape}"""
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.fca.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.fca.bias
  
  														# output
  														__SCREAMING_SNAKE_CASE								=      layer.output
  														assert (
  														    bert_output.dense.weight.shape == dataavec_layer.fca.weight.shape
  														), f"""Shape for bert_output.dense.weight should be {dataavec_layer.fca.weight.shape}"""
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.fca.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.fca.bias
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.final_layer_norm.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_layer.final_layer_norm.bias
  														# end of layer
  
  							if classification_head:
  														__SCREAMING_SNAKE_CASE								=      dataavec.model.classification_heads['mnli'].dense.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec.model.classification_heads['mnli'].dense.bias
  														__SCREAMING_SNAKE_CASE								=      dataavec.model.classification_heads['mnli'].out_proj.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec.model.classification_heads['mnli'].out_proj.bias
  							else:
  														# LM Head
  														__SCREAMING_SNAKE_CASE								=      dataavec_model.encoder.lm_head.dense.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_model.encoder.lm_head.dense.bias
  														__SCREAMING_SNAKE_CASE								=      dataavec_model.encoder.lm_head.layer_norm.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_model.encoder.lm_head.layer_norm.bias
  														__SCREAMING_SNAKE_CASE								=      dataavec_model.encoder.lm_head.weight
  														__SCREAMING_SNAKE_CASE								=      dataavec_model.encoder.lm_head.bias
  
  							# Let's check that we get the same results.
  							__SCREAMING_SNAKE_CASE								=      dataavec.encode(a__     ).unsqueeze(0     )  # batch of size 1
  
  							__SCREAMING_SNAKE_CASE								=      model(a__     )[0]
  							if classification_head:
  														__SCREAMING_SNAKE_CASE								=      dataavec.model.classification_heads['mnli'](dataavec.extract_features(a__     )     )
  							else:
  														__SCREAMING_SNAKE_CASE								=      dataavec_model(a__     )[0]
  							print(our_output.shape							,     their_output.shape     )
  							__SCREAMING_SNAKE_CASE								=      torch.max(torch.abs(our_output - their_output     )     ).item()
  							print(f"""max_absolute_diff = {max_absolute_diff}"""     )  # ~ 1e-7
  							__SCREAMING_SNAKE_CASE								=      torch.allclose(a__							,     a__							,     atol=1E-3     )
  							print('Do both models output the same tensors?'							,     '🔥' if success else '💩'     )
  							if not success:
  														raise Exception('Something went wRoNg'     )
  
  							pathlib.Path(a__     ).mkdir(parents=a__							,     exist_ok=a__     )
  							print(f"""Saving model to {pytorch_dump_folder_path}"""     )
  							model.save_pretrained(a__     )
  
  
  if __name__ == "__main__":
  			lowerCAmelCase__	:  Optional[Any] 		=argparse.ArgumentParser()
  			# Required parameters
  			parser.add_argument(
  			    '''--checkpoint_path''', default=None, type=str, required=True, help='''Path the official PyTorch dump.'''
  			)
  			parser.add_argument(
  			    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
  			)
  			parser.add_argument(
  			    '''--classification_head''', action='''store_true''', help='''Whether to convert a final classification head.'''
  			)
  			lowerCAmelCase__	:  Union[str, Any] 		=parser.parse_args()
  			convert_dataavec_checkpoint_to_pytorch(
  			    args.checkpoint_path, args.pytorch_dump_folder_path, args.classification_head
  			)
  
  
  
  
  
  
  
style_context: 
  
  
  
  import argparse
  import datetime
  import json
  import time
  import warnings
  from logging import getLogger
  from pathlib import Path
  from typing import Dict, List
  
  import torch
  from tqdm import tqdm
  
  from transformers import AutoModelForSeqaSeqLM, AutoTokenizer
  from utils import calculate_bleu, calculate_rouge, chunks, parse_numeric_n_bool_cl_kwargs, use_task_specific_params
  
  
  lowerCAmelCase__	:  Union[str, Any] 		=getLogger(__name__)
  
  
  lowerCAmelCase__	:  Optional[int] 		='''cuda''' if torch.cuda.is_available() else '''cpu'''
  
  
  
  
  
  
  def 			__lowercase     (				a__							,     a__							,     a__							,     a__ = 8							,     a__ = DEFAULT_DEVICE							,     a__=False							,     a__="summarization"							,     a__=None							,     **a__							,     )       ->				Dict:
  							__SCREAMING_SNAKE_CASE								=      Path(a__     ).open('w'							,     encoding='utf-8'     )
  							__SCREAMING_SNAKE_CASE								=      str(a__     )
  							__SCREAMING_SNAKE_CASE								=      AutoModelForSeqaSeqLM.from_pretrained(a__     ).to(a__     )
  							if fpaa:
  														__SCREAMING_SNAKE_CASE								=      model.half()
  
  							__SCREAMING_SNAKE_CASE								=      AutoTokenizer.from_pretrained(a__     )
  							logger.info(f"""Inferred tokenizer type: {tokenizer.__class__}"""     )  # if this is wrong, check config.model_type.
  
  							__SCREAMING_SNAKE_CASE								=      time.time()
  							# update config with task specific params
  							use_task_specific_params(a__							,     a__     )
  							if prefix is None:
  														__SCREAMING_SNAKE_CASE								=      prefix or getattr(model.config							,     'prefix'							,     ''     ) or ''
  							for examples_chunk in tqdm(list(chunks(a__							,     a__     )     )     ):
  														__SCREAMING_SNAKE_CASE								=      [prefix + text for text in examples_chunk]
  														__SCREAMING_SNAKE_CASE								=      tokenizer(a__							,     return_tensors='pt'							,     truncation=a__							,     padding='longest'     ).to(a__     )
  														__SCREAMING_SNAKE_CASE								=      model.generate(
  														    input_ids=batch.input_ids							,     attention_mask=batch.attention_mask							,     **a__							,     )
  														__SCREAMING_SNAKE_CASE								=      tokenizer.batch_decode(a__							,     skip_special_tokens=a__							,     clean_up_tokenization_spaces=a__     )
  														for hypothesis in dec:
  																					fout.write(hypothesis + '\n'     )
  																					fout.flush()
  							fout.close()
  							__SCREAMING_SNAKE_CASE								=      int(time.time() - start_time     )  # seconds
  							__SCREAMING_SNAKE_CASE								=      len(a__     )
  							return {"n_obs": n_obs, "runtime": runtime, "seconds_per_sample": round(runtime / n_obs							,     4     )}
  
  
  
  
  
  
  def 			__lowercase     (				)       ->				Dict:
  							return datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'     )
  
  
  
  
  
  
  def 			__lowercase     (				a__=True     )       ->				Optional[Any]:
  							__SCREAMING_SNAKE_CASE								=      argparse.ArgumentParser()
  							parser.add_argument('model_name'							,     type=a__							,     help='like facebook/bart-large-cnn,t5-base, etc.'     )
  							parser.add_argument('input_path'							,     type=a__							,     help='like cnn_dm/test.source'     )
  							parser.add_argument('save_path'							,     type=a__							,     help='where to save summaries'     )
  							parser.add_argument('--reference_path'							,     type=a__							,     required=a__							,     help='like cnn_dm/test.target'     )
  							parser.add_argument('--score_path'							,     type=a__							,     required=a__							,     default='metrics.json'							,     help='where to save metrics'     )
  							parser.add_argument('--device'							,     type=a__							,     required=a__							,     default=a__							,     help='cuda, cuda:1, cpu etc.'     )
  							parser.add_argument(
  							    '--prefix'							,     type=a__							,     required=a__							,     default=a__							,     help='will be added to the begininng of src examples'     )
  							parser.add_argument('--task'							,     type=a__							,     default='summarization'							,     help='used for task_specific_params + metrics'     )
  							parser.add_argument('--bs'							,     type=a__							,     default=8							,     required=a__							,     help='batch size'     )
  							parser.add_argument(
  							    '--n_obs'							,     type=a__							,     default=-1							,     required=a__							,     help='How many observations. Defaults to all.'     )
  							parser.add_argument('--fp16'							,     action='store_true'     )
  							parser.add_argument('--dump-args'							,     action='store_true'							,     help='print the custom hparams with the results'     )
  							parser.add_argument(
  							    '--info'							,     nargs='?'							,     type=a__							,     const=datetime_now()							,     help=(
  							        'use in conjunction w/ --dump-args to print with the results whatever other info you\'d like, e.g.'
  							        ' lang=en-ru. If no value is passed, the current datetime string will be used.'
  							    )							,     )
  							# Unspecified args like --num_beams=2 --decoder_start_token_id=4 are passed to model.generate
  							__SCREAMING_SNAKE_CASE							, __SCREAMING_SNAKE_CASE								=      parser.parse_known_args()
  							__SCREAMING_SNAKE_CASE								=      parse_numeric_n_bool_cl_kwargs(a__     )
  							if parsed_args and verbose:
  														print(f"""parsed the following generate kwargs: {parsed_args}"""     )
  							with open(args.input_path     ) as f:
  														__SCREAMING_SNAKE_CASE								=      [' ' + x.rstrip() if 't5' in args.model_name else x.rstrip() for x in f.readlines()]
  							if args.n_obs > 0:
  														__SCREAMING_SNAKE_CASE								=      examples[: args.n_obs]
  							Path(args.save_path     ).parent.mkdir(exist_ok=a__     )
  							if args.reference_path is None and Path(args.score_path     ).exists():
  														warnings.warn(f"""score_path {args.score_path} will be overwritten unless you type ctrl-c."""     )
  							__SCREAMING_SNAKE_CASE								=      generate_summaries_or_translations(
  							    a__							,     args.save_path							,     args.model_name							,     batch_size=args.bs							,     device=args.device							,     fpaa=args.fpaa							,     task=args.task							,     prefix=args.prefix							,     **a__							,     )
  
  							if args.reference_path is None:
  														return {}
  
  							# Compute scores
  							__SCREAMING_SNAKE_CASE								=      calculate_bleu if 'translation' in args.task else calculate_rouge
  							__SCREAMING_SNAKE_CASE								=      [x.rstrip() for x in open(args.save_path     ).readlines()]
  							__SCREAMING_SNAKE_CASE								=      [x.rstrip() for x in open(args.reference_path     ).readlines()][: len(a__     )]
  							__SCREAMING_SNAKE_CASE								=      score_fn(a__							,     a__     )
  							scores.update(a__     )
  
  							if args.dump_args:
  														scores.update(a__     )
  							if args.info:
  														__SCREAMING_SNAKE_CASE								=      args.info
  
  							if verbose:
  														print(a__     )
  
  							if args.score_path is not None:
  														json.dump(a__							,     open(args.score_path							,     'w'     )     )
  
  							return scores
  
  
  if __name__ == "__main__":
  			# Usage for MT:
  			# python run_eval.py MODEL_NAME $DATA_DIR/test.source $save_dir/test_translations.txt --reference_path $DATA_DIR/test.target --score_path $save_dir/test_bleu.json  --task translation $@
  			run_generate(verbose=True)
  
  
  
  
  
  
  