





"""simple docstring"""




# Usage:
# ./gen-card-facebook-wmt19.py

import os
from pathlib import Path






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    Optional[int]    ,       _lowercase						:    Tuple    ,       _lowercase						:    Any					) ->Optional[Any]:






	'''simple docstring'''






	a      :		Any	=				{
	    "en": "Machine learning is great, isn't it?",
	    "ru": "Машинное обучение - это здорово, не так ли?",
	    "de": "Maschinelles Lernen ist großartig, oder?",
	}

	# BLUE scores as follows:
	# "pair": [fairseq, transformers]
	a      :		Union[str, Any]	=				{
	    "ru-en": ["[41.3](http://matrix.statmt.org/matrix/output/1907?run_id=6937)", "39.20"],
	    "en-ru": ["[36.4](http://matrix.statmt.org/matrix/output/1914?run_id=6724)", "33.47"],
	    "en-de": ["[43.1](http://matrix.statmt.org/matrix/output/1909?run_id=6862)", "42.83"],
	    "de-en": ["[42.3](http://matrix.statmt.org/matrix/output/1902?run_id=6750)", "41.35"],
	}
	a      :		List[Any]	=				F"""{src_lang}-{tgt_lang}"""

	a      :		Optional[int]	=				F"""
---
language: 
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- facebook
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of [fairseq wmt19 transformer](https://github.com/pytorch/fairseq/blob/master/examples/wmt19/README.md) for {src_lang}-{tgt_lang}.

For more details, please see, [Facebook FAIR's WMT19 News Translation Task Submission](https://arxiv.org/abs/1907.06616).

The abbreviation FSMT stands for FairSeqMachineTranslation

All four models are available:

* [wmt19-en-ru](https://huggingface.co/facebook/wmt19-en-ru)
* [wmt19-ru-en](https://huggingface.co/facebook/wmt19-ru-en)
* [wmt19-en-de](https://huggingface.co/facebook/wmt19-en-de)
* [wmt19-de-en](https://huggingface.co/facebook/wmt19-de-en)

## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = \"facebook/wmt19-{src_lang}-{tgt_lang}\"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = \"{texts[src_lang]}\"
input_ids = tokenizer.encode(input, return_tensors=\"pt\")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias

- The original (and this ported model) doesn't seem to handle well inputs with repeated sub-phrases, [content gets truncated](https://discuss.huggingface.co/t/issues-with-translating-inputs-containing-repeated-phrases/981)

## Training data

Pretrained weights were left identical to the original model released by fairseq. For more details, please, see the [paper](https://arxiv.org/abs/1907.06616).

## Eval results

pair   | fairseq | transformers
-------|---------|----------
{pair}  | {scores[pair][0]} | {scores[pair][1]}

The score is slightly below the score reported by `fairseq`, since `transformers`` currently doesn't support:
- model ensemble, therefore the best performing checkpoint was ported (``model4.pt``).
- re-ranking

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=15
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH=\"src:examples/seq2seq\" python examples/seq2seq/run_eval.py facebook/wmt19-$PAIR $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```
note: fairseq reports using a beam of 50, so you should get a slightly higher score if re-run with `--num_beams 50`.

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```bibtex
@inproceedings{{...,
  year={{2020}},
  title={{Facebook FAIR's WMT19 News Translation Task Submission}},
  author={{Ng, Nathan and Yee, Kyra and Baevski, Alexei and Ott, Myle and Auli, Michael and Edunov, Sergey}},
  booktitle={{Proc. of WMT}},
}}
```


## TODO

- port model ensemble (fairseq uses 4 model checkpoints)

"""
	os.makedirs(_lowercase    ,       exist_ok=_lowercase					)
	a      :		Tuple	=				os.path.join(_lowercase    ,       "README.md"					)
	print(F"""Generating {path}"""					)
	with open(_lowercase    ,       "w"    ,       encoding="utf-8"					) as f:
		f.write(_lowercase					)


# make sure we are under the root of the project
a						: int     	=			Path(__file__).resolve().parent.parent.parent
a						: Union[str, Any]     	=			repo_dir / '''model_cards'''

for model_name in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
		a	, a	, a						: List[str]     	=			model_name.split('''-''')
		a						: List[str]     	=			model_cards_dir / '''facebook''' / model_name
		write_model_card(model_card_dir, src_lang=src_lang, tgt_lang=tgt_lang)




import argparse
import json

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import ViTImageProcessor, ViTMSNConfig, ViTMSNModel
from transformers.image_utils import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD


torch.set_grad_enabled(False)




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[int]  ,   lowerCAmelCase__							:				str=False  ):
 __a :  Any						=  []
 for i in range(config.num_hidden_layers  ):
  # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
  rename_keys.append((f"module.blocks.{i}.norm1.weight", f"vit.encoder.layer.{i}.layernorm_before.weight")  )
  rename_keys.append((f"module.blocks.{i}.norm1.bias", f"vit.encoder.layer.{i}.layernorm_before.bias")  )
  rename_keys.append(
      (f"module.blocks.{i}.attn.proj.weight", f"vit.encoder.layer.{i}.attention.output.dense.weight")  )
  rename_keys.append((f"module.blocks.{i}.attn.proj.bias", f"vit.encoder.layer.{i}.attention.output.dense.bias")  )
  rename_keys.append((f"module.blocks.{i}.norm2.weight", f"vit.encoder.layer.{i}.layernorm_after.weight")  )
  rename_keys.append((f"module.blocks.{i}.norm2.bias", f"vit.encoder.layer.{i}.layernorm_after.bias")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc1.weight", f"vit.encoder.layer.{i}.intermediate.dense.weight")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc1.bias", f"vit.encoder.layer.{i}.intermediate.dense.bias")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc2.weight", f"vit.encoder.layer.{i}.output.dense.weight")  )
  rename_keys.append((f"module.blocks.{i}.mlp.fc2.bias", f"vit.encoder.layer.{i}.output.dense.bias")  )

 # projection layer + position embeddings
 rename_keys.extend(
     [
         ('''module.cls_token''', '''vit.embeddings.cls_token'''),
         ('''module.patch_embed.proj.weight''', '''vit.embeddings.patch_embeddings.projection.weight'''),
         ('''module.patch_embed.proj.bias''', '''vit.embeddings.patch_embeddings.projection.bias'''),
         ('''module.pos_embed''', '''vit.embeddings.position_embeddings'''),
     ]  )

 if base_model:
  # layernorm + pooler
  rename_keys.extend(
      [
          ('''module.norm.weight''', '''layernorm.weight'''),
          ('''module.norm.bias''', '''layernorm.bias'''),
      ]  )

  # if just the base model, we should remove "vit" from all keys that start with "vit"
  __a :  Optional[int]						=  [(pair[0], pair[1][4:]) if pair[1].startswith('''vit'''  ) else pair for pair in rename_keys]
 else:
  # layernorm + classification head
  rename_keys.extend(
      [
          ('''norm.weight''', '''vit.layernorm.weight'''),
          ('''norm.bias''', '''vit.layernorm.bias'''),
          ('''head.weight''', '''classifier.weight'''),
          ('''head.bias''', '''classifier.bias'''),
      ]  )

 return rename_keys




def    __UpperCamelCase     (				lowerCAmelCase__							:				Dict  ,   lowerCAmelCase__							:				Union[str, Any]  ,   lowerCAmelCase__							:				List[Any]=False  ):
 for i in range(config.num_hidden_layers  ):
  if base_model:
   __a :  Dict						=  ''''''
  else:
   __a :  str						=  '''vit.'''
  # read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
  __a :  List[Any]						=  state_dict.pop(f"module.blocks.{i}.attn.qkv.weight"  )
  __a :  Tuple						=  state_dict.pop(f"module.blocks.{i}.attn.qkv.bias"  )
  # next, add query, keys and values (in that order) to the state dict
  __a :  Dict						=  in_proj_weight[
      : config.hidden_size, :
  ]
  __a :  str						=  in_proj_bias[: config.hidden_size]
  __a :  int						=  in_proj_weight[
      config.hidden_size : config.hidden_size * 2, :
  ]
  __a :  Dict						=  in_proj_bias[
      config.hidden_size : config.hidden_size * 2
  ]
  __a :  str						=  in_proj_weight[
      -config.hidden_size :, :
  ]
  __a :  Optional[Any]						=  in_proj_bias[-config.hidden_size :]




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[int]  ):
 __a :  Dict						=  ['''head.weight''', '''head.bias''']
 for k in ignore_keys:
  state_dict.pop(lowerCAmelCase__  ,   lowerCAmelCase__  )




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[Any]  ):
 # projection head is used in the self-supervised pre-training in MSN,
 # for downstream task it's not needed.
 __a :  str						=  [
     '''module.fc.fc1.weight''',
     '''module.fc.fc1.bias''',
     '''module.fc.bn1.weight''',
     '''module.fc.bn1.bias''',
     '''module.fc.bn1.running_mean''',
     '''module.fc.bn1.running_var''',
     '''module.fc.bn1.num_batches_tracked''',
     '''module.fc.fc2.weight''',
     '''module.fc.fc2.bias''',
     '''module.fc.bn2.weight''',
     '''module.fc.bn2.bias''',
     '''module.fc.bn2.running_mean''',
     '''module.fc.bn2.running_var''',
     '''module.fc.bn2.num_batches_tracked''',
     '''module.fc.fc3.weight''',
     '''module.fc.fc3.bias''',
 ]
 for k in ignore_keys:
  state_dict.pop(lowerCAmelCase__  ,   lowerCAmelCase__  )




def    __UpperCamelCase     (				lowerCAmelCase__							:				Dict  ,   lowerCAmelCase__							:				List[Any]  ,   lowerCAmelCase__							:				List[Any]  ):
 __a :  Optional[Any]						=  dct.pop(lowerCAmelCase__  )
 __a :  Optional[Any]						=  val




def    __UpperCamelCase     (				lowerCAmelCase__							:				Optional[Any]  ,   lowerCAmelCase__							:				str  ):
 __a :  str						=  ViTMSNConfig()
 __a :  List[Any]						=  1_0_0_0

 __a :  Union[str, Any]						=  '''datasets/huggingface/label-files'''
 __a :  Optional[int]						=  '''imagenet-1k-id2label.json'''
 __a :  Optional[int]						=  json.load(open(hf_hub_download(lowerCAmelCase__  ,   lowerCAmelCase__  )  ,   '''r'''  )  )
 __a :  Dict						=  {int(lowerCAmelCase__  ): v for k, v in idalabel.items()}
 __a :  Tuple						=  idalabel
 __a :  int						=  {v: k for k, v in idalabel.items()}

 if "s16" in checkpoint_url:
  __a :  Union[str, Any]						=  3_8_4
  __a :  Union[str, Any]						=  1_5_3_6
  __a :  Union[str, Any]						=  6
 elif "l16" in checkpoint_url:
  __a :  str						=  1_0_2_4
  __a :  Union[str, Any]						=  4_0_9_6
  __a :  Optional[int]						=  2_4
  __a :  int						=  1_6
  __a :  List[Any]						=  0.1
 elif "b4" in checkpoint_url:
  __a :  int						=  4
 elif "l7" in checkpoint_url:
  __a :  int						=  7
  __a :  List[Any]						=  1_0_2_4
  __a :  Union[str, Any]						=  4_0_9_6
  __a :  List[str]						=  2_4
  __a :  int						=  1_6
  __a :  Tuple						=  0.1

 __a :  Dict						=  ViTMSNModel(lowerCAmelCase__  )

 __a :  Union[str, Any]						=  torch.hub.load_state_dict_from_url(lowerCAmelCase__  ,   map_location='''cpu'''  )['''target_encoder''']

 __a :  Optional[int]						=  ViTImageProcessor(size=config.image_size  )

 remove_projection_head(lowerCAmelCase__  )
 __a :  str						=  create_rename_keys(lowerCAmelCase__  ,   base_model=lowerCAmelCase__  )

 for src, dest in rename_keys:
  rename_key(lowerCAmelCase__  ,   lowerCAmelCase__  ,   lowerCAmelCase__  )
 read_in_q_k_v(lowerCAmelCase__  ,   lowerCAmelCase__  ,   base_model=lowerCAmelCase__  )

 model.load_state_dict(lowerCAmelCase__  )
 model.eval()

 __a :  str						=  '''http://images.cocodataset.org/val2017/000000039769.jpg'''

 __a :  str						=  Image.open(requests.get(lowerCAmelCase__  ,   stream=lowerCAmelCase__  ).raw  )
 __a :  Any						=  ViTImageProcessor(
     size=config.image_size  ,   image_mean=lowerCAmelCase__  ,   image_std=lowerCAmelCase__  )
 __a :  Tuple						=  image_processor(images=lowerCAmelCase__  ,   return_tensors='''pt'''  )

 # forward pass
 torch.manual_seed(2  )
 __a :  str						=  model(**lowerCAmelCase__  )
 __a :  List[Any]						=  outputs.last_hidden_state

 # The following Colab Notebook was used to generate these outputs:
 # https://colab.research.google.com/gist/sayakpaul/3672419a04f5997827503fd84079bdd1/scratchpad.ipynb
 if "s16" in checkpoint_url:
  __a :  Dict						=  torch.tensor([[-1.09_15, -1.48_76, -1.18_09]]  )
 elif "b16" in checkpoint_url:
  __a :  List[str]						=  torch.tensor([[14.28_89, -18.90_45, 11.72_81]]  )
 elif "l16" in checkpoint_url:
  __a :  str						=  torch.tensor([[41.50_28, -22.86_81, 45.64_75]]  )
 elif "b4" in checkpoint_url:
  __a :  Union[str, Any]						=  torch.tensor([[-4.38_68, 5.29_32, -0.41_37]]  )
 else:
  __a :  Dict						=  torch.tensor([[-0.17_92, -0.64_65, 2.42_63]]  )

 # verify logits
 assert torch.allclose(last_hidden_state[:, 0, :3]  ,   lowerCAmelCase__  ,   atol=1e-4  )

 print(f"Saving model to {pytorch_dump_folder_path}"  )
 model.save_pretrained(lowerCAmelCase__  )

 print(f"Saving image processor to {pytorch_dump_folder_path}"  )
 image_processor.save_pretrained(lowerCAmelCase__  )


if __name__ == "__main__":
      lowercase__       =argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '--checkpoint_url',
          default='https://dl.fbaipublicfiles.com/msn/vits16_800ep.pth.tar',
          type=str,
          help='URL of the checkpoint you\'d like to convert.',
      )
      parser.add_argument(
          '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
      )

      lowercase__       =parser.parse_args()
      convert_vit_msn_checkpoint(args.checkpoint_url, args.pytorch_dump_folder_path)
