



'''simple docstring'''






# Usage:
# ./gen-card-allenai-wmt19.py

import os
from pathlib import Path




def 					snake_case_      (  lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_ ,     lowerCAmelCase_	)->	str:





       '''simple docstring'''


       _UpperCAmelCase						:		Optional[int]      =				{
           """en""": """Machine learning is great, isn't it?""",
           """ru""": """Машинное обучение - это здорово, не так ли?""",
           """de""": """Maschinelles Lernen ist großartig, nicht wahr?""",
       }

       # BLUE scores as follows:
       # "pair": [fairseq, transformers]
       _UpperCAmelCase						:		List[str]      =				{
           """wmt19-de-en-6-6-base""": [0, 3_8.3_7],
           """wmt19-de-en-6-6-big""": [0, 3_9.9_0],
       }
       _UpperCAmelCase						:		str      =				F'''{src_lang}-{tgt_lang}'''

       _UpperCAmelCase						:		List[Any]      =				F'''
---

language:
- {src_lang}
- {tgt_lang}
thumbnail:
tags:
- translation
- wmt19
- allenai
license: apache-2.0
datasets:
- wmt19
metrics:
- bleu
---

# FSMT

## Model description

This is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.

For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).

2 models are available:

* [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big)
* [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base)


## Intended uses & limitations

#### How to use

```python
from transformers import FSMTForConditionalGeneration, FSMTTokenizer
mname = "allenai/{model_name}"
tokenizer = FSMTTokenizer.from_pretrained(mname)
model = FSMTForConditionalGeneration.from_pretrained(mname)

input = "{texts[src_lang]}"
input_ids = tokenizer.encode(input, return_tensors="pt")
outputs = model.generate(input_ids)
decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
print(decoded) # {texts[tgt_lang]}

```

#### Limitations and bias


## Training data

Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).

## Eval results

Here are the BLEU scores:

model   |  transformers
-------|---------
{model_name}  |  {scores[model_name][1]}

The score was calculated using this code:

```bash
git clone https://github.com/huggingface/transformers
cd transformers
export PAIR={pair}
export DATA_DIR=data/$PAIR
export SAVE_DIR=data/$PAIR
export BS=8
export NUM_BEAMS=5
mkdir -p $DATA_DIR
sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
echo $PAIR
PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
```

## Data Sources

- [training, etc.](http://www.statmt.org/wmt19/)
- [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)


### BibTeX entry and citation info

```
@misc{{kasai2020deep,
    title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
    author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
    year={{2020}},
    eprint={{2006.10369}},
    archivePrefix={{arXiv}},
    primaryClass={{cs.CL}}
}}
```

'''
       model_card_dir.mkdir(parents=lowerCAmelCase_ ,     exist_ok=lowerCAmelCase_	)
       _UpperCAmelCase						:		Optional[int]      =				os.path.join(lowerCAmelCase_ ,     """README.md"""	)
       print(F'''Generating {path}'''	)
       with open(lowerCAmelCase_ ,     """w""" ,     encoding="""utf-8"""	) as f:
              f.write(lowerCAmelCase_	)


# make sure we are under the root of the project
A_							:				Dict						=			Path(__file__).resolve().parent.parent.parent
A_							:				Tuple						=			repo_dir / """model_cards"""

for model_name in ["wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big"]:
   A_							:				Tuple						=			model_cards_dir / """allenai""" / model_name
   write_model_card(model_card_dir, src_lang="""de""", tgt_lang="""en""", model_name=model_name)








'''simple docstring'''






import collections
import inspect
import unittest

from transformers import NatConfig
from transformers.testing_utils import require_natten, require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_backbone_common import BackboneTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
   import torch
   from torch import nn

   from transformers import NatBackbone, NatForImageClassification, NatModel
   from transformers.models.nat.modeling_nat import NAT_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
   from PIL import Image

   from transformers import AutoImageProcessor



class 							lowercase :


 """simple docstring"""


 def __init__(							self						,a_						,a_=13						,a_=64						,a_=4						,a_=3						,a_=16						,a_=[1, 2, 1]						,a_=[2, 4, 8]						,a_=3						,a_=2.0						,a_=True						,a_=0.0						,a_=0.0						,a_=0.1						,a_="gelu"						,a_=True						,a_=0.02						,a_=1E-5						,a_=True						,a_=None						,a_=True						,a_=10						,a_=["stage1", "stage2"]						,a_=[1, 2]						,)	->     List[Any]:
        _UpperCAmelCase						:		Tuple      =				parent
        _UpperCAmelCase						:		Any      =				batch_size
        _UpperCAmelCase						:		Any      =				image_size
        _UpperCAmelCase						:		Optional[Any]      =				patch_size
        _UpperCAmelCase						:		str      =				num_channels
        _UpperCAmelCase						:		List[str]      =				embed_dim
        _UpperCAmelCase						:		str      =				depths
        _UpperCAmelCase						:		Union[str, Any]      =				num_heads
        _UpperCAmelCase						:		List[Any]      =				kernel_size
        _UpperCAmelCase						:		Optional[int]      =				mlp_ratio
        _UpperCAmelCase						:		Tuple      =				qkv_bias
        _UpperCAmelCase						:		Any      =				hidden_dropout_prob
        _UpperCAmelCase						:		Tuple      =				attention_probs_dropout_prob
        _UpperCAmelCase						:		Optional[Any]      =				drop_path_rate
        _UpperCAmelCase						:		Dict      =				hidden_act
        _UpperCAmelCase						:		str      =				patch_norm
        _UpperCAmelCase						:		Tuple      =				layer_norm_eps
        _UpperCAmelCase						:		Tuple      =				initializer_range
        _UpperCAmelCase						:		Any      =				is_training
        _UpperCAmelCase						:		str      =				scope
        _UpperCAmelCase						:		Union[str, Any]      =				use_labels
        _UpperCAmelCase						:		Tuple      =				num_labels
        _UpperCAmelCase						:		Dict      =				out_features
        _UpperCAmelCase						:		List[Any]      =				out_indices







 def _snake_case				(							self							)	->     str:
        _UpperCAmelCase						:		Any      =				floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

        _UpperCAmelCase						:		Union[str, Any]      =				None
        if self.use_labels:
               _UpperCAmelCase						:		List[Any]      =				ids_tensor([self.batch_size]						,self.num_labels							)

        _UpperCAmelCase						:		Optional[int]      =				self.get_config()

        return config, pixel_values, labels







 def _snake_case				(							self							)	->     List[str]:
        return NatConfig(
            num_labels=self.num_labels						,image_size=self.image_size						,patch_size=self.patch_size						,num_channels=self.num_channels						,embed_dim=self.embed_dim						,depths=self.depths						,num_heads=self.num_heads						,kernel_size=self.kernel_size						,mlp_ratio=self.mlp_ratio						,qkv_bias=self.qkv_bias						,hidden_dropout_prob=self.hidden_dropout_prob						,attention_probs_dropout_prob=self.attention_probs_dropout_prob						,drop_path_rate=self.drop_path_rate						,hidden_act=self.hidden_act						,patch_norm=self.patch_norm						,layer_norm_eps=self.layer_norm_eps						,initializer_range=self.initializer_range						,out_features=self.out_features						,out_indices=self.out_indices						,)







 def _snake_case				(							self						,a_						,a_						,a_							)	->     Dict:
        _UpperCAmelCase						:		Any      =				NatModel(config=a_							)
        model.to(a_							)
        model.eval()
        _UpperCAmelCase						:		Any      =				model(a_							)

        _UpperCAmelCase						:		Union[str, Any]      =				(config.image_size // config.patch_size) // (2 ** (len(config.depths							) - 1))
        _UpperCAmelCase						:		List[str]      =				int(config.embed_dim * 2 ** (len(config.depths							) - 1)							)

        self.parent.assertEqual(
            result.last_hidden_state.shape						,(self.batch_size, expected_height, expected_width, expected_dim)							)







 def _snake_case				(							self						,a_						,a_						,a_							)	->     List[str]:
        _UpperCAmelCase						:		Dict      =				NatForImageClassification(a_							)
        model.to(a_							)
        model.eval()
        _UpperCAmelCase						:		int      =				model(a_						,labels=a_							)
        self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_labels)							)

        # test greyscale images
        _UpperCAmelCase						:		Any      =				1
        _UpperCAmelCase						:		Optional[int]      =				NatForImageClassification(a_							)
        model.to(a_							)
        model.eval()

        _UpperCAmelCase						:		List[Any]      =				floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
        _UpperCAmelCase						:		Optional[int]      =				model(a_							)
        self.parent.assertEqual(result.logits.shape						,(self.batch_size, self.num_labels)							)







 def _snake_case				(							self						,a_						,a_						,a_							)	->     Tuple:
        _UpperCAmelCase						:		int      =				NatBackbone(config=a_							)
        model.to(a_							)
        model.eval()
        _UpperCAmelCase						:		List[str]      =				model(a_							)

        # verify hidden states
        self.parent.assertEqual(len(result.feature_maps							)						,len(config.out_features							)							)
        self.parent.assertListEqual(list(result.feature_maps[0].shape							)						,[self.batch_size, model.channels[0], 16, 16]							)

        # verify channels
        self.parent.assertEqual(len(model.channels							)						,len(config.out_features							)							)

        # verify backbone works with out_features=None
        _UpperCAmelCase						:		Optional[Any]      =				None
        _UpperCAmelCase						:		Optional[int]      =				NatBackbone(config=a_							)
        model.to(a_							)
        model.eval()
        _UpperCAmelCase						:		Optional[Any]      =				model(a_							)

        # verify feature maps
        self.parent.assertEqual(len(result.feature_maps							)						,1							)
        self.parent.assertListEqual(list(result.feature_maps[0].shape							)						,[self.batch_size, model.channels[-1], 4, 4]							)

        # verify channels
        self.parent.assertEqual(len(model.channels							)						,1							)







 def _snake_case				(							self							)	->     int:
        _UpperCAmelCase						:		int      =				self.prepare_config_and_inputs()
        _UpperCAmelCase  ,_UpperCAmelCase  ,_UpperCAmelCase						:		Optional[Any]      =				config_and_inputs
        _UpperCAmelCase						:		int      =				{"""pixel_values""": pixel_values}
        return config, inputs_dict







@require_natten
@require_torch
class 							lowercase (						_lowerCamelCase      ,      _lowerCamelCase      ,      unittest.TestCase       ):


 """simple docstring"""


 UpperCAmelCase					=						(
     (
         NatModel,
         NatForImageClassification,
         NatBackbone,
     )
     if is_torch_available()
     else ()
 )
 UpperCAmelCase					=						(
     {"""feature-extraction""": NatModel, """image-classification""": NatForImageClassification}
     if is_torch_available()
     else {}
 )
 UpperCAmelCase					=						False

 UpperCAmelCase					=						False
 UpperCAmelCase					=						False
 UpperCAmelCase					=						False
 UpperCAmelCase					=						False







 def _snake_case				(							self							)	->     str:
        _UpperCAmelCase						:		List[str]      =				NatModelTester(self							)
        _UpperCAmelCase						:		Optional[int]      =				ConfigTester(self						,config_class=a_						,embed_dim=37							)







 def _snake_case				(							self							)	->     Optional[int]:
        self.create_and_test_config_common_properties()
        self.config_tester.create_and_test_config_to_json_string()
        self.config_tester.create_and_test_config_to_json_file()
        self.config_tester.create_and_test_config_from_and_save_pretrained()
        self.config_tester.create_and_test_config_with_num_labels()
        self.config_tester.check_config_can_be_init_without_params()
        self.config_tester.check_config_arguments_init()







 def _snake_case				(							self							)	->     Optional[Any]:
        return







 def _snake_case				(							self							)	->     Tuple:
        _UpperCAmelCase						:		List[str]      =				self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_model(*a_							)







 def _snake_case				(							self							)	->     int:
        _UpperCAmelCase						:		Union[str, Any]      =				self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_for_image_classification(*a_							)







 def _snake_case				(							self							)	->     List[str]:
        _UpperCAmelCase						:		Optional[Any]      =				self.model_tester.prepare_config_and_inputs()
        self.model_tester.create_and_check_backbone(*a_							)







 @unittest.skip(reason="""Nat does not use inputs_embeds"""							)
 def _snake_case				(							self							)	->     str:
        pass







 @unittest.skip(reason="""Nat does not use feedforward chunking"""							)
 def _snake_case				(							self							)	->     Tuple:
        pass







 def _snake_case				(							self							)	->     Optional[Any]:
        _UpperCAmelCase  ,_UpperCAmelCase						:		List[str]      =				self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               _UpperCAmelCase						:		str      =				model_class(a_							)
               self.assertIsInstance(model.get_input_embeddings()						,(nn.Module)							)
               _UpperCAmelCase						:		Optional[Any]      =				model.get_output_embeddings()
               self.assertTrue(x is None or isinstance(a_						,nn.Linear							)							)







 def _snake_case				(							self							)	->     List[Any]:
        _UpperCAmelCase  ,_UpperCAmelCase						:		Union[str, Any]      =				self.model_tester.prepare_config_and_inputs_for_common()

        for model_class in self.all_model_classes:
               _UpperCAmelCase						:		List[Any]      =				model_class(a_							)
               _UpperCAmelCase						:		Optional[int]      =				inspect.signature(model.forward							)
               # signature.parameters is an OrderedDict => so arg_names order is deterministic
               _UpperCAmelCase						:		Optional[Any]      =				[*signature.parameters.keys()]

               _UpperCAmelCase						:		List[Any]      =				["""pixel_values"""]
               self.assertListEqual(arg_names[:1]						,a_							)







 def _snake_case				(							self							)	->     List[Any]:
        self.skipTest("""Nat's attention operation is handled entirely by NATTEN."""							)







 def _snake_case				(							self						,a_						,a_						,a_						,a_							)	->     str:
        _UpperCAmelCase						:		Tuple      =				model_class(a_							)
        model.to(a_							)
        model.eval()

        with torch.no_grad():
               _UpperCAmelCase						:		List[str]      =				model(**self._prepare_for_class(a_						,a_							)							)

        _UpperCAmelCase						:		List[str]      =				outputs.hidden_states

        _UpperCAmelCase						:		Optional[int]      =				getattr(
            self.model_tester						,"""expected_num_hidden_layers"""						,len(self.model_tester.depths							) + 1							)
        self.assertEqual(len(a_							)						,a_							)

        # Nat has a different seq_length
        _UpperCAmelCase						:		Tuple      =				(
            config.patch_size
            if isinstance(config.patch_size						,collections.abc.Iterable							)
            else (config.patch_size, config.patch_size)
        )

        _UpperCAmelCase						:		str      =				image_size[0] // patch_size[0]
        _UpperCAmelCase						:		Tuple      =				image_size[1] // patch_size[1]

        self.assertListEqual(
            list(hidden_states[0].shape[-3:]							)						,[height, width, self.model_tester.embed_dim]						,)

        if model_class.__name__ != "NatBackbone":
               _UpperCAmelCase						:		Optional[int]      =				outputs.reshaped_hidden_states
               self.assertEqual(len(a_							)						,a_							)

               _UpperCAmelCase  ,_UpperCAmelCase  ,_UpperCAmelCase  ,_UpperCAmelCase						:		Optional[Any]      =				reshaped_hidden_states[0].shape
               _UpperCAmelCase						:		Union[str, Any]      =				(
                   reshaped_hidden_states[0].view(a_						,a_						,a_						,a_							).permute(0						,2						,3						,1							)
               )
               self.assertListEqual(
                   list(reshaped_hidden_states.shape[-3:]							)						,[height, width, self.model_tester.embed_dim]						,)







 def _snake_case				(							self							)	->     List[str]:
        _UpperCAmelCase  ,_UpperCAmelCase						:		Any      =				self.model_tester.prepare_config_and_inputs_for_common()

        _UpperCAmelCase						:		Dict      =				(
            self.model_tester.image_size
            if isinstance(self.model_tester.image_size						,collections.abc.Iterable							)
            else (self.model_tester.image_size, self.model_tester.image_size)
        )

        for model_class in self.all_model_classes:
               _UpperCAmelCase						:		List[str]      =				True
               self.check_hidden_states_output(a_						,a_						,a_						,a_							)

               # check that output_hidden_states also work using config
               del inputs_dict["output_hidden_states"]
               _UpperCAmelCase						:		List[Any]      =				True

               self.check_hidden_states_output(a_						,a_						,a_						,a_							)







 @slow
 def _snake_case				(							self							)	->     List[Any]:
        for model_name in NAT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
               _UpperCAmelCase						:		int      =				NatModel.from_pretrained(a_							)
               self.assertIsNotNone(a_							)







 def _snake_case				(							self							)	->     int:
        _UpperCAmelCase  ,_UpperCAmelCase						:		List[Any]      =				self.model_tester.prepare_config_and_inputs_for_common()

        _UpperCAmelCase						:		Tuple      =				_config_zero_init(a_							)
        for model_class in self.all_model_classes:
               _UpperCAmelCase						:		Optional[Any]      =				model_class(config=a_							)
               for name, param in model.named_parameters():
                      if "embeddings" not in name and param.requires_grad:
                             self.assertIn(
                                 ((param.data.mean() * 1E9).round() / 1E9).item()						,[0.0, 1.0]						,msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''						,)







@require_natten
@require_vision
@require_torch
class 							lowercase (						unittest.TestCase       ):


 """simple docstring"""


 @cached_property
 def _snake_case				(							self							)	->     Union[str, Any]:
        return AutoImageProcessor.from_pretrained("""shi-labs/nat-mini-in1k-224"""							) if is_vision_available() else None







 @slow
 def _snake_case				(							self							)	->     Tuple:
        _UpperCAmelCase						:		List[str]      =				NatForImageClassification.from_pretrained("""shi-labs/nat-mini-in1k-224"""							).to(a_							)
        _UpperCAmelCase						:		int      =				self.default_image_processor

        _UpperCAmelCase						:		Optional[int]      =				Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""							)
        _UpperCAmelCase						:		Optional[int]      =				image_processor(images=a_						,return_tensors="""pt"""							).to(a_							)

        # forward pass
        with torch.no_grad():
               _UpperCAmelCase						:		Union[str, Any]      =				model(**a_							)

        # verify the logits
        _UpperCAmelCase						:		Any      =				torch.Size((1, 1_000)							)
        self.assertEqual(outputs.logits.shape						,a_							)
        _UpperCAmelCase						:		str      =				torch.tensor([0.3805, -0.8676, -0.3912]							).to(a_							)
        self.assertTrue(torch.allclose(outputs.logits[0, :3]						,a_						,atol=1E-4							)							)







@require_torch
@require_natten
class 							lowercase (						unittest.TestCase      ,      _lowerCamelCase       ):


 """simple docstring"""


 UpperCAmelCase					=						(NatBackbone,) if is_torch_available() else ()
 UpperCAmelCase					=						NatConfig







 def _snake_case				(							self							)	->     Optional[Any]:
        _UpperCAmelCase						:		Tuple      =				NatModelTester(self							)



