code: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import os
  from pathlib import Path
  
  
  
  
  
  
  def 	_lowercase	(			lowercase_		,							lowercase_		,							lowercase_		,							lowercase_						):
  
  
  
  		'''simple docstring'''
  
  
  
  
  
  
  
  		_snake_case			 =							{
  		    "en": "Machine learning is great, isn't it?",
  		    "ru": "Машинное обучение - это здорово, не так ли?",
  		    "de": "Maschinelles Lernen ist großartig, nicht wahr?",
  		}
  
  		# BLUE scores as follows:
  		# "pair": [fairseq, transformers]
  		_snake_case			 =							{
  		    "wmt19-de-en-6-6-base": [0, 38.37],
  		    "wmt19-de-en-6-6-big": [0, 39.90],
  		}
  		_snake_case			 =							f'''{src_lang}-{tgt_lang}'''
  
  		_snake_case			 =							f'''
  ---
  
  language:
  - {src_lang}
  - {tgt_lang}
  thumbnail:
  tags:
  - translation
  - wmt19
  - allenai
  license: apache-2.0
  datasets:
  - wmt19
  metrics:
  - bleu
  ---
  
  # FSMT
  
  ## Model description
  
  This is a ported version of fairseq-based [wmt19 transformer](https://github.com/jungokasai/deep-shallow/) for {src_lang}-{tgt_lang}.
  
  For more details, please, see [Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation](https://arxiv.org/abs/2006.10369).
  
  2 models are available:
  
  * [wmt19-de-en-6-6-big](https://huggingface.co/allenai/wmt19-de-en-6-6-big)
  * [wmt19-de-en-6-6-base](https://huggingface.co/allenai/wmt19-de-en-6-6-base)
  
  
  ## Intended uses & limitations
  
  #### How to use
  
  ```python
  from transformers import FSMTForConditionalGeneration, FSMTTokenizer
  mname = "allenai/{model_name}"
  tokenizer = FSMTTokenizer.from_pretrained(mname)
  model = FSMTForConditionalGeneration.from_pretrained(mname)
  
  input = "{texts[src_lang]}"
  input_ids = tokenizer.encode(input, return_tensors="pt")
  outputs = model.generate(input_ids)
  decoded = tokenizer.decode(outputs[0], skip_special_tokens=True)
  print(decoded) # {texts[tgt_lang]}
  
  ```
  
  #### Limitations and bias
  
  
  ## Training data
  
  Pretrained weights were left identical to the original model released by allenai. For more details, please, see the [paper](https://arxiv.org/abs/2006.10369).
  
  ## Eval results
  
  Here are the BLEU scores:
  
  model   |  transformers
  -------|---------
  {model_name}  |  {scores[model_name][1]}
  
  The score was calculated using this code:
  
  ```bash
  git clone https://github.com/huggingface/transformers
  cd transformers
  export PAIR={pair}
  export DATA_DIR=data/$PAIR
  export SAVE_DIR=data/$PAIR
  export BS=8
  export NUM_BEAMS=5
  mkdir -p $DATA_DIR
  sacrebleu -t wmt19 -l $PAIR --echo src > $DATA_DIR/val.source
  sacrebleu -t wmt19 -l $PAIR --echo ref > $DATA_DIR/val.target
  echo $PAIR
  PYTHONPATH="src:examples/seq2seq" python examples/seq2seq/run_eval.py allenai/{model_name} $DATA_DIR/val.source $SAVE_DIR/test_translations.txt --reference_path $DATA_DIR/val.target --score_path $SAVE_DIR/test_bleu.json --bs $BS --task translation --num_beams $NUM_BEAMS
  ```
  
  ## Data Sources
  
  - [training, etc.](http://www.statmt.org/wmt19/)
  - [test set](http://matrix.statmt.org/test_sets/newstest2019.tgz?1556572561)
  
  
  ### BibTeX entry and citation info
  
  ```
  @misc{{kasai2020deep,
      title={{Deep Encoder, Shallow Decoder: Reevaluating the Speed-Quality Tradeoff in Machine Translation}},
      author={{Jungo Kasai and Nikolaos Pappas and Hao Peng and James Cross and Noah A. Smith}},
      year={{2020}},
      eprint={{2006.10369}},
      archivePrefix={{arXiv}},
      primaryClass={{cs.CL}}
  }}
  ```
  
  '''
  		model_card_dir.mkdir(parents=lowercase_		,							exist_ok=lowercase_						)
  		_snake_case			 =							os.path.join(lowercase_		,							"README.md"						)
  		print(f'''Generating {path}'''						)
  		with open(lowercase_		,							"w"		,							encoding="utf-8"						) as f:
  				f.write(lowercase_						)
  
  
  # make sure we are under the root of the project
  _lowerCamelCase							:		List[str]										=						Path(__file__).resolve().parent.parent.parent
  _lowerCamelCase							:		List[Any]										=						repo_dir / 'model_cards'
  
  for model_name in ["wmt19-de-en-6-6-base", "wmt19-de-en-6-6-big"]:
  						_lowerCamelCase							:		int										=						model_cards_dir / 'allenai' / model_name
  						write_model_card(model_card_dir, src_lang='de', tgt_lang='en', model_name=model_name)
  
  
  
  
  
  
style_context: 
  
  
  
  
  '''simple docstring'''
  
  
  
  
  
  
  
  import json
  import os
  import unittest
  
  from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
      VOCAB_FILES_NAMES,
      GPTNeoXJapaneseTokenizer,
  )
  from transformers.testing_utils import require_tokenizers, slow
  
  from ...test_tokenization_common import TokenizerTesterMixin
  
  
  
  
  
  @require_tokenizers
  class   lowerCamelCase_							( UpperCAmelCase_, unittest.TestCase							):
  
  
  
  
  					'''simple docstring'''
  
  
  
  
  
  
  					lowercase__					=				GPTNeoXJapaneseTokenizer
  					lowercase__					=				False
  					lowercase__					=				{"""do_clean_text""": False, """add_prefix_space""": False}
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							super().setUp()
  
  							_snake_case			 = [
  							    "こん",
  							    "こんに",
  							    "にちは",
  							    "ばんは",
  							    "世界,㔺界",
  							    "、",
  							    "。",
  							    "<BR>",
  							    "<SP>",
  							    "<TAB>",
  							    "<URL>",
  							    "<EMAIL>",
  							    "<TEL>",
  							    "<DATE>",
  							    "<PRICE>",
  							    "<BLOCK>",
  							    "<KIGOU>",
  							    "<U2000U2BFF>",
  							    "<|emoji1|>",
  							    "<unk>",
  							    "<|startoftext|>",
  							    "<|endoftext|>",
  							]
  							_snake_case			 = {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
  							_snake_case			 = {"unk_token": "<unk>"}
  
  							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["vocab_file"]				)
  							_snake_case			 = os.path.join(self.tmpdirname,						VOCAB_FILES_NAMES["emoji_file"]				)
  							with open(self.vocab_file,						"w",						encoding="utf-8"				) as vocab_writer:
  									vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)
  							with open(self.emoji_file,						"w"				) as emoji_writer:
  									emoji_writer.write(json.dumps(__UpperCamelCase				)				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						**__UpperCamelCase				):
  							kwargs.update(self.special_tokens_map				)
  							return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname,						**__UpperCamelCase				)
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case			 = "こんにちは、世界。 \nこんばんは、㔺界。😀"
  							_snake_case			 = "こんにちは、世界。 \nこんばんは、世界。😀"
  							return input_text, output_text
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self,						__UpperCamelCase				):
  							_snake_case ,					_snake_case			 = self.get_input_output_texts(__UpperCamelCase				)
  							_snake_case			 = tokenizer.encode(__UpperCamelCase,						add_special_tokens=__UpperCamelCase				)
  							_snake_case			 = tokenizer.decode(__UpperCamelCase,						clean_up_tokenization_spaces=__UpperCamelCase				)
  							return text, ids
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							pass  # TODO add if relevant
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = self.get_tokenizer()
  
  							# Testing tokenization
  							_snake_case			 = "こんにちは、世界。　こんばんは、㔺界。"
  							_snake_case			 = ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
  							_snake_case			 = tokenizer.tokenize(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  							# Testing conversion to ids without special tokens
  							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
  							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  							# Testing conversion to ids with special tokens
  							_snake_case			 = tokens + [tokenizer.unk_token]
  							_snake_case			 = [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
  							_snake_case			 = tokenizer.convert_tokens_to_ids(__UpperCamelCase				)
  							self.assertListEqual(__UpperCamelCase,						__UpperCamelCase				)
  
  
  
  
  
  
  					@slow
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							_snake_case			 = self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"				)
  
  							_snake_case			 = tokenizer.encode("ありがとう。",						add_special_tokens=__UpperCamelCase				)
  							_snake_case			 = tokenizer.encode("どういたしまして。",						add_special_tokens=__UpperCamelCase				)
  
  							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase				)
  							_snake_case			 = tokenizer.build_inputs_with_special_tokens(__UpperCamelCase,						__UpperCamelCase				)
  
  							assert encoded_sentence == ids_a
  							assert encoded_pair == ids_a + ids_a
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							# Intentionally convert some words to accommodate character fluctuations unique to Japanese
  							pass
  
  
  
  
  
  
  					def        __SCREAMING_SNAKE_CASE			(					self				):
  							# tokenizer has no padding token
  							pass
  
  
  
  
  
  