<jupyter_start><jupyter_text>Benchmark Transformers GPT2<jupyter_code># These parameters can be injected from Papermill
train_file = "wikitext-2/wiki.train.tokens"
valid_file = "wikitext-2/wiki.valid.tokens"
epochs = 10
warmup_steps = 1000
# Parameters
train_file = "wikitext-103-raw/wiki.train.raw"
valid_file = "wikitext-103-raw/wiki.valid.raw"
epochs = 10
warmup_steps = 0<jupyter_output><empty_output><jupyter_text>Install Library<jupyter_code>!pip install transformers==3.4.0<jupyter_output>Requirement already satisfied: transformers==3.4.0 in /opt/conda/lib/python3.7/site-packages (3.4.0)
Requirement already satisfied: tokenizers==0.9.2 in /opt/conda/lib/python3.7/site-packages (from transformers==3.4.0) (0.9.2)
Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.7/site-packages (from transformers==3.4.0) (2020.10.23)
Requirement already satisfied: numpy in /opt/conda/lib/python3.7/site-packages (from transformers==3.4.0) (1.18.5)
Requirement already satisfied: packaging in /opt/conda/lib/python3.7/site-packages (from transformers==3.4.0) (20.4)
Requirement already satisfied: sentencepiece!=0.1.92 in /opt/conda/lib/python3.7/site-packages (from transformers==3.4.0) (0.1.94)
Requirement already satisfied: sacremoses in /opt/conda/lib/python3.7/site-packages (from transformers==3.4.0) (0.0.43)
Requirement already satisfied: protobuf in /opt/conda/lib/python3.7/site-packages (from transformers==3.4.0) (3.13.0)
Requirement already satisfied: requests i[...]<jupyter_text>Download training script<jupyter_code>import urllib

url = "https://raw.githubusercontent.com/huggingface/transformers/v3.4.0/examples/language-modeling/run_language_modeling.py"
urllib.request.urlretrieve(url, "run_language_modeling.py")<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code># Train model from scratch
!python run_language_modeling.py \
    --output_dir=output \
    --model_type=gpt2 \
    --tokenizer_name=gpt2 \
    --do_train \
    --train_data_file={train_file} \
    --do_eval \
    --eval_data_file={valid_file} \
    --per_device_train_batch_size=2 \
    --num_train_epochs={epochs} \
    --evaluate_during_training \
    --load_best_model_at_end \
    --disable_tqdm=False \
    --warmup_steps={warmup_steps} \
    --output_dir=transformers_output \
    --save_steps=10000 \
    --eval_steps=10000<jupyter_output>/opt/conda/lib/python3.7/site-packages/transformers/training_args.py:339: FutureWarning: The `evaluate_during_training` argument is deprecated in favor of `evaluation_strategy` (which has more options)
  FutureWarning,
10/29/2020 01:27:30 - WARNING - __main__ -   Process rank: -1, device: cuda:0, n_gpu: 1, distributed training: False, 16-bits training: False
10/29/2020 01:27:30 - INFO - __main__ -   Training/evaluation parameters TrainingArguments(output_dir='transformers_output', overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluate_during_training=True, evaluation_strategy=<EvaluationStrategy.STEPS: 'steps'>, prediction_loss_only=False, per_device_train_batch_size=2, per_device_eval_batch_size=8, per_gpu_train_batch_size=None, per_gpu_eval_batch_size=None, gradient_accumulation_steps=1, eval_accumulation_steps=None, learning_rate=5e-05, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=10.0, max_s[...]
StarcoderdataJupyterStructuredCleanDedup