Add overall stable checkpoint-25000 from "trained_model" - Model: myt5-large

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+byte_maps.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,127 @@
+{
+  "<extra_id_0>": 259,
+  "<extra_id_100>": 359,
+  "<extra_id_101>": 360,
+  "<extra_id_102>": 361,
+  "<extra_id_103>": 362,
+  "<extra_id_104>": 363,
+  "<extra_id_105>": 364,
+  "<extra_id_106>": 365,
+  "<extra_id_107>": 366,
+  "<extra_id_108>": 367,
+  "<extra_id_109>": 368,
+  "<extra_id_10>": 269,
+  "<extra_id_110>": 369,
+  "<extra_id_111>": 370,
+  "<extra_id_112>": 371,
+  "<extra_id_113>": 372,
+  "<extra_id_114>": 373,
+  "<extra_id_115>": 374,
+  "<extra_id_116>": 375,
+  "<extra_id_117>": 376,
+  "<extra_id_118>": 377,
+  "<extra_id_119>": 378,
+  "<extra_id_11>": 270,
+  "<extra_id_120>": 379,
+  "<extra_id_121>": 380,
+  "<extra_id_122>": 381,
+  "<extra_id_123>": 382,
+  "<extra_id_124>": 383,
+  "<extra_id_12>": 271,
+  "<extra_id_13>": 272,
+  "<extra_id_14>": 273,
+  "<extra_id_15>": 274,
+  "<extra_id_16>": 275,
+  "<extra_id_17>": 276,
+  "<extra_id_18>": 277,
+  "<extra_id_19>": 278,
+  "<extra_id_1>": 260,
+  "<extra_id_20>": 279,
+  "<extra_id_21>": 280,
+  "<extra_id_22>": 281,
+  "<extra_id_23>": 282,
+  "<extra_id_24>": 283,
+  "<extra_id_25>": 284,
+  "<extra_id_26>": 285,
+  "<extra_id_27>": 286,
+  "<extra_id_28>": 287,
+  "<extra_id_29>": 288,
+  "<extra_id_2>": 261,
+  "<extra_id_30>": 289,
+  "<extra_id_31>": 290,
+  "<extra_id_32>": 291,
+  "<extra_id_33>": 292,
+  "<extra_id_34>": 293,
+  "<extra_id_35>": 294,
+  "<extra_id_36>": 295,
+  "<extra_id_37>": 296,
+  "<extra_id_38>": 297,
+  "<extra_id_39>": 298,
+  "<extra_id_3>": 262,
+  "<extra_id_40>": 299,
+  "<extra_id_41>": 300,
+  "<extra_id_42>": 301,
+  "<extra_id_43>": 302,
+  "<extra_id_44>": 303,
+  "<extra_id_45>": 304,
+  "<extra_id_46>": 305,
+  "<extra_id_47>": 306,
+  "<extra_id_48>": 307,
+  "<extra_id_49>": 308,
+  "<extra_id_4>": 263,
+  "<extra_id_50>": 309,
+  "<extra_id_51>": 310,
+  "<extra_id_52>": 311,
+  "<extra_id_53>": 312,
+  "<extra_id_54>": 313,
+  "<extra_id_55>": 314,
+  "<extra_id_56>": 315,
+  "<extra_id_57>": 316,
+  "<extra_id_58>": 317,
+  "<extra_id_59>": 318,
+  "<extra_id_5>": 264,
+  "<extra_id_60>": 319,
+  "<extra_id_61>": 320,
+  "<extra_id_62>": 321,
+  "<extra_id_63>": 322,
+  "<extra_id_64>": 323,
+  "<extra_id_65>": 324,
+  "<extra_id_66>": 325,
+  "<extra_id_67>": 326,
+  "<extra_id_68>": 327,
+  "<extra_id_69>": 328,
+  "<extra_id_6>": 265,
+  "<extra_id_70>": 329,
+  "<extra_id_71>": 330,
+  "<extra_id_72>": 331,
+  "<extra_id_73>": 332,
+  "<extra_id_74>": 333,
+  "<extra_id_75>": 334,
+  "<extra_id_76>": 335,
+  "<extra_id_77>": 336,
+  "<extra_id_78>": 337,
+  "<extra_id_79>": 338,
+  "<extra_id_7>": 266,
+  "<extra_id_80>": 339,
+  "<extra_id_81>": 340,
+  "<extra_id_82>": 341,
+  "<extra_id_83>": 342,
+  "<extra_id_84>": 343,
+  "<extra_id_85>": 344,
+  "<extra_id_86>": 345,
+  "<extra_id_87>": 346,
+  "<extra_id_88>": 347,
+  "<extra_id_89>": 348,
+  "<extra_id_8>": 267,
+  "<extra_id_90>": 349,
+  "<extra_id_91>": 350,
+  "<extra_id_92>": 351,
+  "<extra_id_93>": 352,
+  "<extra_id_94>": 353,
+  "<extra_id_95>": 354,
+  "<extra_id_96>": 355,
+  "<extra_id_97>": 356,
+  "<extra_id_98>": 357,
+  "<extra_id_99>": 358,
+  "<extra_id_9>": 268
+}

byte_maps.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2117fbbdacd95d9c5c1327bf096fd4048877f67aea7b68c07cbd77d8c68d003
+size 11644649

config.json

@@ -0,0 +1,33 @@
+{
+  "architectures": [
+    "T5ForConditionalGeneration"
+  ],
+  "classifier_dropout": 0.0,
+  "d_ff": 3840,
+  "d_kv": 64,
+  "d_model": 1536,
+  "decoder_start_token_id": 0,
+  "dense_act_fn": "gelu_new",
+  "dropout_rate": 0.0,
+  "eos_token_id": 1,
+  "feed_forward_proj": "gated-gelu",
+  "gradient_checkpointing": false,
+  "initializer_factor": 1.0,
+  "is_encoder_decoder": true,
+  "is_gated_act": true,
+  "layer_norm_epsilon": 1e-06,
+  "model_type": "t5",
+  "num_decoder_layers": 12,
+  "num_heads": 16,
+  "num_layers": 36,
+  "output_past": true,
+  "pad_token_id": 0,
+  "relative_attention_max_distance": 128,
+  "relative_attention_num_buckets": 32,
+  "tie_word_embeddings": false,
+  "tokenizer_class": "MyT5Tokenizer",
+  "torch_dtype": "float32",
+  "transformers_version": "4.52.2",
+  "use_cache": true,
+  "vocab_size": 384
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "decoder_start_token_id": 0,
+  "eos_token_id": 1,
+  "pad_token_id": 0,
+  "transformers_version": "4.52.2"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf6259a1c4169e48dc461f3d2c233e621eadb3c599afb58b22ae7e5202dd207
+size 4912795416

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19666fc256ff6199564f54065299e4a22a9cb8942a87e4f8271dee5995201188
+size 2500805707

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0676bf75358ef5347eeb5b68e2f9bcd272783146837335dabe76f8adbfdb1957
+size 14645

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c50d7f913c4429bf52d23c2f0d4f1093de6d29339591bc8af8bbdb6935cabade
+size 1465

special_tokens_map.json

@@ -0,0 +1,150 @@
+{
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>",
+    "<extra_id_100>",
+    "<extra_id_101>",
+    "<extra_id_102>",
+    "<extra_id_103>",
+    "<extra_id_104>",
+    "<extra_id_105>",
+    "<extra_id_106>",
+    "<extra_id_107>",
+    "<extra_id_108>",
+    "<extra_id_109>",
+    "<extra_id_110>",
+    "<extra_id_111>",
+    "<extra_id_112>",
+    "<extra_id_113>",
+    "<extra_id_114>",
+    "<extra_id_115>",
+    "<extra_id_116>",
+    "<extra_id_117>",
+    "<extra_id_118>",
+    "<extra_id_119>",
+    "<extra_id_120>",
+    "<extra_id_121>",
+    "<extra_id_122>",
+    "<extra_id_123>",
+    "<extra_id_124>"
+  ],
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,1163 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "259": {
+      "content": "<extra_id_0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "260": {
+      "content": "<extra_id_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "261": {
+      "content": "<extra_id_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "262": {
+      "content": "<extra_id_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "263": {
+      "content": "<extra_id_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "264": {
+      "content": "<extra_id_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "265": {
+      "content": "<extra_id_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "266": {
+      "content": "<extra_id_7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "267": {
+      "content": "<extra_id_8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "268": {
+      "content": "<extra_id_9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "269": {
+      "content": "<extra_id_10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "270": {
+      "content": "<extra_id_11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "271": {
+      "content": "<extra_id_12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "272": {
+      "content": "<extra_id_13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "273": {
+      "content": "<extra_id_14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "274": {
+      "content": "<extra_id_15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "275": {
+      "content": "<extra_id_16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "276": {
+      "content": "<extra_id_17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "277": {
+      "content": "<extra_id_18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "278": {
+      "content": "<extra_id_19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "279": {
+      "content": "<extra_id_20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "280": {
+      "content": "<extra_id_21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "281": {
+      "content": "<extra_id_22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "282": {
+      "content": "<extra_id_23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "283": {
+      "content": "<extra_id_24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "284": {
+      "content": "<extra_id_25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "285": {
+      "content": "<extra_id_26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "286": {
+      "content": "<extra_id_27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "287": {
+      "content": "<extra_id_28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "288": {
+      "content": "<extra_id_29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "289": {
+      "content": "<extra_id_30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "290": {
+      "content": "<extra_id_31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "291": {
+      "content": "<extra_id_32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "292": {
+      "content": "<extra_id_33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "293": {
+      "content": "<extra_id_34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "294": {
+      "content": "<extra_id_35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "295": {
+      "content": "<extra_id_36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "296": {
+      "content": "<extra_id_37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "297": {
+      "content": "<extra_id_38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "298": {
+      "content": "<extra_id_39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "299": {
+      "content": "<extra_id_40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "300": {
+      "content": "<extra_id_41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "301": {
+      "content": "<extra_id_42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "302": {
+      "content": "<extra_id_43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "303": {
+      "content": "<extra_id_44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "304": {
+      "content": "<extra_id_45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "305": {
+      "content": "<extra_id_46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "306": {
+      "content": "<extra_id_47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "307": {
+      "content": "<extra_id_48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "308": {
+      "content": "<extra_id_49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "309": {
+      "content": "<extra_id_50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "310": {
+      "content": "<extra_id_51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "311": {
+      "content": "<extra_id_52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "312": {
+      "content": "<extra_id_53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "313": {
+      "content": "<extra_id_54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "314": {
+      "content": "<extra_id_55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "315": {
+      "content": "<extra_id_56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "316": {
+      "content": "<extra_id_57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "317": {
+      "content": "<extra_id_58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "318": {
+      "content": "<extra_id_59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "319": {
+      "content": "<extra_id_60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "320": {
+      "content": "<extra_id_61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "321": {
+      "content": "<extra_id_62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "322": {
+      "content": "<extra_id_63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "323": {
+      "content": "<extra_id_64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "324": {
+      "content": "<extra_id_65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "325": {
+      "content": "<extra_id_66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "326": {
+      "content": "<extra_id_67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "327": {
+      "content": "<extra_id_68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "328": {
+      "content": "<extra_id_69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "329": {
+      "content": "<extra_id_70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "330": {
+      "content": "<extra_id_71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "331": {
+      "content": "<extra_id_72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "332": {
+      "content": "<extra_id_73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "333": {
+      "content": "<extra_id_74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "334": {
+      "content": "<extra_id_75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "335": {
+      "content": "<extra_id_76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "336": {
+      "content": "<extra_id_77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "337": {
+      "content": "<extra_id_78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "338": {
+      "content": "<extra_id_79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "339": {
+      "content": "<extra_id_80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "340": {
+      "content": "<extra_id_81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "341": {
+      "content": "<extra_id_82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "342": {
+      "content": "<extra_id_83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "343": {
+      "content": "<extra_id_84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "344": {
+      "content": "<extra_id_85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "345": {
+      "content": "<extra_id_86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "346": {
+      "content": "<extra_id_87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "347": {
+      "content": "<extra_id_88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "348": {
+      "content": "<extra_id_89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "349": {
+      "content": "<extra_id_90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "350": {
+      "content": "<extra_id_91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "351": {
+      "content": "<extra_id_92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "352": {
+      "content": "<extra_id_93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "353": {
+      "content": "<extra_id_94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "354": {
+      "content": "<extra_id_95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "355": {
+      "content": "<extra_id_96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "356": {
+      "content": "<extra_id_97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "357": {
+      "content": "<extra_id_98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "358": {
+      "content": "<extra_id_99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "359": {
+      "content": "<extra_id_100>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "360": {
+      "content": "<extra_id_101>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "361": {
+      "content": "<extra_id_102>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "362": {
+      "content": "<extra_id_103>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "363": {
+      "content": "<extra_id_104>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "364": {
+      "content": "<extra_id_105>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "365": {
+      "content": "<extra_id_106>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "366": {
+      "content": "<extra_id_107>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "367": {
+      "content": "<extra_id_108>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "368": {
+      "content": "<extra_id_109>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "369": {
+      "content": "<extra_id_110>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "370": {
+      "content": "<extra_id_111>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "371": {
+      "content": "<extra_id_112>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "372": {
+      "content": "<extra_id_113>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "373": {
+      "content": "<extra_id_114>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "374": {
+      "content": "<extra_id_115>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "375": {
+      "content": "<extra_id_116>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "376": {
+      "content": "<extra_id_117>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "377": {
+      "content": "<extra_id_118>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "378": {
+      "content": "<extra_id_119>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "379": {
+      "content": "<extra_id_120>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "380": {
+      "content": "<extra_id_121>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "381": {
+      "content": "<extra_id_122>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "382": {
+      "content": "<extra_id_123>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "383": {
+      "content": "<extra_id_124>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<extra_id_0>",
+    "<extra_id_1>",
+    "<extra_id_2>",
+    "<extra_id_3>",
+    "<extra_id_4>",
+    "<extra_id_5>",
+    "<extra_id_6>",
+    "<extra_id_7>",
+    "<extra_id_8>",
+    "<extra_id_9>",
+    "<extra_id_10>",
+    "<extra_id_11>",
+    "<extra_id_12>",
+    "<extra_id_13>",
+    "<extra_id_14>",
+    "<extra_id_15>",
+    "<extra_id_16>",
+    "<extra_id_17>",
+    "<extra_id_18>",
+    "<extra_id_19>",
+    "<extra_id_20>",
+    "<extra_id_21>",
+    "<extra_id_22>",
+    "<extra_id_23>",
+    "<extra_id_24>",
+    "<extra_id_25>",
+    "<extra_id_26>",
+    "<extra_id_27>",
+    "<extra_id_28>",
+    "<extra_id_29>",
+    "<extra_id_30>",
+    "<extra_id_31>",
+    "<extra_id_32>",
+    "<extra_id_33>",
+    "<extra_id_34>",
+    "<extra_id_35>",
+    "<extra_id_36>",
+    "<extra_id_37>",
+    "<extra_id_38>",
+    "<extra_id_39>",
+    "<extra_id_40>",
+    "<extra_id_41>",
+    "<extra_id_42>",
+    "<extra_id_43>",
+    "<extra_id_44>",
+    "<extra_id_45>",
+    "<extra_id_46>",
+    "<extra_id_47>",
+    "<extra_id_48>",
+    "<extra_id_49>",
+    "<extra_id_50>",
+    "<extra_id_51>",
+    "<extra_id_52>",
+    "<extra_id_53>",
+    "<extra_id_54>",
+    "<extra_id_55>",
+    "<extra_id_56>",
+    "<extra_id_57>",
+    "<extra_id_58>",
+    "<extra_id_59>",
+    "<extra_id_60>",
+    "<extra_id_61>",
+    "<extra_id_62>",
+    "<extra_id_63>",
+    "<extra_id_64>",
+    "<extra_id_65>",
+    "<extra_id_66>",
+    "<extra_id_67>",
+    "<extra_id_68>",
+    "<extra_id_69>",
+    "<extra_id_70>",
+    "<extra_id_71>",
+    "<extra_id_72>",
+    "<extra_id_73>",
+    "<extra_id_74>",
+    "<extra_id_75>",
+    "<extra_id_76>",
+    "<extra_id_77>",
+    "<extra_id_78>",
+    "<extra_id_79>",
+    "<extra_id_80>",
+    "<extra_id_81>",
+    "<extra_id_82>",
+    "<extra_id_83>",
+    "<extra_id_84>",
+    "<extra_id_85>",
+    "<extra_id_86>",
+    "<extra_id_87>",
+    "<extra_id_88>",
+    "<extra_id_89>",
+    "<extra_id_90>",
+    "<extra_id_91>",
+    "<extra_id_92>",
+    "<extra_id_93>",
+    "<extra_id_94>",
+    "<extra_id_95>",
+    "<extra_id_96>",
+    "<extra_id_97>",
+    "<extra_id_98>",
+    "<extra_id_99>",
+    "<extra_id_100>",
+    "<extra_id_101>",
+    "<extra_id_102>",
+    "<extra_id_103>",
+    "<extra_id_104>",
+    "<extra_id_105>",
+    "<extra_id_106>",
+    "<extra_id_107>",
+    "<extra_id_108>",
+    "<extra_id_109>",
+    "<extra_id_110>",
+    "<extra_id_111>",
+    "<extra_id_112>",
+    "<extra_id_113>",
+    "<extra_id_114>",
+    "<extra_id_115>",
+    "<extra_id_116>",
+    "<extra_id_117>",
+    "<extra_id_118>",
+    "<extra_id_119>",
+    "<extra_id_120>",
+    "<extra_id_121>",
+    "<extra_id_122>",
+    "<extra_id_123>",
+    "<extra_id_124>"
+  ],
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "extra_ids": 0,
+  "extra_special_tokens": {},
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "tokenizer_class": "MyT5Tokenizer",
+  "unk_token": "<unk>"
+}

train_stable_myt5-large.py

@@ -0,0 +1,347 @@
+import os
+import json
+import gc
+import logging
+import torch
+import pickle
+from torch.utils.data import Dataset
+from transformers import (
+    AutoModelForSeq2SeqLM,
+    AutoTokenizer,
+    TrainingArguments,
+    Trainer,
+    TrainerCallback,
+    DataCollatorForSeq2Seq,
+)
+
+# CONFIGURATION
+MAX_ITEMS = None
+MAX_LENGTH = 256
+PER_DEVICE_BATCH = 1 
+GRAD_ACC_STEPS = 16  # Increased due to higher MAX_LENGTH
+LEARNING_RATE = 5e-5 
+NUM_TRAIN_EPOCHS = 1
+WARMUP_STEPS = 200
+FP16_TRAINING = False # fix windows
+OPTIMIZER_CHOICE = "adamw_8bit" 
+MAX_GRAD_NORM_CLIP = 0.0 
+GRADIENT_CHECKPOINTING = True 
+LOGGING_STEPS = 50
+SAVE_STEPS = 1000
+EVAL_STEPS = 500
+SAVE_TOTAL_LIMIT = 20  # each 7GB
+FIXED_PROMPT_FOR_GENERATION = "Create stable diffusion metadata based on the given english description. a futuristic city"
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s — %(levelname)s — %(name)s — %(message)s")
+log = logging.getLogger(__name__)
+
+class SDPromptDataset(Dataset):
+    def __init__(self, raw_data_list, tokenizer, max_length, dataset_type="train", cache_dir="cache"):
+        self.raw_data = raw_data_list
+        self.tokenizer = tokenizer
+        self.max_length = max_length
+        self.dataset_type = dataset_type
+        
+        os.makedirs(cache_dir, exist_ok=True)
+        cache_file = os.path.join(cache_dir, f"{dataset_type}_{len(raw_data_list)}_{max_length}.pkl")
+        
+        if os.path.exists(cache_file):
+            log.info(f"Loading cached {dataset_type} dataset from {cache_file}")
+            with open(cache_file, 'rb') as f:
+                self.examples = pickle.load(f)
+            log.info(f"Loaded {len(self.examples)} cached examples for {dataset_type}")
+        else:
+            log.info(f"Tokenizing {len(raw_data_list)} samples for {dataset_type} with {type(tokenizer).__name__}...")
+            self.examples = []
+            
+            for i, item in enumerate(raw_data_list):
+                if i > 0 and (i % 5000 == 0 or i == len(raw_data_list) - 1):
+                    log.info(f"Tokenized {i+1} / {len(raw_data_list)} samples for {dataset_type}")
+
+                instruction = item.get("instruction", "")
+                output = item.get("output", "")
+
+                input_encoding = tokenizer(
+                    instruction, max_length=max_length, padding="max_length",
+                    truncation=True, return_tensors="pt",
+                )
+                
+                if self.dataset_type == "train" or (self.dataset_type == "eval" and output):
+                    target_encoding = tokenizer(
+                        output, max_length=max_length, padding="max_length",
+                        truncation=True, return_tensors="pt",
+                    )
+                    labels = target_encoding["input_ids"].squeeze()
+                    labels[labels == tokenizer.pad_token_id] = -100
+                else: 
+                    labels = None 
+
+                example_data = {
+                    "input_ids": input_encoding["input_ids"].squeeze(),
+                    "attention_mask": input_encoding["attention_mask"].squeeze(),
+                }
+                if labels is not None:
+                    example_data["labels"] = labels
+                
+                self.examples.append(example_data)
+            
+            log.info(f"Tokenization complete for {dataset_type}. Saving cache to {cache_file}")
+            with open(cache_file, 'wb') as f:
+                pickle.dump(self.examples, f)
+            log.info(f"Cache saved successfully")
+
+    def __len__(self): 
+        return len(self.examples)
+    
+    def __getitem__(self, idx): 
+        return self.examples[idx]
+    
+    def get_raw_example(self, idx): 
+        return self.raw_data[idx]
+
+def load_and_split_json_data(data_path, max_items_from_config=None):
+    log.info(f"Loading data from {data_path}...")
+    if not os.path.exists(data_path):
+        log.error(f"Data file not found: {data_path}")
+        raise FileNotFoundError(f"Data file not found: {data_path}")
+    
+    with open(data_path, "r", encoding="utf-8") as f: 
+        all_data = json.load(f)
+    log.info(f"Successfully loaded {len(all_data)} total items from JSON.")
+    
+    if max_items_from_config is not None and max_items_from_config > 0: 
+        num_to_take = min(max_items_from_config, len(all_data))
+        log.info(f"Keeping the first {num_to_take} samples as per MAX_ITEMS config.")
+        all_data = all_data[:num_to_take]
+    else:
+        log.info("Using the full dataset.")
+            
+    if not all_data: 
+        log.error("No data loaded or remaining.")
+        raise ValueError("No data to process.")
+    
+    if len(all_data) < 20:
+        split_idx = max(1, int(0.5 * len(all_data)))
+        log.warning(f"Dataset very small ({len(all_data)} items). Adjusting split.")
+    else:
+        split_idx = int(0.9 * len(all_data))
+        split_idx = max(1, split_idx)
+
+    train_data = all_data[:split_idx]
+    val_data = all_data[split_idx:]
+    
+    if not val_data and train_data:
+        val_data = [train_data[-1]] 
+        log.warning("Validation set was empty after split, using one sample from training data for validation.")
+        if len(train_data) > 1:
+            train_data = train_data[:-1]
+
+    val_data = val_data[:min(len(val_data), 2000)] if val_data else None 
+
+    if not train_data: 
+        log.error("Training data empty.")
+        raise ValueError("Training data empty.")
+    
+    log.info(f"Train samples: {len(train_data)}, Validation samples: {len(val_data) if val_data else 0}")
+    return train_data, val_data
+
+def find_latest_checkpoint(output_dir):
+    if not os.path.isdir(output_dir):
+        return None
+    
+    checkpoints = [d for d in os.listdir(output_dir) if d.startswith("checkpoint-") and os.path.isdir(os.path.join(output_dir, d))]
+    if not checkpoints:
+        return None
+    
+    checkpoints.sort(key=lambda x: int(x.split('-')[-1]))
+    latest_checkpoint = os.path.join(output_dir, checkpoints[-1])
+    
+    if os.path.exists(os.path.join(latest_checkpoint, "pytorch_model.bin")) or os.path.exists(os.path.join(latest_checkpoint, "model.safetensors")):
+        return latest_checkpoint
+    
+    return None
+
+def clear_cuda_cache():
+    log.info("Clearing CUDA cache...")
+    gc.collect()
+    if torch.cuda.is_available(): 
+        torch.cuda.empty_cache()
+
+def generate_and_log_fixed_sample(model, tokenizer, prompt_text, device, log_prefix="Sample"):
+    log.info(f"\n--- {log_prefix} Generation ---")
+    log.info(f"Input Prompt: {prompt_text}")
+    model.eval() 
+    inputs = tokenizer(prompt_text, return_tensors="pt", max_length=MAX_LENGTH, truncation=True)
+    inputs = {k: v.to(device) for k, v in inputs.items()}
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs, max_length=MAX_LENGTH + 50,
+            num_beams=5, early_stopping=True, no_repeat_ngram_size=3,
+            temperature=0.7, top_k=50, top_p=0.95
+        )
+    generated_text = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    log.info(f"Generated Output: {generated_text}")
+    log.info(f"--- End {log_prefix} Generation ---\n")
+
+class ShowFixedEvalSampleCallback(TrainerCallback):
+    def __init__(self, tokenizer, prompt_text):
+        self.tokenizer = tokenizer
+        self.prompt_text = prompt_text
+        
+    def on_evaluate(self, args, state, control, model=None, **kwargs):
+        if model is None: 
+            return
+        device = next(model.parameters()).device
+        generate_and_log_fixed_sample(model, self.tokenizer, self.prompt_text, device, log_prefix="Evaluation Callback Sample")
+        model.train()
+
+def Train(model_id: str, output_dir: str, data_path: str):
+    os.makedirs(output_dir, exist_ok=True)
+    clear_cuda_cache()
+    
+    # Check for existing checkpoint to resume
+    resume_from_checkpoint = find_latest_checkpoint(output_dir)
+    if resume_from_checkpoint:
+        log.info(f"Found checkpoint to resume from: {resume_from_checkpoint}")
+    else:
+        log.info("No existing checkpoint found, starting fresh training")
+    
+    log.info(f"Attempting to load MyT5Tokenizer for {model_id} (trust_remote_code=True).")
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True)
+        log.info(f"Successfully loaded tokenizer: {type(tokenizer).__name__}")
+    except Exception as e:
+        log.error(f"Failed to load tokenizer for {model_id} (trust_remote_code=True): {e}")
+        return
+
+    train_raw_data, eval_raw_data = load_and_split_json_data(data_path, max_items_from_config=MAX_ITEMS)
+    if not train_raw_data: 
+        return
+
+    train_dataset = SDPromptDataset(train_raw_data, tokenizer, MAX_LENGTH, dataset_type="train")
+    eval_dataset = SDPromptDataset(eval_raw_data, tokenizer, MAX_LENGTH, dataset_type="eval") if eval_raw_data else None
+
+    log.info(f"Loading model: {model_id}")
+    model = AutoModelForSeq2SeqLM.from_pretrained(
+        model_id,
+        torch_dtype=torch.float16 if FP16_TRAINING else torch.float32,
+        device_map="auto", 
+        low_cpu_mem_usage=True,
+    )
+    
+    if GRADIENT_CHECKPOINTING: 
+        model.gradient_checkpointing_enable()
+        log.info("Grad-ckpt enabled.")
+    
+    if OPTIMIZER_CHOICE == "adamw_8bit":
+        try:
+            import bitsandbytes
+            log.info(f"bitsandbytes version: {bitsandbytes.__version__} imported for adamw_8bit.")
+        except ImportError:
+            log.error("bitsandbytes not installed, required for optim='adamw_8bit'. Install: pip install bitsandbytes")
+            return
+            
+    training_args = TrainingArguments(
+        output_dir=output_dir,
+        per_device_train_batch_size=PER_DEVICE_BATCH,
+        per_device_eval_batch_size=PER_DEVICE_BATCH * 2,
+        gradient_accumulation_steps=GRAD_ACC_STEPS,
+        learning_rate=LEARNING_RATE, 
+        num_train_epochs=NUM_TRAIN_EPOCHS, 
+        warmup_steps=WARMUP_STEPS,
+        logging_steps=LOGGING_STEPS, 
+        save_strategy="steps", 
+        save_steps=SAVE_STEPS,
+        eval_strategy="steps" if eval_dataset else "no",
+        eval_steps=EVAL_STEPS if eval_dataset else None,
+        save_total_limit=SAVE_TOTAL_LIMIT, 
+        load_best_model_at_end=True if eval_dataset else False,
+        fp16=FP16_TRAINING, 
+        optim=OPTIMIZER_CHOICE, 
+        max_grad_norm=MAX_GRAD_NORM_CLIP,
+        gradient_checkpointing=GRADIENT_CHECKPOINTING, 
+        group_by_length=True,
+        lr_scheduler_type="cosine", 
+        weight_decay=0.01, 
+        report_to="none",
+    )
+    
+    fixed_sample_callback = ShowFixedEvalSampleCallback(tokenizer=tokenizer, prompt_text=FIXED_PROMPT_FOR_GENERATION)
+    callbacks_to_use = [fixed_sample_callback] if eval_dataset else []
+
+    data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model, padding="longest")
+    trainer = Trainer( 
+        model=model, 
+        args=training_args, 
+        train_dataset=train_dataset,
+        eval_dataset=eval_dataset, 
+        data_collator=data_collator, 
+        tokenizer=tokenizer,
+        callbacks=callbacks_to_use 
+    )
+    
+    log.info(f"Starting training with FP16_TRAINING={FP16_TRAINING}, optim='{OPTIMIZER_CHOICE}', LR={LEARNING_RATE}, GradClip={MAX_GRAD_NORM_CLIP}...")
+    try:
+        trainer.train(resume_from_checkpoint=resume_from_checkpoint)
+    except Exception as e:
+        log.exception(f"Unhandled error during trainer.train(): {e}")
+        return
+        
+    log.info("Training completed.")
+    try:
+        final_model_path = os.path.join(output_dir, "final_model_after_train") 
+        if not os.path.exists(final_model_path): 
+             trainer.save_model(final_model_path)
+             log.info(f"Final model state explicitly saved to {final_model_path}")
+        else:
+            log.info(f"Best model was likely saved by load_best_model_at_end to a checkpoint within {output_dir}")
+    except Exception as e: 
+        log.exception(f"Error saving final explicit model: {e}")
+    log.info("Train function finished.")
+
+def Inference(base_model_id_for_tokenizer: str, trained_model_output_dir: str):
+    log.info(f"\n--- Starting Inference ---")
+    
+    path_to_load_model_from = trained_model_output_dir 
+    potential_final_model = os.path.join(trained_model_output_dir, "final_model_after_train")
+    
+    if os.path.exists(potential_final_model) and (os.path.exists(os.path.join(potential_final_model, "pytorch_model.bin")) or os.path.exists(os.path.join(potential_final_model, "model.safetensors"))):
+        path_to_load_model_from = potential_final_model
+        log.info(f"Found 'final_model_after_train' at: {path_to_load_model_from}")
+    else:
+        latest_checkpoint = find_latest_checkpoint(trained_model_output_dir)
+        if latest_checkpoint:
+            path_to_load_model_from = latest_checkpoint
+            log.info(f"Found latest checkpoint: {path_to_load_model_from}")
+        elif not (os.path.exists(os.path.join(path_to_load_model_from, "pytorch_model.bin")) or os.path.exists(os.path.join(path_to_load_model_from, "model.safetensors"))):
+            log.error(f"No valid model found in {trained_model_output_dir} or its subdirectories. Cannot run inference.")
+            return
+
+    log.info(f"Attempting to load fine-tuned model from: {path_to_load_model_from}")
+
+    try:
+        model = AutoModelForSeq2SeqLM.from_pretrained(path_to_load_model_from, device_map="auto")
+        try:
+            tokenizer = AutoTokenizer.from_pretrained(path_to_load_model_from, trust_remote_code=True)
+        except Exception: 
+            log.warning(f"Could not load tokenizer from {path_to_load_model_from}, trying base {base_model_id_for_tokenizer}")
+            tokenizer = AutoTokenizer.from_pretrained(base_model_id_for_tokenizer, trust_remote_code=True)
+        log.info(f"Successfully loaded model and tokenizer for inference. Model is on: {model.device}")
+    except Exception as e:
+        log.error(f"Failed to load model or tokenizer for inference: {e}")
+        return
+
+    device = next(model.parameters()).device
+    generate_and_log_fixed_sample(model, tokenizer, FIXED_PROMPT_FOR_GENERATION, device, log_prefix="Final Inference")
+    log.info(f"--- Inference Demo Finished ---")
+
+def main():
+    Train('Tomlim/myt5-large', 'trained_model', 'DiscordPromptSD.json')
+    Inference('Tomlim/myt5-large', 'trained_model')
+
+if __name__ == "__main__":
+    main()
+
+# - SFW Cyberpunk City: `Nikon Z9 200mm f_8 ISO 160, (giant rifle structure), flawless ornate architecture, cyberpunk, neon lights, busy street, realistic, ray tracing, hasselblad`
+# - **SFW Fantasy Dragon Rider: `masterpiece, best quality, cinematic lighting, 1girl, solo, <lora:add_detail:0.55>`
+# - **NSFW Anime Succubus: `masterpiece, best quality, highly detailed background, intricate, 1girl, (full-face blush, aroused:1.3), long hair, medium breasts, nipples`

trainer_state.json

@@ -0,0 +1,3434 @@
+{
+  "best_global_step": 24500,
+  "best_metric": 0.5816783905029297,
+  "best_model_checkpoint": "trained_model\\checkpoint-19000",
+  "epoch": 0.7204104898971434,
+  "eval_steps": 500,
+  "global_step": 25000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0014408209797942867,
+      "learning_rate": 1.225e-05,
+      "loss": 4.8426,
+      "step": 50
+    },
+    {
+      "epoch": 0.0028816419595885734,
+      "learning_rate": 2.4750000000000002e-05,
+      "loss": 1.8657,
+      "step": 100
+    },
+    {
+      "epoch": 0.004322462939382861,
+      "learning_rate": 3.7250000000000004e-05,
+      "loss": 1.6201,
+      "step": 150
+    },
+    {
+      "epoch": 0.005763283919177147,
+      "learning_rate": 4.975e-05,
+      "loss": 1.4427,
+      "step": 200
+    },
+    {
+      "epoch": 0.007204104898971434,
+      "learning_rate": 4.999975117874187e-05,
+      "loss": 1.3279,
+      "step": 250
+    },
+    {
+      "epoch": 0.008644925878765721,
+      "learning_rate": 4.99989843045885e-05,
+      "loss": 1.2472,
+      "step": 300
+    },
+    {
+      "epoch": 0.010085746858560007,
+      "learning_rate": 4.9997699289801957e-05,
+      "loss": 1.2063,
+      "step": 350
+    },
+    {
+      "epoch": 0.011526567838354294,
+      "learning_rate": 4.999589616101606e-05,
+      "loss": 1.1545,
+      "step": 400
+    },
+    {
+      "epoch": 0.012967388818148582,
+      "learning_rate": 4.9993574955603304e-05,
+      "loss": 1.1197,
+      "step": 450
+    },
+    {
+      "epoch": 0.014408209797942868,
+      "learning_rate": 4.9990735721674075e-05,
+      "loss": 1.0864,
+      "step": 500
+    },
+    {
+      "epoch": 0.014408209797942868,
+      "eval_loss": 1.0425041913986206,
+      "eval_runtime": 80.2488,
+      "eval_samples_per_second": 24.922,
+      "eval_steps_per_second": 12.461,
+      "step": 500
+    },
+    {
+      "epoch": 0.015849030777737156,
+      "learning_rate": 4.9987378518075656e-05,
+      "loss": 1.0661,
+      "step": 550
+    },
+    {
+      "epoch": 0.017289851757531442,
+      "learning_rate": 4.998350341439103e-05,
+      "loss": 1.0467,
+      "step": 600
+    },
+    {
+      "epoch": 0.01873067273732573,
+      "learning_rate": 4.997911049093741e-05,
+      "loss": 1.0132,
+      "step": 650
+    },
+    {
+      "epoch": 0.020171493717120015,
+      "learning_rate": 4.9974199838764554e-05,
+      "loss": 1.0067,
+      "step": 700
+    },
+    {
+      "epoch": 0.0216123146969143,
+      "learning_rate": 4.9968771559652967e-05,
+      "loss": 1.0036,
+      "step": 750
+    },
+    {
+      "epoch": 0.023053135676708587,
+      "learning_rate": 4.996282576611168e-05,
+      "loss": 0.9847,
+      "step": 800
+    },
+    {
+      "epoch": 0.024493956656502874,
+      "learning_rate": 4.9956362581375996e-05,
+      "loss": 0.975,
+      "step": 850
+    },
+    {
+      "epoch": 0.025934777636297163,
+      "learning_rate": 4.994938213940489e-05,
+      "loss": 0.9665,
+      "step": 900
+    },
+    {
+      "epoch": 0.02737559861609145,
+      "learning_rate": 4.994188458487828e-05,
+      "loss": 0.9548,
+      "step": 950
+    },
+    {
+      "epoch": 0.028816419595885736,
+      "learning_rate": 4.9933870073193985e-05,
+      "loss": 0.9419,
+      "step": 1000
+    },
+    {
+      "epoch": 0.028816419595885736,
+      "eval_loss": 0.8812388777732849,
+      "eval_runtime": 80.8889,
+      "eval_samples_per_second": 24.725,
+      "eval_steps_per_second": 12.363,
+      "step": 1000
+    },
+    {
+      "epoch": 0.030257240575680022,
+      "learning_rate": 4.992533877046451e-05,
+      "loss": 0.9482,
+      "step": 1050
+    },
+    {
+      "epoch": 0.03169806155547431,
+      "learning_rate": 4.991629085351363e-05,
+      "loss": 0.9323,
+      "step": 1100
+    },
+    {
+      "epoch": 0.033138882535268595,
+      "learning_rate": 4.99067265098727e-05,
+      "loss": 0.9204,
+      "step": 1150
+    },
+    {
+      "epoch": 0.034579703515062885,
+      "learning_rate": 4.9896645937776775e-05,
+      "loss": 0.9359,
+      "step": 1200
+    },
+    {
+      "epoch": 0.03602052449485717,
+      "learning_rate": 4.98860493461605e-05,
+      "loss": 0.9161,
+      "step": 1250
+    },
+    {
+      "epoch": 0.03746134547465146,
+      "learning_rate": 4.98749369546538e-05,
+      "loss": 0.9078,
+      "step": 1300
+    },
+    {
+      "epoch": 0.03890216645444574,
+      "learning_rate": 4.986330899357729e-05,
+      "loss": 0.8987,
+      "step": 1350
+    },
+    {
+      "epoch": 0.04034298743424003,
+      "learning_rate": 4.985116570393751e-05,
+      "loss": 0.8908,
+      "step": 1400
+    },
+    {
+      "epoch": 0.04178380841403432,
+      "learning_rate": 4.983850733742197e-05,
+      "loss": 0.8959,
+      "step": 1450
+    },
+    {
+      "epoch": 0.0432246293938286,
+      "learning_rate": 4.9825334156393885e-05,
+      "loss": 0.8728,
+      "step": 1500
+    },
+    {
+      "epoch": 0.0432246293938286,
+      "eval_loss": 0.8146671652793884,
+      "eval_runtime": 79.7347,
+      "eval_samples_per_second": 25.083,
+      "eval_steps_per_second": 12.542,
+      "step": 1500
+    },
+    {
+      "epoch": 0.04466545037362289,
+      "learning_rate": 4.9811646433886774e-05,
+      "loss": 0.8628,
+      "step": 1550
+    },
+    {
+      "epoch": 0.046106271353417175,
+      "learning_rate": 4.979744445359876e-05,
+      "loss": 0.8627,
+      "step": 1600
+    },
+    {
+      "epoch": 0.047547092333211464,
+      "learning_rate": 4.978272850988673e-05,
+      "loss": 0.8474,
+      "step": 1650
+    },
+    {
+      "epoch": 0.04898791331300575,
+      "learning_rate": 4.976749890776021e-05,
+      "loss": 0.8706,
+      "step": 1700
+    },
+    {
+      "epoch": 0.05042873429280004,
+      "learning_rate": 4.9751755962875054e-05,
+      "loss": 0.8522,
+      "step": 1750
+    },
+    {
+      "epoch": 0.05186955527259433,
+      "learning_rate": 4.9735500001526894e-05,
+      "loss": 0.8411,
+      "step": 1800
+    },
+    {
+      "epoch": 0.05331037625238861,
+      "learning_rate": 4.971873136064438e-05,
+      "loss": 0.8373,
+      "step": 1850
+    },
+    {
+      "epoch": 0.0547511972321829,
+      "learning_rate": 4.970145038778221e-05,
+      "loss": 0.8577,
+      "step": 1900
+    },
+    {
+      "epoch": 0.05619201821197718,
+      "learning_rate": 4.9683657441113884e-05,
+      "loss": 0.8387,
+      "step": 1950
+    },
+    {
+      "epoch": 0.05763283919177147,
+      "learning_rate": 4.9665352889424354e-05,
+      "loss": 0.8216,
+      "step": 2000
+    },
+    {
+      "epoch": 0.05763283919177147,
+      "eval_loss": 0.7666909098625183,
+      "eval_runtime": 79.8485,
+      "eval_samples_per_second": 25.047,
+      "eval_steps_per_second": 12.524,
+      "step": 2000
+    },
+    {
+      "epoch": 0.059073660171565755,
+      "learning_rate": 4.964653711210231e-05,
+      "loss": 0.8395,
+      "step": 2050
+    },
+    {
+      "epoch": 0.060514481151360044,
+      "learning_rate": 4.962721049913233e-05,
+      "loss": 0.8323,
+      "step": 2100
+    },
+    {
+      "epoch": 0.061955302131154334,
+      "learning_rate": 4.960737345108685e-05,
+      "loss": 0.8028,
+      "step": 2150
+    },
+    {
+      "epoch": 0.06339612311094862,
+      "learning_rate": 4.958702637911779e-05,
+      "loss": 0.818,
+      "step": 2200
+    },
+    {
+      "epoch": 0.0648369440907429,
+      "learning_rate": 4.9566169704948065e-05,
+      "loss": 0.8121,
+      "step": 2250
+    },
+    {
+      "epoch": 0.06627776507053719,
+      "learning_rate": 4.9544803860862856e-05,
+      "loss": 0.8276,
+      "step": 2300
+    },
+    {
+      "epoch": 0.06771858605033147,
+      "learning_rate": 4.952292928970065e-05,
+      "loss": 0.8066,
+      "step": 2350
+    },
+    {
+      "epoch": 0.06915940703012577,
+      "learning_rate": 4.950054644484401e-05,
+      "loss": 0.8043,
+      "step": 2400
+    },
+    {
+      "epoch": 0.07060022800992005,
+      "learning_rate": 4.9477655790210256e-05,
+      "loss": 0.7998,
+      "step": 2450
+    },
+    {
+      "epoch": 0.07204104898971433,
+      "learning_rate": 4.945425780024179e-05,
+      "loss": 0.8015,
+      "step": 2500
+    },
+    {
+      "epoch": 0.07204104898971433,
+      "eval_loss": 0.7467618584632874,
+      "eval_runtime": 82.6828,
+      "eval_samples_per_second": 24.189,
+      "eval_steps_per_second": 12.094,
+      "step": 2500
+    },
+    {
+      "epoch": 0.07348186996950863,
+      "learning_rate": 4.943035295989629e-05,
+      "loss": 0.8048,
+      "step": 2550
+    },
+    {
+      "epoch": 0.07492269094930291,
+      "learning_rate": 4.940594176463665e-05,
+      "loss": 0.7857,
+      "step": 2600
+    },
+    {
+      "epoch": 0.0763635119290972,
+      "learning_rate": 4.938102472042071e-05,
+      "loss": 0.8022,
+      "step": 2650
+    },
+    {
+      "epoch": 0.07780433290889148,
+      "learning_rate": 4.935560234369078e-05,
+      "loss": 0.7792,
+      "step": 2700
+    },
+    {
+      "epoch": 0.07924515388868578,
+      "learning_rate": 4.932967516136291e-05,
+      "loss": 0.7967,
+      "step": 2750
+    },
+    {
+      "epoch": 0.08068597486848006,
+      "learning_rate": 4.9303243710816006e-05,
+      "loss": 0.7749,
+      "step": 2800
+    },
+    {
+      "epoch": 0.08212679584827434,
+      "learning_rate": 4.927630853988068e-05,
+      "loss": 0.7777,
+      "step": 2850
+    },
+    {
+      "epoch": 0.08356761682806864,
+      "learning_rate": 4.924887020682785e-05,
+      "loss": 0.7793,
+      "step": 2900
+    },
+    {
+      "epoch": 0.08500843780786292,
+      "learning_rate": 4.922092928035725e-05,
+      "loss": 0.7788,
+      "step": 2950
+    },
+    {
+      "epoch": 0.0864492587876572,
+      "learning_rate": 4.919248633958557e-05,
+      "loss": 0.7706,
+      "step": 3000
+    },
+    {
+      "epoch": 0.0864492587876572,
+      "eval_loss": 0.7307117581367493,
+      "eval_runtime": 91.6423,
+      "eval_samples_per_second": 21.824,
+      "eval_steps_per_second": 10.912,
+      "step": 3000
+    },
+    {
+      "epoch": 0.08789007976745149,
+      "learning_rate": 4.91635419740345e-05,
+      "loss": 0.7728,
+      "step": 3050
+    },
+    {
+      "epoch": 0.08933090074724578,
+      "learning_rate": 4.913409678361849e-05,
+      "loss": 0.7687,
+      "step": 3100
+    },
+    {
+      "epoch": 0.09077172172704007,
+      "learning_rate": 4.910415137863232e-05,
+      "loss": 0.7868,
+      "step": 3150
+    },
+    {
+      "epoch": 0.09221254270683435,
+      "learning_rate": 4.907370637973845e-05,
+      "loss": 0.7774,
+      "step": 3200
+    },
+    {
+      "epoch": 0.09365336368662865,
+      "learning_rate": 4.9042762417954144e-05,
+      "loss": 0.754,
+      "step": 3250
+    },
+    {
+      "epoch": 0.09509418466642293,
+      "learning_rate": 4.901132013463844e-05,
+      "loss": 0.7621,
+      "step": 3300
+    },
+    {
+      "epoch": 0.09653500564621721,
+      "learning_rate": 4.897938018147878e-05,
+      "loss": 0.7558,
+      "step": 3350
+    },
+    {
+      "epoch": 0.0979758266260115,
+      "learning_rate": 4.894694322047757e-05,
+      "loss": 0.748,
+      "step": 3400
+    },
+    {
+      "epoch": 0.09941664760580579,
+      "learning_rate": 4.891400992393842e-05,
+      "loss": 0.759,
+      "step": 3450
+    },
+    {
+      "epoch": 0.10085746858560007,
+      "learning_rate": 4.888058097445223e-05,
+      "loss": 0.7614,
+      "step": 3500
+    },
+    {
+      "epoch": 0.10085746858560007,
+      "eval_loss": 0.7035399079322815,
+      "eval_runtime": 81.6585,
+      "eval_samples_per_second": 24.492,
+      "eval_steps_per_second": 12.246,
+      "step": 3500
+    },
+    {
+      "epoch": 0.10229828956539436,
+      "learning_rate": 4.8846657064883025e-05,
+      "loss": 0.7653,
+      "step": 3550
+    },
+    {
+      "epoch": 0.10373911054518865,
+      "learning_rate": 4.881223889835363e-05,
+      "loss": 0.7561,
+      "step": 3600
+    },
+    {
+      "epoch": 0.10517993152498294,
+      "learning_rate": 4.877732718823103e-05,
+      "loss": 0.739,
+      "step": 3650
+    },
+    {
+      "epoch": 0.10662075250477722,
+      "learning_rate": 4.8741922658111656e-05,
+      "loss": 0.7314,
+      "step": 3700
+    },
+    {
+      "epoch": 0.1080615734845715,
+      "learning_rate": 4.870602604180634e-05,
+      "loss": 0.7459,
+      "step": 3750
+    },
+    {
+      "epoch": 0.1095023944643658,
+      "learning_rate": 4.866963808332515e-05,
+      "loss": 0.7445,
+      "step": 3800
+    },
+    {
+      "epoch": 0.11094321544416008,
+      "learning_rate": 4.86327595368619e-05,
+      "loss": 0.7233,
+      "step": 3850
+    },
+    {
+      "epoch": 0.11238403642395436,
+      "learning_rate": 4.859539116677858e-05,
+      "loss": 0.743,
+      "step": 3900
+    },
+    {
+      "epoch": 0.11382485740374866,
+      "learning_rate": 4.855753374758951e-05,
+      "loss": 0.7251,
+      "step": 3950
+    },
+    {
+      "epoch": 0.11526567838354294,
+      "learning_rate": 4.851918806394525e-05,
+      "loss": 0.7273,
+      "step": 4000
+    },
+    {
+      "epoch": 0.11526567838354294,
+      "eval_loss": 0.6973708271980286,
+      "eval_runtime": 85.9653,
+      "eval_samples_per_second": 23.265,
+      "eval_steps_per_second": 11.633,
+      "step": 4000
+    },
+    {
+      "epoch": 0.11670649936333723,
+      "learning_rate": 4.8480354910616355e-05,
+      "loss": 0.7324,
+      "step": 4050
+    },
+    {
+      "epoch": 0.11814732034313151,
+      "learning_rate": 4.844103509247692e-05,
+      "loss": 0.7307,
+      "step": 4100
+    },
+    {
+      "epoch": 0.1195881413229258,
+      "learning_rate": 4.840122942448786e-05,
+      "loss": 0.7299,
+      "step": 4150
+    },
+    {
+      "epoch": 0.12102896230272009,
+      "learning_rate": 4.836093873168006e-05,
+      "loss": 0.7336,
+      "step": 4200
+    },
+    {
+      "epoch": 0.12246978328251437,
+      "learning_rate": 4.832016384913726e-05,
+      "loss": 0.7377,
+      "step": 4250
+    },
+    {
+      "epoch": 0.12391060426230867,
+      "learning_rate": 4.827890562197873e-05,
+      "loss": 0.7233,
+      "step": 4300
+    },
+    {
+      "epoch": 0.12535142524210294,
+      "learning_rate": 4.823716490534177e-05,
+      "loss": 0.7234,
+      "step": 4350
+    },
+    {
+      "epoch": 0.12679224622189725,
+      "learning_rate": 4.8194942564363975e-05,
+      "loss": 0.7171,
+      "step": 4400
+    },
+    {
+      "epoch": 0.12823306720169153,
+      "learning_rate": 4.815223947416533e-05,
+      "loss": 0.7242,
+      "step": 4450
+    },
+    {
+      "epoch": 0.1296738881814858,
+      "learning_rate": 4.810905651983004e-05,
+      "loss": 0.7214,
+      "step": 4500
+    },
+    {
+      "epoch": 0.1296738881814858,
+      "eval_loss": 0.6867671012878418,
+      "eval_runtime": 97.3752,
+      "eval_samples_per_second": 20.539,
+      "eval_steps_per_second": 10.27,
+      "step": 4500
+    },
+    {
+      "epoch": 0.1311147091612801,
+      "learning_rate": 4.8065394596388186e-05,
+      "loss": 0.7045,
+      "step": 4550
+    },
+    {
+      "epoch": 0.13255553014107438,
+      "learning_rate": 4.8021254608797214e-05,
+      "loss": 0.7144,
+      "step": 4600
+    },
+    {
+      "epoch": 0.13399635112086866,
+      "learning_rate": 4.7976637471923145e-05,
+      "loss": 0.7194,
+      "step": 4650
+    },
+    {
+      "epoch": 0.13543717210066294,
+      "learning_rate": 4.7931544110521584e-05,
+      "loss": 0.7149,
+      "step": 4700
+    },
+    {
+      "epoch": 0.13687799308045726,
+      "learning_rate": 4.788597545921865e-05,
+      "loss": 0.7198,
+      "step": 4750
+    },
+    {
+      "epoch": 0.13831881406025154,
+      "learning_rate": 4.7839932462491485e-05,
+      "loss": 0.6961,
+      "step": 4800
+    },
+    {
+      "epoch": 0.13975963504004582,
+      "learning_rate": 4.779341607464877e-05,
+      "loss": 0.716,
+      "step": 4850
+    },
+    {
+      "epoch": 0.1412004560198401,
+      "learning_rate": 4.774642725981089e-05,
+      "loss": 0.7102,
+      "step": 4900
+    },
+    {
+      "epoch": 0.1426412769996344,
+      "learning_rate": 4.769896699189001e-05,
+      "loss": 0.702,
+      "step": 4950
+    },
+    {
+      "epoch": 0.14408209797942867,
+      "learning_rate": 4.765103625456979e-05,
+      "loss": 0.7044,
+      "step": 5000
+    },
+    {
+      "epoch": 0.14408209797942867,
+      "eval_loss": 0.6795601844787598,
+      "eval_runtime": 83.068,
+      "eval_samples_per_second": 24.077,
+      "eval_steps_per_second": 12.038,
+      "step": 5000
+    },
+    {
+      "epoch": 0.14552291895922295,
+      "learning_rate": 4.760263604128511e-05,
+      "loss": 0.6948,
+      "step": 5050
+    },
+    {
+      "epoch": 0.14696373993901726,
+      "learning_rate": 4.755376735520143e-05,
+      "loss": 0.7073,
+      "step": 5100
+    },
+    {
+      "epoch": 0.14840456091881155,
+      "learning_rate": 4.750443120919395e-05,
+      "loss": 0.6887,
+      "step": 5150
+    },
+    {
+      "epoch": 0.14984538189860583,
+      "learning_rate": 4.7454628625826716e-05,
+      "loss": 0.6787,
+      "step": 5200
+    },
+    {
+      "epoch": 0.1512862028784001,
+      "learning_rate": 4.740436063733133e-05,
+      "loss": 0.6967,
+      "step": 5250
+    },
+    {
+      "epoch": 0.1527270238581944,
+      "learning_rate": 4.7353628285585625e-05,
+      "loss": 0.7072,
+      "step": 5300
+    },
+    {
+      "epoch": 0.15416784483798868,
+      "learning_rate": 4.730243262209204e-05,
+      "loss": 0.6821,
+      "step": 5350
+    },
+    {
+      "epoch": 0.15560866581778296,
+      "learning_rate": 4.7250774707955816e-05,
+      "loss": 0.6674,
+      "step": 5400
+    },
+    {
+      "epoch": 0.15704948679757727,
+      "learning_rate": 4.719865561386303e-05,
+      "loss": 0.6979,
+      "step": 5450
+    },
+    {
+      "epoch": 0.15849030777737155,
+      "learning_rate": 4.714607642005839e-05,
+      "loss": 0.688,
+      "step": 5500
+    },
+    {
+      "epoch": 0.15849030777737155,
+      "eval_loss": 0.6694391369819641,
+      "eval_runtime": 82.0167,
+      "eval_samples_per_second": 24.385,
+      "eval_steps_per_second": 12.193,
+      "step": 5500
+    },
+    {
+      "epoch": 0.15993112875716584,
+      "learning_rate": 4.709303821632284e-05,
+      "loss": 0.7001,
+      "step": 5550
+    },
+    {
+      "epoch": 0.16137194973696012,
+      "learning_rate": 4.703954210195097e-05,
+      "loss": 0.6761,
+      "step": 5600
+    },
+    {
+      "epoch": 0.1628127707167544,
+      "learning_rate": 4.6985589185728273e-05,
+      "loss": 0.6769,
+      "step": 5650
+    },
+    {
+      "epoch": 0.16425359169654868,
+      "learning_rate": 4.693118058590809e-05,
+      "loss": 0.6892,
+      "step": 5700
+    },
+    {
+      "epoch": 0.16569441267634297,
+      "learning_rate": 4.68763174301885e-05,
+      "loss": 0.6897,
+      "step": 5750
+    },
+    {
+      "epoch": 0.16713523365613728,
+      "learning_rate": 4.6821000855688903e-05,
+      "loss": 0.67,
+      "step": 5800
+    },
+    {
+      "epoch": 0.16857605463593156,
+      "learning_rate": 4.6765232008926485e-05,
+      "loss": 0.6681,
+      "step": 5850
+    },
+    {
+      "epoch": 0.17001687561572584,
+      "learning_rate": 4.6709012045792426e-05,
+      "loss": 0.6885,
+      "step": 5900
+    },
+    {
+      "epoch": 0.17145769659552013,
+      "learning_rate": 4.665234213152796e-05,
+      "loss": 0.6764,
+      "step": 5950
+    },
+    {
+      "epoch": 0.1728985175753144,
+      "learning_rate": 4.659522344070022e-05,
+      "loss": 0.675,
+      "step": 6000
+    },
+    {
+      "epoch": 0.1728985175753144,
+      "eval_loss": 0.6623409390449524,
+      "eval_runtime": 77.9602,
+      "eval_samples_per_second": 25.654,
+      "eval_steps_per_second": 12.827,
+      "step": 6000
+    },
+    {
+      "epoch": 0.1743393385551087,
+      "learning_rate": 4.653765715717788e-05,
+      "loss": 0.6609,
+      "step": 6050
+    },
+    {
+      "epoch": 0.17578015953490297,
+      "learning_rate": 4.6479644474106657e-05,
+      "loss": 0.6632,
+      "step": 6100
+    },
+    {
+      "epoch": 0.17722098051469728,
+      "learning_rate": 4.642118659388452e-05,
+      "loss": 0.6764,
+      "step": 6150
+    },
+    {
+      "epoch": 0.17866180149449157,
+      "learning_rate": 4.636228472813682e-05,
+      "loss": 0.6659,
+      "step": 6200
+    },
+    {
+      "epoch": 0.18010262247428585,
+      "learning_rate": 4.630294009769115e-05,
+      "loss": 0.6946,
+      "step": 6250
+    },
+    {
+      "epoch": 0.18154344345408013,
+      "learning_rate": 4.624315393255207e-05,
+      "loss": 0.6616,
+      "step": 6300
+    },
+    {
+      "epoch": 0.18298426443387442,
+      "learning_rate": 4.618292747187559e-05,
+      "loss": 0.6614,
+      "step": 6350
+    },
+    {
+      "epoch": 0.1844250854136687,
+      "learning_rate": 4.612226196394347e-05,
+      "loss": 0.6712,
+      "step": 6400
+    },
+    {
+      "epoch": 0.18586590639346298,
+      "learning_rate": 4.6061158666137376e-05,
+      "loss": 0.6708,
+      "step": 6450
+    },
+    {
+      "epoch": 0.1873067273732573,
+      "learning_rate": 4.599961884491283e-05,
+      "loss": 0.649,
+      "step": 6500
+    },
+    {
+      "epoch": 0.1873067273732573,
+      "eval_loss": 0.6525030732154846,
+      "eval_runtime": 79.56,
+      "eval_samples_per_second": 25.138,
+      "eval_steps_per_second": 12.569,
+      "step": 6500
+    },
+    {
+      "epoch": 0.18874754835305158,
+      "learning_rate": 4.593764377577293e-05,
+      "loss": 0.6663,
+      "step": 6550
+    },
+    {
+      "epoch": 0.19018836933284586,
+      "learning_rate": 4.587523474324193e-05,
+      "loss": 0.654,
+      "step": 6600
+    },
+    {
+      "epoch": 0.19162919031264014,
+      "learning_rate": 4.58123930408386e-05,
+      "loss": 0.646,
+      "step": 6650
+    },
+    {
+      "epoch": 0.19307001129243442,
+      "learning_rate": 4.5749119971049424e-05,
+      "loss": 0.6426,
+      "step": 6700
+    },
+    {
+      "epoch": 0.1945108322722287,
+      "learning_rate": 4.5685416845301644e-05,
+      "loss": 0.6521,
+      "step": 6750
+    },
+    {
+      "epoch": 0.195951653252023,
+      "learning_rate": 4.562128498393601e-05,
+      "loss": 0.6577,
+      "step": 6800
+    },
+    {
+      "epoch": 0.1973924742318173,
+      "learning_rate": 4.5556725716179454e-05,
+      "loss": 0.6592,
+      "step": 6850
+    },
+    {
+      "epoch": 0.19883329521161158,
+      "learning_rate": 4.5491740380117534e-05,
+      "loss": 0.6621,
+      "step": 6900
+    },
+    {
+      "epoch": 0.20027411619140587,
+      "learning_rate": 4.5426330322666696e-05,
+      "loss": 0.6401,
+      "step": 6950
+    },
+    {
+      "epoch": 0.20171493717120015,
+      "learning_rate": 4.536049689954637e-05,
+      "loss": 0.651,
+      "step": 7000
+    },
+    {
+      "epoch": 0.20171493717120015,
+      "eval_loss": 0.6546201109886169,
+      "eval_runtime": 78.0665,
+      "eval_samples_per_second": 25.619,
+      "eval_steps_per_second": 12.81,
+      "step": 7000
+    },
+    {
+      "epoch": 0.20315575815099443,
+      "learning_rate": 4.529424147525086e-05,
+      "loss": 0.6629,
+      "step": 7050
+    },
+    {
+      "epoch": 0.2045965791307887,
+      "learning_rate": 4.522756542302103e-05,
+      "loss": 0.6654,
+      "step": 7100
+    },
+    {
+      "epoch": 0.206037400110583,
+      "learning_rate": 4.516047012481594e-05,
+      "loss": 0.6455,
+      "step": 7150
+    },
+    {
+      "epoch": 0.2074782210903773,
+      "learning_rate": 4.509295697128407e-05,
+      "loss": 0.6417,
+      "step": 7200
+    },
+    {
+      "epoch": 0.2089190420701716,
+      "learning_rate": 4.502502736173462e-05,
+      "loss": 0.6423,
+      "step": 7250
+    },
+    {
+      "epoch": 0.21035986304996587,
+      "learning_rate": 4.495668270410841e-05,
+      "loss": 0.654,
+      "step": 7300
+    },
+    {
+      "epoch": 0.21180068402976016,
+      "learning_rate": 4.488792441494876e-05,
+      "loss": 0.6385,
+      "step": 7350
+    },
+    {
+      "epoch": 0.21324150500955444,
+      "learning_rate": 4.481875391937211e-05,
+      "loss": 0.6462,
+      "step": 7400
+    },
+    {
+      "epoch": 0.21468232598934872,
+      "learning_rate": 4.4749172651038493e-05,
+      "loss": 0.6564,
+      "step": 7450
+    },
+    {
+      "epoch": 0.216123146969143,
+      "learning_rate": 4.4679182052121794e-05,
+      "loss": 0.6293,
+      "step": 7500
+    },
+    {
+      "epoch": 0.216123146969143,
+      "eval_loss": 0.6418629884719849,
+      "eval_runtime": 79.3564,
+      "eval_samples_per_second": 25.203,
+      "eval_steps_per_second": 12.601,
+      "step": 7500
+    },
+    {
+      "epoch": 0.21756396794893731,
+      "learning_rate": 4.4608783573279867e-05,
+      "loss": 0.6527,
+      "step": 7550
+    },
+    {
+      "epoch": 0.2190047889287316,
+      "learning_rate": 4.453797867362449e-05,
+      "loss": 0.6462,
+      "step": 7600
+    },
+    {
+      "epoch": 0.22044560990852588,
+      "learning_rate": 4.446676882069112e-05,
+      "loss": 0.633,
+      "step": 7650
+    },
+    {
+      "epoch": 0.22188643088832016,
+      "learning_rate": 4.4395155490408456e-05,
+      "loss": 0.6243,
+      "step": 7700
+    },
+    {
+      "epoch": 0.22332725186811445,
+      "learning_rate": 4.4323140167067835e-05,
+      "loss": 0.6415,
+      "step": 7750
+    },
+    {
+      "epoch": 0.22476807284790873,
+      "learning_rate": 4.425072434329252e-05,
+      "loss": 0.6413,
+      "step": 7800
+    },
+    {
+      "epoch": 0.226208893827703,
+      "learning_rate": 4.4177909520006714e-05,
+      "loss": 0.6213,
+      "step": 7850
+    },
+    {
+      "epoch": 0.22764971480749732,
+      "learning_rate": 4.4104697206404484e-05,
+      "loss": 0.6647,
+      "step": 7900
+    },
+    {
+      "epoch": 0.2290905357872916,
+      "learning_rate": 4.403108891991846e-05,
+      "loss": 0.6275,
+      "step": 7950
+    },
+    {
+      "epoch": 0.2305313567670859,
+      "learning_rate": 4.395708618618839e-05,
+      "loss": 0.6367,
+      "step": 8000
+    },
+    {
+      "epoch": 0.2305313567670859,
+      "eval_loss": 0.6377586722373962,
+      "eval_runtime": 79.238,
+      "eval_samples_per_second": 25.24,
+      "eval_steps_per_second": 12.62,
+      "step": 8000
+    },
+    {
+      "epoch": 0.23197217774688017,
+      "learning_rate": 4.3882690539029526e-05,
+      "loss": 0.6302,
+      "step": 8050
+    },
+    {
+      "epoch": 0.23341299872667445,
+      "learning_rate": 4.380790352040082e-05,
+      "loss": 0.6295,
+      "step": 8100
+    },
+    {
+      "epoch": 0.23485381970646874,
+      "learning_rate": 4.373272668037298e-05,
+      "loss": 0.6278,
+      "step": 8150
+    },
+    {
+      "epoch": 0.23629464068626302,
+      "learning_rate": 4.365716157709632e-05,
+      "loss": 0.6136,
+      "step": 8200
+    },
+    {
+      "epoch": 0.23773546166605733,
+      "learning_rate": 4.35812097767685e-05,
+      "loss": 0.6312,
+      "step": 8250
+    },
+    {
+      "epoch": 0.2391762826458516,
+      "learning_rate": 4.350487285360203e-05,
+      "loss": 0.6261,
+      "step": 8300
+    },
+    {
+      "epoch": 0.2406171036256459,
+      "learning_rate": 4.3428152389791654e-05,
+      "loss": 0.6348,
+      "step": 8350
+    },
+    {
+      "epoch": 0.24205792460544018,
+      "learning_rate": 4.335104997548157e-05,
+      "loss": 0.625,
+      "step": 8400
+    },
+    {
+      "epoch": 0.24349874558523446,
+      "learning_rate": 4.3273567208732454e-05,
+      "loss": 0.6418,
+      "step": 8450
+    },
+    {
+      "epoch": 0.24493956656502874,
+      "learning_rate": 4.319570569548834e-05,
+      "loss": 0.6282,
+      "step": 8500
+    },
+    {
+      "epoch": 0.24493956656502874,
+      "eval_loss": 0.6374307870864868,
+      "eval_runtime": 81.6998,
+      "eval_samples_per_second": 24.48,
+      "eval_steps_per_second": 12.24,
+      "step": 8500
+    },
+    {
+      "epoch": 0.24638038754482305,
+      "learning_rate": 4.3117467049543355e-05,
+      "loss": 0.6284,
+      "step": 8550
+    },
+    {
+      "epoch": 0.24782120852461734,
+      "learning_rate": 4.303885289250823e-05,
+      "loss": 0.6069,
+      "step": 8600
+    },
+    {
+      "epoch": 0.24926202950441162,
+      "learning_rate": 4.2959864853776736e-05,
+      "loss": 0.6072,
+      "step": 8650
+    },
+    {
+      "epoch": 0.2507028504842059,
+      "learning_rate": 4.288050457049188e-05,
+      "loss": 0.6032,
+      "step": 8700
+    },
+    {
+      "epoch": 0.2521436714640002,
+      "learning_rate": 4.280077368751198e-05,
+      "loss": 0.6178,
+      "step": 8750
+    },
+    {
+      "epoch": 0.2535844924437945,
+      "learning_rate": 4.27206738573766e-05,
+      "loss": 0.6217,
+      "step": 8800
+    },
+    {
+      "epoch": 0.25502531342358875,
+      "learning_rate": 4.264020674027224e-05,
+      "loss": 0.6204,
+      "step": 8850
+    },
+    {
+      "epoch": 0.25646613440338306,
+      "learning_rate": 4.255937400399799e-05,
+      "loss": 0.617,
+      "step": 8900
+    },
+    {
+      "epoch": 0.2579069553831773,
+      "learning_rate": 4.247817732393091e-05,
+      "loss": 0.6161,
+      "step": 8950
+    },
+    {
+      "epoch": 0.2593477763629716,
+      "learning_rate": 4.239661838299136e-05,
+      "loss": 0.6009,
+      "step": 9000
+    },
+    {
+      "epoch": 0.2593477763629716,
+      "eval_loss": 0.6277636885643005,
+      "eval_runtime": 80.2755,
+      "eval_samples_per_second": 24.914,
+      "eval_steps_per_second": 12.457,
+      "step": 9000
+    },
+    {
+      "epoch": 0.2607885973427659,
+      "learning_rate": 4.231469887160806e-05,
+      "loss": 0.618,
+      "step": 9050
+    },
+    {
+      "epoch": 0.2622294183225602,
+      "learning_rate": 4.22324204876831e-05,
+      "loss": 0.6253,
+      "step": 9100
+    },
+    {
+      "epoch": 0.2636702393023545,
+      "learning_rate": 4.214978493655671e-05,
+      "loss": 0.6022,
+      "step": 9150
+    },
+    {
+      "epoch": 0.26511106028214876,
+      "learning_rate": 4.206679393097196e-05,
+      "loss": 0.6196,
+      "step": 9200
+    },
+    {
+      "epoch": 0.26655188126194307,
+      "learning_rate": 4.19834491910392e-05,
+      "loss": 0.6031,
+      "step": 9250
+    },
+    {
+      "epoch": 0.2679927022417373,
+      "learning_rate": 4.189975244420049e-05,
+      "loss": 0.6251,
+      "step": 9300
+    },
+    {
+      "epoch": 0.26943352322153163,
+      "learning_rate": 4.18157054251937e-05,
+      "loss": 0.6217,
+      "step": 9350
+    },
+    {
+      "epoch": 0.2708743442013259,
+      "learning_rate": 4.1731309876016656e-05,
+      "loss": 0.6172,
+      "step": 9400
+    },
+    {
+      "epoch": 0.2723151651811202,
+      "learning_rate": 4.1646567545890934e-05,
+      "loss": 0.6074,
+      "step": 9450
+    },
+    {
+      "epoch": 0.2737559861609145,
+      "learning_rate": 4.1561480191225694e-05,
+      "loss": 0.6087,
+      "step": 9500
+    },
+    {
+      "epoch": 0.2737559861609145,
+      "eval_loss": 0.6340173482894897,
+      "eval_runtime": 77.5417,
+      "eval_samples_per_second": 25.793,
+      "eval_steps_per_second": 12.896,
+      "step": 9500
+    },
+    {
+      "epoch": 0.27519680714070877,
+      "learning_rate": 4.147604957558121e-05,
+      "loss": 0.6242,
+      "step": 9550
+    },
+    {
+      "epoch": 0.2766376281205031,
+      "learning_rate": 4.139027746963235e-05,
+      "loss": 0.603,
+      "step": 9600
+    },
+    {
+      "epoch": 0.27807844910029733,
+      "learning_rate": 4.130416565113188e-05,
+      "loss": 0.6047,
+      "step": 9650
+    },
+    {
+      "epoch": 0.27951927008009164,
+      "learning_rate": 4.121771590487358e-05,
+      "loss": 0.6049,
+      "step": 9700
+    },
+    {
+      "epoch": 0.2809600910598859,
+      "learning_rate": 4.113093002265531e-05,
+      "loss": 0.5943,
+      "step": 9750
+    },
+    {
+      "epoch": 0.2824009120396802,
+      "learning_rate": 4.104380980324184e-05,
+      "loss": 0.6229,
+      "step": 9800
+    },
+    {
+      "epoch": 0.2838417330194745,
+      "learning_rate": 4.0956357052327536e-05,
+      "loss": 0.586,
+      "step": 9850
+    },
+    {
+      "epoch": 0.2852825539992688,
+      "learning_rate": 4.0868573582499004e-05,
+      "loss": 0.6058,
+      "step": 9900
+    },
+    {
+      "epoch": 0.2867233749790631,
+      "learning_rate": 4.078046121319746e-05,
+      "loss": 0.6081,
+      "step": 9950
+    },
+    {
+      "epoch": 0.28816419595885734,
+      "learning_rate": 4.0692021770681066e-05,
+      "loss": 0.5998,
+      "step": 10000
+    },
+    {
+      "epoch": 0.28816419595885734,
+      "eval_loss": 0.6245400309562683,
+      "eval_runtime": 81.0137,
+      "eval_samples_per_second": 24.687,
+      "eval_steps_per_second": 12.344,
+      "step": 10000
+    },
+    {
+      "epoch": 0.28960501693865165,
+      "learning_rate": 4.0603257087987025e-05,
+      "loss": 0.5999,
+      "step": 10050
+    },
+    {
+      "epoch": 0.2910458379184459,
+      "learning_rate": 4.0514169004893656e-05,
+      "loss": 0.5977,
+      "step": 10100
+    },
+    {
+      "epoch": 0.2924866588982402,
+      "learning_rate": 4.042475936788219e-05,
+      "loss": 0.5891,
+      "step": 10150
+    },
+    {
+      "epoch": 0.2939274798780345,
+      "learning_rate": 4.033503003009858e-05,
+      "loss": 0.5896,
+      "step": 10200
+    },
+    {
+      "epoch": 0.2953683008578288,
+      "learning_rate": 4.0244982851315006e-05,
+      "loss": 0.5929,
+      "step": 10250
+    },
+    {
+      "epoch": 0.2968091218376231,
+      "learning_rate": 4.015461969789138e-05,
+      "loss": 0.6017,
+      "step": 10300
+    },
+    {
+      "epoch": 0.29824994281741735,
+      "learning_rate": 4.006394244273666e-05,
+      "loss": 0.5839,
+      "step": 10350
+    },
+    {
+      "epoch": 0.29969076379721166,
+      "learning_rate": 3.9972952965270006e-05,
+      "loss": 0.6026,
+      "step": 10400
+    },
+    {
+      "epoch": 0.3011315847770059,
+      "learning_rate": 3.9881653151381884e-05,
+      "loss": 0.5985,
+      "step": 10450
+    },
+    {
+      "epoch": 0.3025724057568002,
+      "learning_rate": 3.9790044893394886e-05,
+      "loss": 0.5996,
+      "step": 10500
+    },
+    {
+      "epoch": 0.3025724057568002,
+      "eval_loss": 0.6157839894294739,
+      "eval_runtime": 82.2023,
+      "eval_samples_per_second": 24.33,
+      "eval_steps_per_second": 12.165,
+      "step": 10500
+    },
+    {
+      "epoch": 0.30401322673659453,
+      "learning_rate": 3.969813009002459e-05,
+      "loss": 0.595,
+      "step": 10550
+    },
+    {
+      "epoch": 0.3054540477163888,
+      "learning_rate": 3.960591064634017e-05,
+      "loss": 0.59,
+      "step": 10600
+    },
+    {
+      "epoch": 0.3068948686961831,
+      "learning_rate": 3.9513388473724935e-05,
+      "loss": 0.5895,
+      "step": 10650
+    },
+    {
+      "epoch": 0.30833568967597735,
+      "learning_rate": 3.942056548983666e-05,
+      "loss": 0.595,
+      "step": 10700
+    },
+    {
+      "epoch": 0.30977651065577166,
+      "learning_rate": 3.93274436185679e-05,
+      "loss": 0.5867,
+      "step": 10750
+    },
+    {
+      "epoch": 0.3112173316355659,
+      "learning_rate": 3.9234024790006085e-05,
+      "loss": 0.5851,
+      "step": 10800
+    },
+    {
+      "epoch": 0.31265815261536023,
+      "learning_rate": 3.9140310940393524e-05,
+      "loss": 0.5963,
+      "step": 10850
+    },
+    {
+      "epoch": 0.31409897359515454,
+      "learning_rate": 3.9046304012087265e-05,
+      "loss": 0.5827,
+      "step": 10900
+    },
+    {
+      "epoch": 0.3155397945749488,
+      "learning_rate": 3.895200595351883e-05,
+      "loss": 0.5885,
+      "step": 10950
+    },
+    {
+      "epoch": 0.3169806155547431,
+      "learning_rate": 3.885741871915386e-05,
+      "loss": 0.5837,
+      "step": 11000
+    },
+    {
+      "epoch": 0.3169806155547431,
+      "eval_loss": 0.616204023361206,
+      "eval_runtime": 80.49,
+      "eval_samples_per_second": 24.848,
+      "eval_steps_per_second": 12.424,
+      "step": 11000
+    },
+    {
+      "epoch": 0.31842143653453736,
+      "learning_rate": 3.87625442694516e-05,
+      "loss": 0.6003,
+      "step": 11050
+    },
+    {
+      "epoch": 0.31986225751433167,
+      "learning_rate": 3.866738457082423e-05,
+      "loss": 0.5922,
+      "step": 11100
+    },
+    {
+      "epoch": 0.3213030784941259,
+      "learning_rate": 3.857194159559613e-05,
+      "loss": 0.5919,
+      "step": 11150
+    },
+    {
+      "epoch": 0.32274389947392024,
+      "learning_rate": 3.8476217321963026e-05,
+      "loss": 0.5888,
+      "step": 11200
+    },
+    {
+      "epoch": 0.32418472045371455,
+      "learning_rate": 3.8380213733950957e-05,
+      "loss": 0.5644,
+      "step": 11250
+    },
+    {
+      "epoch": 0.3256255414335088,
+      "learning_rate": 3.828393282137515e-05,
+      "loss": 0.5837,
+      "step": 11300
+    },
+    {
+      "epoch": 0.3270663624133031,
+      "learning_rate": 3.81873765797988e-05,
+      "loss": 0.6009,
+      "step": 11350
+    },
+    {
+      "epoch": 0.32850718339309737,
+      "learning_rate": 3.8090547010491715e-05,
+      "loss": 0.5814,
+      "step": 11400
+    },
+    {
+      "epoch": 0.3299480043728917,
+      "learning_rate": 3.799344612038879e-05,
+      "loss": 0.5964,
+      "step": 11450
+    },
+    {
+      "epoch": 0.33138882535268593,
+      "learning_rate": 3.789607592204847e-05,
+      "loss": 0.5922,
+      "step": 11500
+    },
+    {
+      "epoch": 0.33138882535268593,
+      "eval_loss": 0.6145178079605103,
+      "eval_runtime": 80.6686,
+      "eval_samples_per_second": 24.793,
+      "eval_steps_per_second": 12.396,
+      "step": 11500
+    },
+    {
+      "epoch": 0.33282964633248024,
+      "learning_rate": 3.779843843361099e-05,
+      "loss": 0.6064,
+      "step": 11550
+    },
+    {
+      "epoch": 0.33427046731227456,
+      "learning_rate": 3.770053567875657e-05,
+      "loss": 0.5772,
+      "step": 11600
+    },
+    {
+      "epoch": 0.3357112882920688,
+      "learning_rate": 3.7602369686663474e-05,
+      "loss": 0.5699,
+      "step": 11650
+    },
+    {
+      "epoch": 0.3371521092718631,
+      "learning_rate": 3.7503942491965924e-05,
+      "loss": 0.5905,
+      "step": 11700
+    },
+    {
+      "epoch": 0.3385929302516574,
+      "learning_rate": 3.7405256134711976e-05,
+      "loss": 0.576,
+      "step": 11750
+    },
+    {
+      "epoch": 0.3400337512314517,
+      "learning_rate": 3.730631266032119e-05,
+      "loss": 0.5911,
+      "step": 11800
+    },
+    {
+      "epoch": 0.34147457221124594,
+      "learning_rate": 3.720711411954226e-05,
+      "loss": 0.576,
+      "step": 11850
+    },
+    {
+      "epoch": 0.34291539319104025,
+      "learning_rate": 3.710766256841051e-05,
+      "loss": 0.5819,
+      "step": 11900
+    },
+    {
+      "epoch": 0.34435621417083456,
+      "learning_rate": 3.7007960068205274e-05,
+      "loss": 0.5831,
+      "step": 11950
+    },
+    {
+      "epoch": 0.3457970351506288,
+      "learning_rate": 3.690800868540716e-05,
+      "loss": 0.5828,
+      "step": 12000
+    },
+    {
+      "epoch": 0.3457970351506288,
+      "eval_loss": 0.6077320575714111,
+      "eval_runtime": 79.5341,
+      "eval_samples_per_second": 25.146,
+      "eval_steps_per_second": 12.573,
+      "step": 12000
+    },
+    {
+      "epoch": 0.34723785613042313,
+      "learning_rate": 3.6807810491655264e-05,
+      "loss": 0.5529,
+      "step": 12050
+    },
+    {
+      "epoch": 0.3486786771102174,
+      "learning_rate": 3.670736756370417e-05,
+      "loss": 0.5784,
+      "step": 12100
+    },
+    {
+      "epoch": 0.3501194980900117,
+      "learning_rate": 3.660668198338095e-05,
+      "loss": 0.5776,
+      "step": 12150
+    },
+    {
+      "epoch": 0.35156031906980595,
+      "learning_rate": 3.650575583754201e-05,
+      "loss": 0.5754,
+      "step": 12200
+    },
+    {
+      "epoch": 0.35300114004960026,
+      "learning_rate": 3.640459121802981e-05,
+      "loss": 0.5754,
+      "step": 12250
+    },
+    {
+      "epoch": 0.35444196102939457,
+      "learning_rate": 3.6303190221629546e-05,
+      "loss": 0.5667,
+      "step": 12300
+    },
+    {
+      "epoch": 0.3558827820091888,
+      "learning_rate": 3.620155495002566e-05,
+      "loss": 0.5636,
+      "step": 12350
+    },
+    {
+      "epoch": 0.35732360298898314,
+      "learning_rate": 3.609968750975829e-05,
+      "loss": 0.5682,
+      "step": 12400
+    },
+    {
+      "epoch": 0.3587644239687774,
+      "learning_rate": 3.599759001217964e-05,
+      "loss": 0.5697,
+      "step": 12450
+    },
+    {
+      "epoch": 0.3602052449485717,
+      "learning_rate": 3.589526457341014e-05,
+      "loss": 0.5721,
+      "step": 12500
+    },
+    {
+      "epoch": 0.3602052449485717,
+      "eval_loss": 0.6059696674346924,
+      "eval_runtime": 79.5408,
+      "eval_samples_per_second": 25.144,
+      "eval_steps_per_second": 12.572,
+      "step": 12500
+    },
+    {
+      "epoch": 0.36164606592836596,
+      "learning_rate": 3.5792713314294654e-05,
+      "loss": 0.5777,
+      "step": 12550
+    },
+    {
+      "epoch": 0.36308688690816027,
+      "learning_rate": 3.568993836035854e-05,
+      "loss": 0.564,
+      "step": 12600
+    },
+    {
+      "epoch": 0.3645277078879546,
+      "learning_rate": 3.558694184176351e-05,
+      "loss": 0.5855,
+      "step": 12650
+    },
+    {
+      "epoch": 0.36596852886774883,
+      "learning_rate": 3.548372589326356e-05,
+      "loss": 0.5532,
+      "step": 12700
+    },
+    {
+      "epoch": 0.36740934984754314,
+      "learning_rate": 3.5380292654160674e-05,
+      "loss": 0.5683,
+      "step": 12750
+    },
+    {
+      "epoch": 0.3688501708273374,
+      "learning_rate": 3.5276644268260505e-05,
+      "loss": 0.5611,
+      "step": 12800
+    },
+    {
+      "epoch": 0.3702909918071317,
+      "learning_rate": 3.517278288382795e-05,
+      "loss": 0.5507,
+      "step": 12850
+    },
+    {
+      "epoch": 0.37173181278692596,
+      "learning_rate": 3.5068710653542615e-05,
+      "loss": 0.5502,
+      "step": 12900
+    },
+    {
+      "epoch": 0.3731726337667203,
+      "learning_rate": 3.496442973445417e-05,
+      "loss": 0.5758,
+      "step": 12950
+    },
+    {
+      "epoch": 0.3746134547465146,
+      "learning_rate": 3.4859942287937706e-05,
+      "loss": 0.5575,
+      "step": 13000
+    },
+    {
+      "epoch": 0.3746134547465146,
+      "eval_loss": 0.6036191582679749,
+      "eval_runtime": 80.0548,
+      "eval_samples_per_second": 24.983,
+      "eval_steps_per_second": 12.491,
+      "step": 13000
+    },
+    {
+      "epoch": 0.37605427572630884,
+      "learning_rate": 3.475525047964887e-05,
+      "loss": 0.5745,
+      "step": 13050
+    },
+    {
+      "epoch": 0.37749509670610315,
+      "learning_rate": 3.4650356479479e-05,
+      "loss": 0.5601,
+      "step": 13100
+    },
+    {
+      "epoch": 0.3789359176858974,
+      "learning_rate": 3.454526246151019e-05,
+      "loss": 0.5612,
+      "step": 13150
+    },
+    {
+      "epoch": 0.3803767386656917,
+      "learning_rate": 3.443997060397015e-05,
+      "loss": 0.5699,
+      "step": 13200
+    },
+    {
+      "epoch": 0.38181755964548597,
+      "learning_rate": 3.4334483089187163e-05,
+      "loss": 0.5648,
+      "step": 13250
+    },
+    {
+      "epoch": 0.3832583806252803,
+      "learning_rate": 3.422880210354475e-05,
+      "loss": 0.5664,
+      "step": 13300
+    },
+    {
+      "epoch": 0.3846992016050746,
+      "learning_rate": 3.412292983743642e-05,
+      "loss": 0.558,
+      "step": 13350
+    },
+    {
+      "epoch": 0.38614002258486885,
+      "learning_rate": 3.401686848522023e-05,
+      "loss": 0.5625,
+      "step": 13400
+    },
+    {
+      "epoch": 0.38758084356466316,
+      "learning_rate": 3.391062024517336e-05,
+      "loss": 0.5511,
+      "step": 13450
+    },
+    {
+      "epoch": 0.3890216645444574,
+      "learning_rate": 3.3804187319446484e-05,
+      "loss": 0.5649,
+      "step": 13500
+    },
+    {
+      "epoch": 0.3890216645444574,
+      "eval_loss": 0.6038739085197449,
+      "eval_runtime": 79.3792,
+      "eval_samples_per_second": 25.196,
+      "eval_steps_per_second": 12.598,
+      "step": 13500
+    },
+    {
+      "epoch": 0.3904624855242517,
+      "learning_rate": 3.3697571914018164e-05,
+      "loss": 0.5447,
+      "step": 13550
+    },
+    {
+      "epoch": 0.391903306504046,
+      "learning_rate": 3.359077623864913e-05,
+      "loss": 0.5457,
+      "step": 13600
+    },
+    {
+      "epoch": 0.3933441274838403,
+      "learning_rate": 3.348380250683647e-05,
+      "loss": 0.5455,
+      "step": 13650
+    },
+    {
+      "epoch": 0.3947849484636346,
+      "learning_rate": 3.337665293576773e-05,
+      "loss": 0.5402,
+      "step": 13700
+    },
+    {
+      "epoch": 0.39622576944342885,
+      "learning_rate": 3.326932974627501e-05,
+      "loss": 0.554,
+      "step": 13750
+    },
+    {
+      "epoch": 0.39766659042322317,
+      "learning_rate": 3.316183516278891e-05,
+      "loss": 0.5501,
+      "step": 13800
+    },
+    {
+      "epoch": 0.3991074114030174,
+      "learning_rate": 3.30541714132924e-05,
+      "loss": 0.5537,
+      "step": 13850
+    },
+    {
+      "epoch": 0.40054823238281173,
+      "learning_rate": 3.294634072927468e-05,
+      "loss": 0.5395,
+      "step": 13900
+    },
+    {
+      "epoch": 0.401989053362606,
+      "learning_rate": 3.28383453456849e-05,
+      "loss": 0.5496,
+      "step": 13950
+    },
+    {
+      "epoch": 0.4034298743424003,
+      "learning_rate": 3.2730187500885864e-05,
+      "loss": 0.557,
+      "step": 14000
+    },
+    {
+      "epoch": 0.4034298743424003,
+      "eval_loss": 0.6013383865356445,
+      "eval_runtime": 80.0388,
+      "eval_samples_per_second": 24.988,
+      "eval_steps_per_second": 12.494,
+      "step": 14000
+    },
+    {
+      "epoch": 0.4048706953221946,
+      "learning_rate": 3.262186943660761e-05,
+      "loss": 0.5464,
+      "step": 14050
+    },
+    {
+      "epoch": 0.40631151630198886,
+      "learning_rate": 3.251339339790096e-05,
+      "loss": 0.5635,
+      "step": 14100
+    },
+    {
+      "epoch": 0.4077523372817832,
+      "learning_rate": 3.2404761633091004e-05,
+      "loss": 0.5543,
+      "step": 14150
+    },
+    {
+      "epoch": 0.4091931582615774,
+      "learning_rate": 3.229597639373047e-05,
+      "loss": 0.5635,
+      "step": 14200
+    },
+    {
+      "epoch": 0.41063397924137174,
+      "learning_rate": 3.2187039934553076e-05,
+      "loss": 0.5547,
+      "step": 14250
+    },
+    {
+      "epoch": 0.412074800221166,
+      "learning_rate": 3.207795451342679e-05,
+      "loss": 0.5631,
+      "step": 14300
+    },
+    {
+      "epoch": 0.4135156212009603,
+      "learning_rate": 3.196872239130704e-05,
+      "loss": 0.5431,
+      "step": 14350
+    },
+    {
+      "epoch": 0.4149564421807546,
+      "learning_rate": 3.1859345832189845e-05,
+      "loss": 0.5519,
+      "step": 14400
+    },
+    {
+      "epoch": 0.41639726316054887,
+      "learning_rate": 3.174982710306489e-05,
+      "loss": 0.5447,
+      "step": 14450
+    },
+    {
+      "epoch": 0.4178380841403432,
+      "learning_rate": 3.164016847386854e-05,
+      "loss": 0.5581,
+      "step": 14500
+    },
+    {
+      "epoch": 0.4178380841403432,
+      "eval_loss": 0.5996755957603455,
+      "eval_runtime": 80.3224,
+      "eval_samples_per_second": 24.9,
+      "eval_steps_per_second": 12.45,
+      "step": 14500
+    },
+    {
+      "epoch": 0.41927890512013744,
+      "learning_rate": 3.1530372217436795e-05,
+      "loss": 0.546,
+      "step": 14550
+    },
+    {
+      "epoch": 0.42071972609993175,
+      "learning_rate": 3.142044060945819e-05,
+      "loss": 0.5568,
+      "step": 14600
+    },
+    {
+      "epoch": 0.422160547079726,
+      "learning_rate": 3.1310375928426616e-05,
+      "loss": 0.5446,
+      "step": 14650
+    },
+    {
+      "epoch": 0.4236013680595203,
+      "learning_rate": 3.120018045559409e-05,
+      "loss": 0.5335,
+      "step": 14700
+    },
+    {
+      "epoch": 0.4250421890393146,
+      "learning_rate": 3.10898564749235e-05,
+      "loss": 0.5446,
+      "step": 14750
+    },
+    {
+      "epoch": 0.4264830100191089,
+      "learning_rate": 3.097940627304122e-05,
+      "loss": 0.532,
+      "step": 14800
+    },
+    {
+      "epoch": 0.4279238309989032,
+      "learning_rate": 3.0868832139189766e-05,
+      "loss": 0.5545,
+      "step": 14850
+    },
+    {
+      "epoch": 0.42936465197869744,
+      "learning_rate": 3.075813636518031e-05,
+      "loss": 0.5482,
+      "step": 14900
+    },
+    {
+      "epoch": 0.43080547295849175,
+      "learning_rate": 3.06473212453452e-05,
+      "loss": 0.5297,
+      "step": 14950
+    },
+    {
+      "epoch": 0.432246293938286,
+      "learning_rate": 3.053638907649042e-05,
+      "loss": 0.5371,
+      "step": 15000
+    },
+    {
+      "epoch": 0.432246293938286,
+      "eval_loss": 0.5982165336608887,
+      "eval_runtime": 79.6872,
+      "eval_samples_per_second": 25.098,
+      "eval_steps_per_second": 12.549,
+      "step": 15000
+    },
+    {
+      "epoch": 0.4336871149180803,
+      "learning_rate": 3.0425342157847957e-05,
+      "loss": 0.5467,
+      "step": 15050
+    },
+    {
+      "epoch": 0.43512793589787463,
+      "learning_rate": 3.0314182791028128e-05,
+      "loss": 0.5436,
+      "step": 15100
+    },
+    {
+      "epoch": 0.4365687568776689,
+      "learning_rate": 3.0202913279971935e-05,
+      "loss": 0.5317,
+      "step": 15150
+    },
+    {
+      "epoch": 0.4380095778574632,
+      "learning_rate": 3.009153593090327e-05,
+      "loss": 0.5357,
+      "step": 15200
+    },
+    {
+      "epoch": 0.43945039883725745,
+      "learning_rate": 2.9980053052281126e-05,
+      "loss": 0.5528,
+      "step": 15250
+    },
+    {
+      "epoch": 0.44089121981705176,
+      "learning_rate": 2.9868466954751758e-05,
+      "loss": 0.541,
+      "step": 15300
+    },
+    {
+      "epoch": 0.442332040796846,
+      "learning_rate": 2.9756779951100766e-05,
+      "loss": 0.5447,
+      "step": 15350
+    },
+    {
+      "epoch": 0.4437728617766403,
+      "learning_rate": 2.9644994356205207e-05,
+      "loss": 0.5467,
+      "step": 15400
+    },
+    {
+      "epoch": 0.44521368275643464,
+      "learning_rate": 2.953311248698556e-05,
+      "loss": 0.5374,
+      "step": 15450
+    },
+    {
+      "epoch": 0.4466545037362289,
+      "learning_rate": 2.942113666235774e-05,
+      "loss": 0.5328,
+      "step": 15500
+    },
+    {
+      "epoch": 0.4466545037362289,
+      "eval_loss": 0.5987485647201538,
+      "eval_runtime": 79.3166,
+      "eval_samples_per_second": 25.215,
+      "eval_steps_per_second": 12.608,
+      "step": 15500
+    },
+    {
+      "epoch": 0.4480953247160232,
+      "learning_rate": 2.9309069203185026e-05,
+      "loss": 0.5398,
+      "step": 15550
+    },
+    {
+      "epoch": 0.44953614569581746,
+      "learning_rate": 2.919691243222996e-05,
+      "loss": 0.5469,
+      "step": 15600
+    },
+    {
+      "epoch": 0.45097696667561177,
+      "learning_rate": 2.9084668674106196e-05,
+      "loss": 0.5552,
+      "step": 15650
+    },
+    {
+      "epoch": 0.452417787655406,
+      "learning_rate": 2.8972340255230322e-05,
+      "loss": 0.5379,
+      "step": 15700
+    },
+    {
+      "epoch": 0.45385860863520033,
+      "learning_rate": 2.8859929503773652e-05,
+      "loss": 0.5358,
+      "step": 15750
+    },
+    {
+      "epoch": 0.45529942961499464,
+      "learning_rate": 2.874743874961397e-05,
+      "loss": 0.5391,
+      "step": 15800
+    },
+    {
+      "epoch": 0.4567402505947889,
+      "learning_rate": 2.863487032428722e-05,
+      "loss": 0.5369,
+      "step": 15850
+    },
+    {
+      "epoch": 0.4581810715745832,
+      "learning_rate": 2.85222265609392e-05,
+      "loss": 0.5273,
+      "step": 15900
+    },
+    {
+      "epoch": 0.45962189255437746,
+      "learning_rate": 2.8409509794277195e-05,
+      "loss": 0.5384,
+      "step": 15950
+    },
+    {
+      "epoch": 0.4610627135341718,
+      "learning_rate": 2.8296722360521593e-05,
+      "loss": 0.5508,
+      "step": 16000
+    },
+    {
+      "epoch": 0.4610627135341718,
+      "eval_loss": 0.5947248339653015,
+      "eval_runtime": 79.6937,
+      "eval_samples_per_second": 25.096,
+      "eval_steps_per_second": 12.548,
+      "step": 16000
+    },
+    {
+      "epoch": 0.46250353451396603,
+      "learning_rate": 2.8183866597357467e-05,
+      "loss": 0.5199,
+      "step": 16050
+    },
+    {
+      "epoch": 0.46394435549376034,
+      "learning_rate": 2.8070944843886104e-05,
+      "loss": 0.5433,
+      "step": 16100
+    },
+    {
+      "epoch": 0.46538517647355465,
+      "learning_rate": 2.7957959440576564e-05,
+      "loss": 0.5167,
+      "step": 16150
+    },
+    {
+      "epoch": 0.4668259974533489,
+      "learning_rate": 2.7844912729217116e-05,
+      "loss": 0.5487,
+      "step": 16200
+    },
+    {
+      "epoch": 0.4682668184331432,
+      "learning_rate": 2.773180705286673e-05,
+      "loss": 0.5448,
+      "step": 16250
+    },
+    {
+      "epoch": 0.46970763941293747,
+      "learning_rate": 2.7618644755806527e-05,
+      "loss": 0.5407,
+      "step": 16300
+    },
+    {
+      "epoch": 0.4711484603927318,
+      "learning_rate": 2.7505428183491168e-05,
+      "loss": 0.5253,
+      "step": 16350
+    },
+    {
+      "epoch": 0.47258928137252604,
+      "learning_rate": 2.7392159682500246e-05,
+      "loss": 0.5276,
+      "step": 16400
+    },
+    {
+      "epoch": 0.47403010235232035,
+      "learning_rate": 2.7278841600489652e-05,
+      "loss": 0.5529,
+      "step": 16450
+    },
+    {
+      "epoch": 0.47547092333211466,
+      "learning_rate": 2.7165476286142922e-05,
+      "loss": 0.5293,
+      "step": 16500
+    },
+    {
+      "epoch": 0.47547092333211466,
+      "eval_loss": 0.5968120098114014,
+      "eval_runtime": 79.5645,
+      "eval_samples_per_second": 25.137,
+      "eval_steps_per_second": 12.568,
+      "step": 16500
+    },
+    {
+      "epoch": 0.4769117443119089,
+      "learning_rate": 2.7052066089122545e-05,
+      "loss": 0.5275,
+      "step": 16550
+    },
+    {
+      "epoch": 0.4783525652917032,
+      "learning_rate": 2.6938613360021277e-05,
+      "loss": 0.5272,
+      "step": 16600
+    },
+    {
+      "epoch": 0.4797933862714975,
+      "learning_rate": 2.68251204503134e-05,
+      "loss": 0.5164,
+      "step": 16650
+    },
+    {
+      "epoch": 0.4812342072512918,
+      "learning_rate": 2.6711589712306018e-05,
+      "loss": 0.5099,
+      "step": 16700
+    },
+    {
+      "epoch": 0.48267502823108605,
+      "learning_rate": 2.6598023499090262e-05,
+      "loss": 0.5379,
+      "step": 16750
+    },
+    {
+      "epoch": 0.48411584921088036,
+      "learning_rate": 2.6484424164492554e-05,
+      "loss": 0.5284,
+      "step": 16800
+    },
+    {
+      "epoch": 0.48555667019067467,
+      "learning_rate": 2.6370794063025796e-05,
+      "loss": 0.5178,
+      "step": 16850
+    },
+    {
+      "epoch": 0.4869974911704689,
+      "learning_rate": 2.6257135549840574e-05,
+      "loss": 0.5271,
+      "step": 16900
+    },
+    {
+      "epoch": 0.48843831215026323,
+      "learning_rate": 2.614345098067637e-05,
+      "loss": 0.534,
+      "step": 16950
+    },
+    {
+      "epoch": 0.4898791331300575,
+      "learning_rate": 2.60297427118127e-05,
+      "loss": 0.5394,
+      "step": 17000
+    },
+    {
+      "epoch": 0.4898791331300575,
+      "eval_loss": 0.5935970544815063,
+      "eval_runtime": 79.5625,
+      "eval_samples_per_second": 25.137,
+      "eval_steps_per_second": 12.569,
+      "step": 17000
+    },
+    {
+      "epoch": 0.4913199541098518,
+      "learning_rate": 2.5916013100020276e-05,
+      "loss": 0.5348,
+      "step": 17050
+    },
+    {
+      "epoch": 0.4927607750896461,
+      "learning_rate": 2.5802264502512203e-05,
+      "loss": 0.5168,
+      "step": 17100
+    },
+    {
+      "epoch": 0.49420159606944036,
+      "learning_rate": 2.5688499276895067e-05,
+      "loss": 0.5137,
+      "step": 17150
+    },
+    {
+      "epoch": 0.4956424170492347,
+      "learning_rate": 2.5574719781120123e-05,
+      "loss": 0.522,
+      "step": 17200
+    },
+    {
+      "epoch": 0.49708323802902893,
+      "learning_rate": 2.546092837343437e-05,
+      "loss": 0.5327,
+      "step": 17250
+    },
+    {
+      "epoch": 0.49852405900882324,
+      "learning_rate": 2.5347127412331712e-05,
+      "loss": 0.5219,
+      "step": 17300
+    },
+    {
+      "epoch": 0.4999648799886175,
+      "learning_rate": 2.5233319256504056e-05,
+      "loss": 0.5225,
+      "step": 17350
+    },
+    {
+      "epoch": 0.5014057009684117,
+      "learning_rate": 2.5119506264792442e-05,
+      "loss": 0.5031,
+      "step": 17400
+    },
+    {
+      "epoch": 0.5028465219482061,
+      "learning_rate": 2.5005690796138125e-05,
+      "loss": 0.5083,
+      "step": 17450
+    },
+    {
+      "epoch": 0.5042873429280004,
+      "learning_rate": 2.48918752095337e-05,
+      "loss": 0.5071,
+      "step": 17500
+    },
+    {
+      "epoch": 0.5042873429280004,
+      "eval_loss": 0.5912173986434937,
+      "eval_runtime": 80.629,
+      "eval_samples_per_second": 24.805,
+      "eval_steps_per_second": 12.402,
+      "step": 17500
+    },
+    {
+      "epoch": 0.5057281639077946,
+      "learning_rate": 2.4778061863974236e-05,
+      "loss": 0.5202,
+      "step": 17550
+    },
+    {
+      "epoch": 0.507168984887589,
+      "learning_rate": 2.4664253118408303e-05,
+      "loss": 0.5196,
+      "step": 17600
+    },
+    {
+      "epoch": 0.5086098058673832,
+      "learning_rate": 2.4550451331689165e-05,
+      "loss": 0.5207,
+      "step": 17650
+    },
+    {
+      "epoch": 0.5100506268471775,
+      "learning_rate": 2.4436658862525845e-05,
+      "loss": 0.5239,
+      "step": 17700
+    },
+    {
+      "epoch": 0.5114914478269718,
+      "learning_rate": 2.4322878069434245e-05,
+      "loss": 0.5311,
+      "step": 17750
+    },
+    {
+      "epoch": 0.5129322688067661,
+      "learning_rate": 2.4209111310688265e-05,
+      "loss": 0.511,
+      "step": 17800
+    },
+    {
+      "epoch": 0.5143730897865604,
+      "learning_rate": 2.4095360944270917e-05,
+      "loss": 0.5054,
+      "step": 17850
+    },
+    {
+      "epoch": 0.5158139107663546,
+      "learning_rate": 2.398162932782547e-05,
+      "loss": 0.518,
+      "step": 17900
+    },
+    {
+      "epoch": 0.517254731746149,
+      "learning_rate": 2.3867918818606554e-05,
+      "loss": 0.5269,
+      "step": 17950
+    },
+    {
+      "epoch": 0.5186955527259433,
+      "learning_rate": 2.3754231773431336e-05,
+      "loss": 0.5332,
+      "step": 18000
+    },
+    {
+      "epoch": 0.5186955527259433,
+      "eval_loss": 0.5898262858390808,
+      "eval_runtime": 80.1046,
+      "eval_samples_per_second": 24.967,
+      "eval_steps_per_second": 12.484,
+      "step": 18000
+    },
+    {
+      "epoch": 0.5201363737057375,
+      "learning_rate": 2.3640570548630653e-05,
+      "loss": 0.5313,
+      "step": 18050
+    },
+    {
+      "epoch": 0.5215771946855318,
+      "learning_rate": 2.3526937500000172e-05,
+      "loss": 0.5333,
+      "step": 18100
+    },
+    {
+      "epoch": 0.5230180156653261,
+      "learning_rate": 2.341333498275157e-05,
+      "loss": 0.5186,
+      "step": 18150
+    },
+    {
+      "epoch": 0.5244588366451204,
+      "learning_rate": 2.3299765351463725e-05,
+      "loss": 0.5167,
+      "step": 18200
+    },
+    {
+      "epoch": 0.5258996576249146,
+      "learning_rate": 2.318623096003389e-05,
+      "loss": 0.5161,
+      "step": 18250
+    },
+    {
+      "epoch": 0.527340478604709,
+      "learning_rate": 2.3072734161628916e-05,
+      "loss": 0.5015,
+      "step": 18300
+    },
+    {
+      "epoch": 0.5287812995845033,
+      "learning_rate": 2.2959277308636518e-05,
+      "loss": 0.4986,
+      "step": 18350
+    },
+    {
+      "epoch": 0.5302221205642975,
+      "learning_rate": 2.2845862752616455e-05,
+      "loss": 0.502,
+      "step": 18400
+    },
+    {
+      "epoch": 0.5316629415440918,
+      "learning_rate": 2.2732492844251817e-05,
+      "loss": 0.5102,
+      "step": 18450
+    },
+    {
+      "epoch": 0.5331037625238861,
+      "learning_rate": 2.261916993330032e-05,
+      "loss": 0.5026,
+      "step": 18500
+    },
+    {
+      "epoch": 0.5331037625238861,
+      "eval_loss": 0.5886390805244446,
+      "eval_runtime": 80.592,
+      "eval_samples_per_second": 24.816,
+      "eval_steps_per_second": 12.408,
+      "step": 18500
+    },
+    {
+      "epoch": 0.5345445835036804,
+      "learning_rate": 2.2505896368545577e-05,
+      "loss": 0.5174,
+      "step": 18550
+    },
+    {
+      "epoch": 0.5359854044834746,
+      "learning_rate": 2.2392674497748438e-05,
+      "loss": 0.5185,
+      "step": 18600
+    },
+    {
+      "epoch": 0.537426225463269,
+      "learning_rate": 2.2279506667598317e-05,
+      "loss": 0.5027,
+      "step": 18650
+    },
+    {
+      "epoch": 0.5388670464430633,
+      "learning_rate": 2.2166395223664553e-05,
+      "loss": 0.5035,
+      "step": 18700
+    },
+    {
+      "epoch": 0.5403078674228575,
+      "learning_rate": 2.2053342510347816e-05,
+      "loss": 0.499,
+      "step": 18750
+    },
+    {
+      "epoch": 0.5417486884026518,
+      "learning_rate": 2.194035087083146e-05,
+      "loss": 0.5136,
+      "step": 18800
+    },
+    {
+      "epoch": 0.5431895093824461,
+      "learning_rate": 2.1827422647033033e-05,
+      "loss": 0.5095,
+      "step": 18850
+    },
+    {
+      "epoch": 0.5446303303622404,
+      "learning_rate": 2.1714560179555675e-05,
+      "loss": 0.5086,
+      "step": 18900
+    },
+    {
+      "epoch": 0.5460711513420347,
+      "learning_rate": 2.160176580763965e-05,
+      "loss": 0.5063,
+      "step": 18950
+    },
+    {
+      "epoch": 0.547511972321829,
+      "learning_rate": 2.1489041869113818e-05,
+      "loss": 0.509,
+      "step": 19000
+    },
+    {
+      "epoch": 0.547511972321829,
+      "eval_loss": 0.5874903798103333,
+      "eval_runtime": 81.4283,
+      "eval_samples_per_second": 24.561,
+      "eval_steps_per_second": 12.281,
+      "step": 19000
+    },
+    {
+      "epoch": 0.5489527933016233,
+      "learning_rate": 2.137639070034722e-05,
+      "loss": 0.5107,
+      "step": 19050
+    },
+    {
+      "epoch": 0.5503936142814175,
+      "learning_rate": 2.1263814636200632e-05,
+      "loss": 0.521,
+      "step": 19100
+    },
+    {
+      "epoch": 0.5518344352612118,
+      "learning_rate": 2.1151316009978183e-05,
+      "loss": 0.493,
+      "step": 19150
+    },
+    {
+      "epoch": 0.5532752562410062,
+      "learning_rate": 2.103889715337898e-05,
+      "loss": 0.4991,
+      "step": 19200
+    },
+    {
+      "epoch": 0.5547160772208004,
+      "learning_rate": 2.092656039644878e-05,
+      "loss": 0.5057,
+      "step": 19250
+    },
+    {
+      "epoch": 0.5561568982005947,
+      "learning_rate": 2.081430806753173e-05,
+      "loss": 0.5132,
+      "step": 19300
+    },
+    {
+      "epoch": 0.557597719180389,
+      "learning_rate": 2.0702142493222055e-05,
+      "loss": 0.5083,
+      "step": 19350
+    },
+    {
+      "epoch": 0.5590385401601833,
+      "learning_rate": 2.059006599831587e-05,
+      "loss": 0.4962,
+      "step": 19400
+    },
+    {
+      "epoch": 0.5604793611399775,
+      "learning_rate": 2.0478080905762998e-05,
+      "loss": 0.513,
+      "step": 19450
+    },
+    {
+      "epoch": 0.5619201821197718,
+      "learning_rate": 2.0366189536618806e-05,
+      "loss": 0.5125,
+      "step": 19500
+    },
+    {
+      "epoch": 0.5619201821197718,
+      "eval_loss": 0.5878966450691223,
+      "eval_runtime": 79.8864,
+      "eval_samples_per_second": 25.036,
+      "eval_steps_per_second": 12.518,
+      "step": 19500
+    },
+    {
+      "epoch": 0.5633610030995662,
+      "learning_rate": 2.0254394209996114e-05,
+      "loss": 0.4956,
+      "step": 19550
+    },
+    {
+      "epoch": 0.5648018240793604,
+      "learning_rate": 2.0142697243017118e-05,
+      "loss": 0.4981,
+      "step": 19600
+    },
+    {
+      "epoch": 0.5662426450591547,
+      "learning_rate": 2.003110095076535e-05,
+      "loss": 0.5043,
+      "step": 19650
+    },
+    {
+      "epoch": 0.567683466038949,
+      "learning_rate": 1.991960764623773e-05,
+      "loss": 0.4866,
+      "step": 19700
+    },
+    {
+      "epoch": 0.5691242870187433,
+      "learning_rate": 1.9808219640296596e-05,
+      "loss": 0.4904,
+      "step": 19750
+    },
+    {
+      "epoch": 0.5705651079985375,
+      "learning_rate": 1.9696939241621824e-05,
+      "loss": 0.4968,
+      "step": 19800
+    },
+    {
+      "epoch": 0.5720059289783318,
+      "learning_rate": 1.9585768756662972e-05,
+      "loss": 0.4955,
+      "step": 19850
+    },
+    {
+      "epoch": 0.5734467499581262,
+      "learning_rate": 1.9474710489591463e-05,
+      "loss": 0.4988,
+      "step": 19900
+    },
+    {
+      "epoch": 0.5748875709379204,
+      "learning_rate": 1.936376674225286e-05,
+      "loss": 0.5132,
+      "step": 19950
+    },
+    {
+      "epoch": 0.5763283919177147,
+      "learning_rate": 1.925293981411912e-05,
+      "loss": 0.4903,
+      "step": 20000
+    },
+    {
+      "epoch": 0.5763283919177147,
+      "eval_loss": 0.589536726474762,
+      "eval_runtime": 79.6161,
+      "eval_samples_per_second": 25.121,
+      "eval_steps_per_second": 12.56,
+      "step": 20000
+    },
+    {
+      "epoch": 0.577769212897509,
+      "learning_rate": 1.914223200224096e-05,
+      "loss": 0.504,
+      "step": 20050
+    },
+    {
+      "epoch": 0.5792100338773033,
+      "learning_rate": 1.9031645601200227e-05,
+      "loss": 0.502,
+      "step": 20100
+    },
+    {
+      "epoch": 0.5806508548570976,
+      "learning_rate": 1.8921182903062363e-05,
+      "loss": 0.5119,
+      "step": 20150
+    },
+    {
+      "epoch": 0.5820916758368918,
+      "learning_rate": 1.8810846197328873e-05,
+      "loss": 0.5023,
+      "step": 20200
+    },
+    {
+      "epoch": 0.5835324968166862,
+      "learning_rate": 1.8700637770889896e-05,
+      "loss": 0.5141,
+      "step": 20250
+    },
+    {
+      "epoch": 0.5849733177964804,
+      "learning_rate": 1.8590559907976784e-05,
+      "loss": 0.4965,
+      "step": 20300
+    },
+    {
+      "epoch": 0.5864141387762747,
+      "learning_rate": 1.8480614890114777e-05,
+      "loss": 0.5073,
+      "step": 20350
+    },
+    {
+      "epoch": 0.587854959756069,
+      "learning_rate": 1.8370804996075707e-05,
+      "loss": 0.5037,
+      "step": 20400
+    },
+    {
+      "epoch": 0.5892957807358633,
+      "learning_rate": 1.8261132501830763e-05,
+      "loss": 0.5152,
+      "step": 20450
+    },
+    {
+      "epoch": 0.5907366017156576,
+      "learning_rate": 1.815159968050332e-05,
+      "loss": 0.5076,
+      "step": 20500
+    },
+    {
+      "epoch": 0.5907366017156576,
+      "eval_loss": 0.5854496955871582,
+      "eval_runtime": 81.6796,
+      "eval_samples_per_second": 24.486,
+      "eval_steps_per_second": 12.243,
+      "step": 20500
+    },
+    {
+      "epoch": 0.5921774226954518,
+      "learning_rate": 1.8042208802321822e-05,
+      "loss": 0.4845,
+      "step": 20550
+    },
+    {
+      "epoch": 0.5936182436752462,
+      "learning_rate": 1.7932962134572744e-05,
+      "loss": 0.5,
+      "step": 20600
+    },
+    {
+      "epoch": 0.5950590646550404,
+      "learning_rate": 1.782386194155358e-05,
+      "loss": 0.5003,
+      "step": 20650
+    },
+    {
+      "epoch": 0.5964998856348347,
+      "learning_rate": 1.7714910484525925e-05,
+      "loss": 0.4996,
+      "step": 20700
+    },
+    {
+      "epoch": 0.5979407066146291,
+      "learning_rate": 1.760611002166861e-05,
+      "loss": 0.5056,
+      "step": 20750
+    },
+    {
+      "epoch": 0.5993815275944233,
+      "learning_rate": 1.749746280803087e-05,
+      "loss": 0.4789,
+      "step": 20800
+    },
+    {
+      "epoch": 0.6008223485742176,
+      "learning_rate": 1.7388971095485644e-05,
+      "loss": 0.5089,
+      "step": 20850
+    },
+    {
+      "epoch": 0.6022631695540118,
+      "learning_rate": 1.728063713268287e-05,
+      "loss": 0.49,
+      "step": 20900
+    },
+    {
+      "epoch": 0.6037039905338062,
+      "learning_rate": 1.7172463165002893e-05,
+      "loss": 0.5004,
+      "step": 20950
+    },
+    {
+      "epoch": 0.6051448115136004,
+      "learning_rate": 1.7064451434509943e-05,
+      "loss": 0.4844,
+      "step": 21000
+    },
+    {
+      "epoch": 0.6051448115136004,
+      "eval_loss": 0.5877842903137207,
+      "eval_runtime": 81.851,
+      "eval_samples_per_second": 24.435,
+      "eval_steps_per_second": 12.217,
+      "step": 21000
+    },
+    {
+      "epoch": 0.6065856324933947,
+      "learning_rate": 1.6956604179905618e-05,
+      "loss": 0.5006,
+      "step": 21050
+    },
+    {
+      "epoch": 0.6080264534731891,
+      "learning_rate": 1.684892363648252e-05,
+      "loss": 0.503,
+      "step": 21100
+    },
+    {
+      "epoch": 0.6094672744529833,
+      "learning_rate": 1.6741412036077927e-05,
+      "loss": 0.5008,
+      "step": 21150
+    },
+    {
+      "epoch": 0.6109080954327776,
+      "learning_rate": 1.6634071607027525e-05,
+      "loss": 0.4878,
+      "step": 21200
+    },
+    {
+      "epoch": 0.6123489164125718,
+      "learning_rate": 1.6526904574119213e-05,
+      "loss": 0.4994,
+      "step": 21250
+    },
+    {
+      "epoch": 0.6137897373923662,
+      "learning_rate": 1.6419913158547e-05,
+      "loss": 0.4964,
+      "step": 21300
+    },
+    {
+      "epoch": 0.6152305583721605,
+      "learning_rate": 1.631309957786498e-05,
+      "loss": 0.4804,
+      "step": 21350
+    },
+    {
+      "epoch": 0.6166713793519547,
+      "learning_rate": 1.620646604594135e-05,
+      "loss": 0.509,
+      "step": 21400
+    },
+    {
+      "epoch": 0.6181122003317491,
+      "learning_rate": 1.610001477291253e-05,
+      "loss": 0.4863,
+      "step": 21450
+    },
+    {
+      "epoch": 0.6195530213115433,
+      "learning_rate": 1.5993747965137344e-05,
+      "loss": 0.5092,
+      "step": 21500
+    },
+    {
+      "epoch": 0.6195530213115433,
+      "eval_loss": 0.582338273525238,
+      "eval_runtime": 80.9155,
+      "eval_samples_per_second": 24.717,
+      "eval_steps_per_second": 12.359,
+      "step": 21500
+    },
+    {
+      "epoch": 0.6209938422913376,
+      "learning_rate": 1.588766782515136e-05,
+      "loss": 0.4863,
+      "step": 21550
+    },
+    {
+      "epoch": 0.6224346632711318,
+      "learning_rate": 1.5781776551621106e-05,
+      "loss": 0.4874,
+      "step": 21600
+    },
+    {
+      "epoch": 0.6238754842509262,
+      "learning_rate": 1.5676076339298622e-05,
+      "loss": 0.4979,
+      "step": 21650
+    },
+    {
+      "epoch": 0.6253163052307205,
+      "learning_rate": 1.5570569378975908e-05,
+      "loss": 0.4825,
+      "step": 21700
+    },
+    {
+      "epoch": 0.6267571262105147,
+      "learning_rate": 1.5465257857439548e-05,
+      "loss": 0.4943,
+      "step": 21750
+    },
+    {
+      "epoch": 0.6281979471903091,
+      "learning_rate": 1.5360143957425345e-05,
+      "loss": 0.4936,
+      "step": 21800
+    },
+    {
+      "epoch": 0.6296387681701033,
+      "learning_rate": 1.525522985757314e-05,
+      "loss": 0.4858,
+      "step": 21850
+    },
+    {
+      "epoch": 0.6310795891498976,
+      "learning_rate": 1.5150517732381583e-05,
+      "loss": 0.4797,
+      "step": 21900
+    },
+    {
+      "epoch": 0.6325204101296918,
+      "learning_rate": 1.5046009752163132e-05,
+      "loss": 0.4866,
+      "step": 21950
+    },
+    {
+      "epoch": 0.6339612311094862,
+      "learning_rate": 1.4941708082999034e-05,
+      "loss": 0.4677,
+      "step": 22000
+    },
+    {
+      "epoch": 0.6339612311094862,
+      "eval_loss": 0.5860006213188171,
+      "eval_runtime": 83.1333,
+      "eval_samples_per_second": 24.058,
+      "eval_steps_per_second": 12.029,
+      "step": 22000
+    },
+    {
+      "epoch": 0.6354020520892805,
+      "learning_rate": 1.4837614886694428e-05,
+      "loss": 0.5115,
+      "step": 22050
+    },
+    {
+      "epoch": 0.6368428730690747,
+      "learning_rate": 1.4733732320733559e-05,
+      "loss": 0.4914,
+      "step": 22100
+    },
+    {
+      "epoch": 0.6382836940488691,
+      "learning_rate": 1.4630062538235043e-05,
+      "loss": 0.4961,
+      "step": 22150
+    },
+    {
+      "epoch": 0.6397245150286633,
+      "learning_rate": 1.452660768790725e-05,
+      "loss": 0.4938,
+      "step": 22200
+    },
+    {
+      "epoch": 0.6411653360084576,
+      "learning_rate": 1.4423369914003748e-05,
+      "loss": 0.4941,
+      "step": 22250
+    },
+    {
+      "epoch": 0.6426061569882519,
+      "learning_rate": 1.4320351356278905e-05,
+      "loss": 0.4951,
+      "step": 22300
+    },
+    {
+      "epoch": 0.6440469779680462,
+      "learning_rate": 1.4217554149943504e-05,
+      "loss": 0.5041,
+      "step": 22350
+    },
+    {
+      "epoch": 0.6454877989478405,
+      "learning_rate": 1.4114980425620474e-05,
+      "loss": 0.5011,
+      "step": 22400
+    },
+    {
+      "epoch": 0.6469286199276347,
+      "learning_rate": 1.4012632309300789e-05,
+      "loss": 0.4811,
+      "step": 22450
+    },
+    {
+      "epoch": 0.6483694409074291,
+      "learning_rate": 1.3910511922299321e-05,
+      "loss": 0.4893,
+      "step": 22500
+    },
+    {
+      "epoch": 0.6483694409074291,
+      "eval_loss": 0.5825563073158264,
+      "eval_runtime": 77.7579,
+      "eval_samples_per_second": 25.721,
+      "eval_steps_per_second": 12.86,
+      "step": 22500
+    },
+    {
+      "epoch": 0.6498102618872234,
+      "learning_rate": 1.3808621381210967e-05,
+      "loss": 0.4794,
+      "step": 22550
+    },
+    {
+      "epoch": 0.6512510828670176,
+      "learning_rate": 1.370696279786669e-05,
+      "loss": 0.4987,
+      "step": 22600
+    },
+    {
+      "epoch": 0.6526919038468119,
+      "learning_rate": 1.3605538279289798e-05,
+      "loss": 0.4858,
+      "step": 22650
+    },
+    {
+      "epoch": 0.6541327248266062,
+      "learning_rate": 1.3504349927652286e-05,
+      "loss": 0.474,
+      "step": 22700
+    },
+    {
+      "epoch": 0.6555735458064005,
+      "learning_rate": 1.3403399840231203e-05,
+      "loss": 0.4939,
+      "step": 22750
+    },
+    {
+      "epoch": 0.6570143667861947,
+      "learning_rate": 1.330269010936527e-05,
+      "loss": 0.4954,
+      "step": 22800
+    },
+    {
+      "epoch": 0.6584551877659891,
+      "learning_rate": 1.3202222822411416e-05,
+      "loss": 0.4865,
+      "step": 22850
+    },
+    {
+      "epoch": 0.6598960087457834,
+      "learning_rate": 1.3102000061701613e-05,
+      "loss": 0.4943,
+      "step": 22900
+    },
+    {
+      "epoch": 0.6613368297255776,
+      "learning_rate": 1.300202390449963e-05,
+      "loss": 0.486,
+      "step": 22950
+    },
+    {
+      "epoch": 0.6627776507053719,
+      "learning_rate": 1.290229642295802e-05,
+      "loss": 0.4781,
+      "step": 23000
+    },
+    {
+      "epoch": 0.6627776507053719,
+      "eval_loss": 0.5843392610549927,
+      "eval_runtime": 81.2552,
+      "eval_samples_per_second": 24.614,
+      "eval_steps_per_second": 12.307,
+      "step": 23000
+    },
+    {
+      "epoch": 0.6642184716851662,
+      "learning_rate": 1.2802819684075198e-05,
+      "loss": 0.4771,
+      "step": 23050
+    },
+    {
+      "epoch": 0.6656592926649605,
+      "learning_rate": 1.270359574965253e-05,
+      "loss": 0.4698,
+      "step": 23100
+    },
+    {
+      "epoch": 0.6671001136447547,
+      "learning_rate": 1.2604626676251668e-05,
+      "loss": 0.4824,
+      "step": 23150
+    },
+    {
+      "epoch": 0.6685409346245491,
+      "learning_rate": 1.2505914515151898e-05,
+      "loss": 0.4739,
+      "step": 23200
+    },
+    {
+      "epoch": 0.6699817556043434,
+      "learning_rate": 1.2407461312307583e-05,
+      "loss": 0.489,
+      "step": 23250
+    },
+    {
+      "epoch": 0.6714225765841376,
+      "learning_rate": 1.2309269108305843e-05,
+      "loss": 0.4891,
+      "step": 23300
+    },
+    {
+      "epoch": 0.6728633975639319,
+      "learning_rate": 1.2211339938324179e-05,
+      "loss": 0.5021,
+      "step": 23350
+    },
+    {
+      "epoch": 0.6743042185437262,
+      "learning_rate": 1.211367583208835e-05,
+      "loss": 0.4603,
+      "step": 23400
+    },
+    {
+      "epoch": 0.6757450395235205,
+      "learning_rate": 1.201627881383027e-05,
+      "loss": 0.4795,
+      "step": 23450
+    },
+    {
+      "epoch": 0.6771858605033148,
+      "learning_rate": 1.1919150902246045e-05,
+      "loss": 0.4789,
+      "step": 23500
+    },
+    {
+      "epoch": 0.6771858605033148,
+      "eval_loss": 0.5845726728439331,
+      "eval_runtime": 82.5914,
+      "eval_samples_per_second": 24.216,
+      "eval_steps_per_second": 12.108,
+      "step": 23500
+    },
+    {
+      "epoch": 0.6786266814831091,
+      "learning_rate": 1.1822294110454193e-05,
+      "loss": 0.4852,
+      "step": 23550
+    },
+    {
+      "epoch": 0.6800675024629034,
+      "learning_rate": 1.172571044595383e-05,
+      "loss": 0.4997,
+      "step": 23600
+    },
+    {
+      "epoch": 0.6815083234426976,
+      "learning_rate": 1.1629401910583146e-05,
+      "loss": 0.4864,
+      "step": 23650
+    },
+    {
+      "epoch": 0.6829491444224919,
+      "learning_rate": 1.1533370500477845e-05,
+      "loss": 0.4797,
+      "step": 23700
+    },
+    {
+      "epoch": 0.6843899654022862,
+      "learning_rate": 1.1437618206029832e-05,
+      "loss": 0.4843,
+      "step": 23750
+    },
+    {
+      "epoch": 0.6858307863820805,
+      "learning_rate": 1.1342147011845911e-05,
+      "loss": 0.4783,
+      "step": 23800
+    },
+    {
+      "epoch": 0.6872716073618748,
+      "learning_rate": 1.124695889670666e-05,
+      "loss": 0.4826,
+      "step": 23850
+    },
+    {
+      "epoch": 0.6887124283416691,
+      "learning_rate": 1.1152055833525463e-05,
+      "loss": 0.4618,
+      "step": 23900
+    },
+    {
+      "epoch": 0.6901532493214634,
+      "learning_rate": 1.1057439789307566e-05,
+      "loss": 0.4733,
+      "step": 23950
+    },
+    {
+      "epoch": 0.6915940703012576,
+      "learning_rate": 1.0963112725109312e-05,
+      "loss": 0.4811,
+      "step": 24000
+    },
+    {
+      "epoch": 0.6915940703012576,
+      "eval_loss": 0.5840329527854919,
+      "eval_runtime": 79.7677,
+      "eval_samples_per_second": 25.073,
+      "eval_steps_per_second": 12.536,
+      "step": 24000
+    },
+    {
+      "epoch": 0.6930348912810519,
+      "learning_rate": 1.0869076595997535e-05,
+      "loss": 0.4812,
+      "step": 24050
+    },
+    {
+      "epoch": 0.6944757122608463,
+      "learning_rate": 1.0775333351008985e-05,
+      "loss": 0.4989,
+      "step": 24100
+    },
+    {
+      "epoch": 0.6959165332406405,
+      "learning_rate": 1.068188493310999e-05,
+      "loss": 0.485,
+      "step": 24150
+    },
+    {
+      "epoch": 0.6973573542204348,
+      "learning_rate": 1.0588733279156118e-05,
+      "loss": 0.481,
+      "step": 24200
+    },
+    {
+      "epoch": 0.6987981752002291,
+      "learning_rate": 1.0495880319852105e-05,
+      "loss": 0.4792,
+      "step": 24250
+    },
+    {
+      "epoch": 0.7002389961800234,
+      "learning_rate": 1.040332797971178e-05,
+      "loss": 0.4842,
+      "step": 24300
+    },
+    {
+      "epoch": 0.7016798171598176,
+      "learning_rate": 1.031107817701819e-05,
+      "loss": 0.4864,
+      "step": 24350
+    },
+    {
+      "epoch": 0.7031206381396119,
+      "learning_rate": 1.0219132823783883e-05,
+      "loss": 0.4731,
+      "step": 24400
+    },
+    {
+      "epoch": 0.7045614591194063,
+      "learning_rate": 1.0127493825711207e-05,
+      "loss": 0.4714,
+      "step": 24450
+    },
+    {
+      "epoch": 0.7060022800992005,
+      "learning_rate": 1.0036163082152886e-05,
+      "loss": 0.4873,
+      "step": 24500
+    },
+    {
+      "epoch": 0.7060022800992005,
+      "eval_loss": 0.5816783905029297,
+      "eval_runtime": 76.778,
+      "eval_samples_per_second": 26.049,
+      "eval_steps_per_second": 13.025,
+      "step": 24500
+    },
+    {
+      "epoch": 0.7074431010789948,
+      "learning_rate": 9.945142486072581e-06,
+      "loss": 0.4673,
+      "step": 24550
+    },
+    {
+      "epoch": 0.7088839220587891,
+      "learning_rate": 9.854433924005727e-06,
+      "loss": 0.468,
+      "step": 24600
+    },
+    {
+      "epoch": 0.7103247430385834,
+      "learning_rate": 9.764039276020367e-06,
+      "loss": 0.4798,
+      "step": 24650
+    },
+    {
+      "epoch": 0.7117655640183776,
+      "learning_rate": 9.67396041567821e-06,
+      "loss": 0.4652,
+      "step": 24700
+    },
+    {
+      "epoch": 0.7132063849981719,
+      "learning_rate": 9.584199209995855e-06,
+      "loss": 0.4787,
+      "step": 24750
+    },
+    {
+      "epoch": 0.7146472059779663,
+      "learning_rate": 9.494757519405987e-06,
+      "loss": 0.4761,
+      "step": 24800
+    },
+    {
+      "epoch": 0.7160880269577605,
+      "learning_rate": 9.405637197718877e-06,
+      "loss": 0.4642,
+      "step": 24850
+    },
+    {
+      "epoch": 0.7175288479375548,
+      "learning_rate": 9.316840092083989e-06,
+      "loss": 0.4811,
+      "step": 24900
+    },
+    {
+      "epoch": 0.7189696689173491,
+      "learning_rate": 9.228368042951622e-06,
+      "loss": 0.4819,
+      "step": 24950
+    },
+    {
+      "epoch": 0.7204104898971434,
+      "learning_rate": 9.140222884034827e-06,
+      "loss": 0.4766,
+      "step": 25000
+    },
+    {
+      "epoch": 0.7204104898971434,
+      "eval_loss": 0.5827475786209106,
+      "eval_runtime": 76.6955,
+      "eval_samples_per_second": 26.077,
+      "eval_steps_per_second": 13.039,
+      "step": 25000
+    }
+  ],
+  "logging_steps": 50,
+  "max_steps": 34703,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 7.542329573376e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b6f8916837d65a8e78a50633145c2eb04bef144cf5989cecb0e64af2747acf5
+size 5649