Training in progress, step 120, checkpoint

checkpoint-120/README.md

@@ -206,4 +206,5 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
 
+- PEFT 0.17.1
 - PEFT 0.17.0

checkpoint-120/adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "down_proj",
-    "k_proj",
-    "q_proj",
-    "up_proj",
+    "fc1",
     "out_proj",
-    "fc2",
     "o_proj",
-    "fc1",
-    "v_proj"
+    "up_proj",
+    "fc2",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "k_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-120/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a63158af57cbbb839baf13c75b5b7e43c917c3b77e730742ef7255b3e7476392
+oid sha256:4719fe220a3d42f1d99077b29605dce4e46d514fa3df7cb4806703e37700baa6
 size 6127553104

checkpoint-120/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d1c42f8f8f0563636de7a5103cc002f89c6b220220d699bd41c5d5dd4bc8d85c
+oid sha256:0f89f816815b7e2d82741bc75254dd212589d1178c85734f0b6b74446908ee0b
 size 12255795061

checkpoint-120/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b6e2d6d9339ca2e421dac098c5e7d65ad9fb55247e8cfbce3aa56958f70cda6
+size 14645

checkpoint-120/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:293f7c577c9a7e0432f26bd9e01d04d35bfd71833c40176261e1d41887787c1a
+oid sha256:c6b282f4e35f05792bc7cf401176337b080e060e2f8a00bbaa0daa1f582160e6
 size 1465

checkpoint-120/trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.997920997920998,
+  "epoch": 0.08301625735039779,
   "eval_steps": 10,
   "global_step": 120,
   "is_hyper_param_search": false,
@@ -10,236 +10,260 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.08316008316008316,
-      "grad_norm": 11.767539024353027,
-      "learning_rate": 0.00019145299145299148,
-      "loss": 18.0054,
-      "mean_token_accuracy": 0.8393091425299645,
-      "num_tokens": 323168.0,
+      "entropy": 3.2428319215774537,
+      "epoch": 0.0069180214458664825,
+      "grad_norm": 45.59263229370117,
+      "learning_rate": 4.0909090909090915e-05,
+      "loss": 8.7229,
+      "mean_token_accuracy": 0.20135476849973202,
+      "num_tokens": 44798.0,
       "step": 10
     },
     {
-      "epoch": 0.08316008316008316,
-      "eval_loss": 0.14767414331436157,
-      "eval_mean_token_accuracy": 0.9865884414085975,
-      "eval_num_tokens": 323168.0,
-      "eval_runtime": 32.0686,
-      "eval_samples_per_second": 6.237,
-      "eval_steps_per_second": 0.405,
+      "epoch": 0.0069180214458664825,
+      "eval_entropy": 3.4517827892303465,
+      "eval_loss": 7.022937774658203,
+      "eval_mean_token_accuracy": 0.29038591831922533,
+      "eval_num_tokens": 44798.0,
+      "eval_runtime": 42.5158,
+      "eval_samples_per_second": 4.704,
+      "eval_steps_per_second": 1.176,
       "step": 10
     },
     {
-      "epoch": 0.16632016632016633,
-      "grad_norm": 4.4300336837768555,
-      "learning_rate": 0.00017435897435897436,
-      "loss": 1.6772,
-      "mean_token_accuracy": 0.9893433898687363,
-      "num_tokens": 646431.0,
+      "entropy": 4.103338432312012,
+      "epoch": 0.013836042891732965,
+      "grad_norm": 15.53995418548584,
+      "learning_rate": 8.636363636363637e-05,
+      "loss": 5.0491,
+      "mean_token_accuracy": 0.44307171255350114,
+      "num_tokens": 89551.0,
       "step": 20
     },
     {
-      "epoch": 0.16632016632016633,
-      "eval_loss": 0.057891350239515305,
-      "eval_mean_token_accuracy": 0.993429972575261,
-      "eval_num_tokens": 646431.0,
-      "eval_runtime": 32.7893,
-      "eval_samples_per_second": 6.1,
-      "eval_steps_per_second": 0.396,
+      "epoch": 0.013836042891732965,
+      "eval_entropy": 4.955911350250244,
+      "eval_loss": 2.6293535232543945,
+      "eval_mean_token_accuracy": 0.6638811433315277,
+      "eval_num_tokens": 89551.0,
+      "eval_runtime": 42.5341,
+      "eval_samples_per_second": 4.702,
+      "eval_steps_per_second": 1.176,
       "step": 20
     },
     {
-      "epoch": 0.2494802494802495,
-      "grad_norm": 2.2051281929016113,
-      "learning_rate": 0.00015726495726495727,
-      "loss": 0.406,
-      "mean_token_accuracy": 0.9940585166215896,
-      "num_tokens": 969623.0,
+      "entropy": 4.794859045743943,
+      "epoch": 0.020754064337599448,
+      "grad_norm": 8.58745002746582,
+      "learning_rate": 0.0001318181818181818,
+      "loss": 1.7872,
+      "mean_token_accuracy": 0.7792469739913941,
+      "num_tokens": 134427.0,
       "step": 30
     },
     {
-      "epoch": 0.2494802494802495,
-      "eval_loss": 0.01031240914016962,
-      "eval_mean_token_accuracy": 0.9945538227374737,
-      "eval_num_tokens": 969623.0,
-      "eval_runtime": 32.2752,
-      "eval_samples_per_second": 6.197,
-      "eval_steps_per_second": 0.403,
+      "epoch": 0.020754064337599448,
+      "eval_entropy": 4.026243486404419,
+      "eval_loss": 1.0728627443313599,
+      "eval_mean_token_accuracy": 0.8464114594459534,
+      "eval_num_tokens": 134427.0,
+      "eval_runtime": 42.5437,
+      "eval_samples_per_second": 4.701,
+      "eval_steps_per_second": 1.175,
       "step": 30
     },
     {
-      "epoch": 0.33264033264033266,
-      "grad_norm": 0.7954460382461548,
-      "learning_rate": 0.00014017094017094016,
-      "loss": 0.156,
-      "mean_token_accuracy": 0.9945382237434387,
-      "num_tokens": 1292839.0,
+      "entropy": 2.9154508650302886,
+      "epoch": 0.02767208578346593,
+      "grad_norm": 5.161023139953613,
+      "learning_rate": 0.00017727272727272728,
+      "loss": 0.7894,
+      "mean_token_accuracy": 0.881743885576725,
+      "num_tokens": 179334.0,
       "step": 40
     },
     {
-      "epoch": 0.33264033264033266,
-      "eval_loss": 0.009185228496789932,
-      "eval_mean_token_accuracy": 0.9949805828241202,
-      "eval_num_tokens": 1292839.0,
-      "eval_runtime": 31.808,
-      "eval_samples_per_second": 6.288,
-      "eval_steps_per_second": 0.409,
+      "epoch": 0.02767208578346593,
+      "eval_entropy": 1.3028265857696533,
+      "eval_loss": 0.27193209528923035,
+      "eval_mean_token_accuracy": 0.8934199070930481,
+      "eval_num_tokens": 179334.0,
+      "eval_runtime": 42.5195,
+      "eval_samples_per_second": 4.704,
+      "eval_steps_per_second": 1.176,
       "step": 40
     },
     {
-      "epoch": 0.4158004158004158,
-      "grad_norm": 0.5219862461090088,
-      "learning_rate": 0.0001230769230769231,
-      "loss": 0.1342,
-      "mean_token_accuracy": 0.9951596394181251,
-      "num_tokens": 1615982.0,
+      "entropy": 0.9636951878666877,
+      "epoch": 0.03459010722933241,
+      "grad_norm": 5.691296100616455,
+      "learning_rate": 0.0001992867332382311,
+      "loss": 0.2666,
+      "mean_token_accuracy": 0.8942964091897011,
+      "num_tokens": 224191.0,
       "step": 50
     },
     {
-      "epoch": 0.4158004158004158,
-      "eval_loss": 0.008571554906666279,
-      "eval_mean_token_accuracy": 0.9952132931122413,
-      "eval_num_tokens": 1615982.0,
-      "eval_runtime": 32.1863,
-      "eval_samples_per_second": 6.214,
-      "eval_steps_per_second": 0.404,
+      "epoch": 0.03459010722933241,
+      "eval_entropy": 0.7585571753978729,
+      "eval_loss": 0.2833440601825714,
+      "eval_mean_token_accuracy": 0.9009752857685089,
+      "eval_num_tokens": 224191.0,
+      "eval_runtime": 42.6014,
+      "eval_samples_per_second": 4.695,
+      "eval_steps_per_second": 1.174,
       "step": 50
     },
     {
-      "epoch": 0.498960498960499,
-      "grad_norm": 0.8362743258476257,
-      "learning_rate": 0.000105982905982906,
-      "loss": 0.1291,
-      "mean_token_accuracy": 0.9950484573841095,
-      "num_tokens": 1939204.0,
+      "entropy": 0.7302425026893615,
+      "epoch": 0.041508128675198895,
+      "grad_norm": 2.4715988636016846,
+      "learning_rate": 0.0001978601997146933,
+      "loss": 0.273,
+      "mean_token_accuracy": 0.888365663588047,
+      "num_tokens": 269084.0,
       "step": 60
     },
     {
-      "epoch": 0.498960498960499,
-      "eval_loss": 0.008207487873733044,
-      "eval_mean_token_accuracy": 0.99532917371163,
-      "eval_num_tokens": 1939204.0,
-      "eval_runtime": 32.3309,
-      "eval_samples_per_second": 6.186,
-      "eval_steps_per_second": 0.402,
+      "epoch": 0.041508128675198895,
+      "eval_entropy": 0.8055200433731079,
+      "eval_loss": 0.21239124238491058,
+      "eval_mean_token_accuracy": 0.9023281943798065,
+      "eval_num_tokens": 269084.0,
+      "eval_runtime": 42.5395,
+      "eval_samples_per_second": 4.702,
+      "eval_steps_per_second": 1.175,
       "step": 60
     },
     {
-      "epoch": 0.5821205821205822,
-      "grad_norm": 0.8683068752288818,
-      "learning_rate": 8.888888888888889e-05,
-      "loss": 0.1313,
-      "mean_token_accuracy": 0.9951223149895668,
-      "num_tokens": 2262372.0,
+      "entropy": 0.7857675984501838,
+      "epoch": 0.048426150121065374,
+      "grad_norm": 1.5094635486602783,
+      "learning_rate": 0.0001964336661911555,
+      "loss": 0.1984,
+      "mean_token_accuracy": 0.9075267255306244,
+      "num_tokens": 313857.0,
       "step": 70
     },
     {
-      "epoch": 0.5821205821205822,
-      "eval_loss": 0.008194765076041222,
-      "eval_mean_token_accuracy": 0.9951937015240009,
-      "eval_num_tokens": 2262372.0,
-      "eval_runtime": 32.1376,
-      "eval_samples_per_second": 6.223,
-      "eval_steps_per_second": 0.405,
+      "epoch": 0.048426150121065374,
+      "eval_entropy": 0.7421065926551819,
+      "eval_loss": 0.17795822024345398,
+      "eval_mean_token_accuracy": 0.9092637586593628,
+      "eval_num_tokens": 313857.0,
+      "eval_runtime": 42.718,
+      "eval_samples_per_second": 4.682,
+      "eval_steps_per_second": 1.17,
       "step": 70
     },
     {
-      "epoch": 0.6652806652806653,
-      "grad_norm": 0.401977002620697,
-      "learning_rate": 7.17948717948718e-05,
-      "loss": 0.1151,
-      "mean_token_accuracy": 0.995387016236782,
-      "num_tokens": 2585532.0,
+      "entropy": 0.7267766013741493,
+      "epoch": 0.05534417156693186,
+      "grad_norm": 1.624765157699585,
+      "learning_rate": 0.0001950071326676177,
+      "loss": 0.1844,
+      "mean_token_accuracy": 0.9070346429944038,
+      "num_tokens": 358622.0,
       "step": 80
     },
     {
-      "epoch": 0.6652806652806653,
-      "eval_loss": 0.007898409850895405,
-      "eval_mean_token_accuracy": 0.9954066093151386,
-      "eval_num_tokens": 2585532.0,
-      "eval_runtime": 32.2572,
-      "eval_samples_per_second": 6.2,
-      "eval_steps_per_second": 0.403,
+      "epoch": 0.05534417156693186,
+      "eval_entropy": 0.8165990447998047,
+      "eval_loss": 0.18048767745494843,
+      "eval_mean_token_accuracy": 0.9136576187610627,
+      "eval_num_tokens": 358622.0,
+      "eval_runtime": 42.5831,
+      "eval_samples_per_second": 4.697,
+      "eval_steps_per_second": 1.174,
       "step": 80
     },
     {
-      "epoch": 0.7484407484407485,
-      "grad_norm": 0.5641180276870728,
-      "learning_rate": 5.470085470085471e-05,
-      "loss": 0.1226,
-      "mean_token_accuracy": 0.9952676251530648,
-      "num_tokens": 2908697.0,
+      "entropy": 1.002037839591503,
+      "epoch": 0.06226219301279834,
+      "grad_norm": 1.328676462173462,
+      "learning_rate": 0.0001935805991440799,
+      "loss": 0.1736,
+      "mean_token_accuracy": 0.9099974319338798,
+      "num_tokens": 403518.0,
       "step": 90
     },
     {
-      "epoch": 0.7484407484407485,
-      "eval_loss": 0.007712052669376135,
-      "eval_mean_token_accuracy": 0.9953293800354004,
-      "eval_num_tokens": 2908697.0,
-      "eval_runtime": 32.1784,
-      "eval_samples_per_second": 6.215,
-      "eval_steps_per_second": 0.404,
+      "epoch": 0.06226219301279834,
+      "eval_entropy": 1.1328275966644288,
+      "eval_loss": 0.15943188965320587,
+      "eval_mean_token_accuracy": 0.9157200062274933,
+      "eval_num_tokens": 403518.0,
+      "eval_runtime": 42.7297,
+      "eval_samples_per_second": 4.681,
+      "eval_steps_per_second": 1.17,
       "step": 90
     },
     {
-      "epoch": 0.8316008316008316,
-      "grad_norm": 0.4296090602874756,
-      "learning_rate": 3.760683760683761e-05,
-      "loss": 0.1181,
-      "mean_token_accuracy": 0.9952058300375939,
-      "num_tokens": 3231892.0,
+      "entropy": 1.098360726237297,
+      "epoch": 0.06918021445866482,
+      "grad_norm": 1.5309491157531738,
+      "learning_rate": 0.0001921540656205421,
+      "loss": 0.1725,
+      "mean_token_accuracy": 0.9140415117144585,
+      "num_tokens": 448405.0,
       "step": 100
     },
     {
-      "epoch": 0.8316008316008316,
-      "eval_loss": 0.0073149907402694225,
-      "eval_mean_token_accuracy": 0.99550382907574,
-      "eval_num_tokens": 3231892.0,
-      "eval_runtime": 32.1177,
-      "eval_samples_per_second": 6.227,
-      "eval_steps_per_second": 0.405,
+      "epoch": 0.06918021445866482,
+      "eval_entropy": 1.028436896800995,
+      "eval_loss": 0.16933326423168182,
+      "eval_mean_token_accuracy": 0.915260488986969,
+      "eval_num_tokens": 448405.0,
+      "eval_runtime": 42.7333,
+      "eval_samples_per_second": 4.68,
+      "eval_steps_per_second": 1.17,
       "step": 100
     },
     {
-      "epoch": 0.9147609147609148,
-      "grad_norm": 0.3773857057094574,
-      "learning_rate": 2.0512820512820512e-05,
-      "loss": 0.1155,
-      "mean_token_accuracy": 0.99525695592165,
-      "num_tokens": 3555132.0,
+      "entropy": 1.0331062585115434,
+      "epoch": 0.0760982359045313,
+      "grad_norm": 1.9411410093307495,
+      "learning_rate": 0.00019072753209700428,
+      "loss": 0.1761,
+      "mean_token_accuracy": 0.9094761416316033,
+      "num_tokens": 493303.0,
       "step": 110
     },
     {
-      "epoch": 0.9147609147609148,
-      "eval_loss": 0.007079997565597296,
-      "eval_mean_token_accuracy": 0.9955812417543851,
-      "eval_num_tokens": 3555132.0,
-      "eval_runtime": 30.7586,
-      "eval_samples_per_second": 6.502,
-      "eval_steps_per_second": 0.423,
+      "epoch": 0.0760982359045313,
+      "eval_entropy": 1.0022910988330842,
+      "eval_loss": 0.16668693721294403,
+      "eval_mean_token_accuracy": 0.9155051994323731,
+      "eval_num_tokens": 493303.0,
+      "eval_runtime": 42.6346,
+      "eval_samples_per_second": 4.691,
+      "eval_steps_per_second": 1.173,
       "step": 110
     },
     {
-      "epoch": 0.997920997920998,
-      "grad_norm": 0.2652049958705902,
-      "learning_rate": 3.4188034188034193e-06,
-      "loss": 0.1145,
-      "mean_token_accuracy": 0.9952491670846939,
-      "num_tokens": 3878323.0,
+      "entropy": 1.0691627204418181,
+      "epoch": 0.08301625735039779,
+      "grad_norm": 1.2629178762435913,
+      "learning_rate": 0.00018930099857346648,
+      "loss": 0.1788,
+      "mean_token_accuracy": 0.9090283781290054,
+      "num_tokens": 538145.0,
       "step": 120
     },
     {
-      "epoch": 0.997920997920998,
-      "eval_loss": 0.006998998112976551,
-      "eval_mean_token_accuracy": 0.9955618839997512,
-      "eval_num_tokens": 3878323.0,
-      "eval_runtime": 30.145,
-      "eval_samples_per_second": 6.635,
-      "eval_steps_per_second": 0.431,
+      "epoch": 0.08301625735039779,
+      "eval_entropy": 1.4147576117515563,
+      "eval_loss": 0.15232698619365692,
+      "eval_mean_token_accuracy": 0.9161913430690766,
+      "eval_num_tokens": 538145.0,
+      "eval_runtime": 42.5837,
+      "eval_samples_per_second": 4.697,
+      "eval_steps_per_second": 1.174,
       "step": 120
     }
   ],
   "logging_steps": 10,
-  "max_steps": 121,
+  "max_steps": 1446,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 40,
@@ -255,7 +279,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 6.438173500170568e+17,
+  "total_flos": 9.133981968547162e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

checkpoint-120/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd81184c4386bdd5320f1754d4cda79540e3bb45d4e9eeffadfdb4c17e09fef2
-size 6353
+oid sha256:c665ff9710ba066622bdc47a0845adeeeb156957d33148906e62f67561245a3f
+size 6481