Training in progress, step 80, checkpoint

checkpoint-80/README.md

@@ -206,4 +206,5 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 [More Information Needed]
 ### Framework versions
 
+- PEFT 0.17.1
 - PEFT 0.17.0

checkpoint-80/adapter_config.json

@@ -28,16 +28,16 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "gate_proj",
-    "down_proj",
-    "k_proj",
-    "q_proj",
-    "up_proj",
+    "fc1",
     "out_proj",
-    "fc2",
     "o_proj",
-    "fc1",
-    "v_proj"
+    "up_proj",
+    "fc2",
+    "down_proj",
+    "q_proj",
+    "v_proj",
+    "gate_proj",
+    "k_proj"
   ],
   "target_parameters": null,
   "task_type": "CAUSAL_LM",

checkpoint-80/adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b38fdc58f221299e173f4d29117d7c3557d9f9ccc7e564535135bd219c57e26c
+oid sha256:fbc16fc88574d23b2a8ef23b626ea5b8d2ba9b068ae0fb038e7729f73e9bb63d
 size 6127553104

checkpoint-80/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e004d33a7ef5e1384aee2fd88d54dbc7274bd2f5f3ab31eeb1872d7eb5dd6b96
+oid sha256:47147ddfd768955a2389ba2c3c456cd779c8b4dc1df7a442302a77a60038a969
 size 12255795061

checkpoint-80/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02c27ca411a3d5c2d78fff32569312c9b77730b1446b5c1a51897f69007f95c3
+size 14645

checkpoint-80/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:221067d3c8e0a8127ca80a998e2319c9553178d4670269f160050a27de5940f7
+oid sha256:e100b2eac31885bc32fe68c39794d75ce11196153bb3d071fa08e1bad94147db
 size 1465

checkpoint-80/trainer_state.json

@@ -2,7 +2,7 @@
   "best_global_step": null,
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.6652806652806653,
+  "epoch": 0.05534417156693186,
   "eval_steps": 10,
   "global_step": 80,
   "is_hyper_param_search": false,
@@ -10,160 +10,176 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.08316008316008316,
-      "grad_norm": 11.767539024353027,
-      "learning_rate": 0.00019145299145299148,
-      "loss": 18.0054,
-      "mean_token_accuracy": 0.8393091425299645,
-      "num_tokens": 323168.0,
+      "entropy": 3.2428319215774537,
+      "epoch": 0.0069180214458664825,
+      "grad_norm": 45.59263229370117,
+      "learning_rate": 4.0909090909090915e-05,
+      "loss": 8.7229,
+      "mean_token_accuracy": 0.20135476849973202,
+      "num_tokens": 44798.0,
       "step": 10
     },
     {
-      "epoch": 0.08316008316008316,
-      "eval_loss": 0.14767414331436157,
-      "eval_mean_token_accuracy": 0.9865884414085975,
-      "eval_num_tokens": 323168.0,
-      "eval_runtime": 32.0686,
-      "eval_samples_per_second": 6.237,
-      "eval_steps_per_second": 0.405,
+      "epoch": 0.0069180214458664825,
+      "eval_entropy": 3.4517827892303465,
+      "eval_loss": 7.022937774658203,
+      "eval_mean_token_accuracy": 0.29038591831922533,
+      "eval_num_tokens": 44798.0,
+      "eval_runtime": 42.5158,
+      "eval_samples_per_second": 4.704,
+      "eval_steps_per_second": 1.176,
       "step": 10
     },
     {
-      "epoch": 0.16632016632016633,
-      "grad_norm": 4.4300336837768555,
-      "learning_rate": 0.00017435897435897436,
-      "loss": 1.6772,
-      "mean_token_accuracy": 0.9893433898687363,
-      "num_tokens": 646431.0,
+      "entropy": 4.103338432312012,
+      "epoch": 0.013836042891732965,
+      "grad_norm": 15.53995418548584,
+      "learning_rate": 8.636363636363637e-05,
+      "loss": 5.0491,
+      "mean_token_accuracy": 0.44307171255350114,
+      "num_tokens": 89551.0,
       "step": 20
     },
     {
-      "epoch": 0.16632016632016633,
-      "eval_loss": 0.057891350239515305,
-      "eval_mean_token_accuracy": 0.993429972575261,
-      "eval_num_tokens": 646431.0,
-      "eval_runtime": 32.7893,
-      "eval_samples_per_second": 6.1,
-      "eval_steps_per_second": 0.396,
+      "epoch": 0.013836042891732965,
+      "eval_entropy": 4.955911350250244,
+      "eval_loss": 2.6293535232543945,
+      "eval_mean_token_accuracy": 0.6638811433315277,
+      "eval_num_tokens": 89551.0,
+      "eval_runtime": 42.5341,
+      "eval_samples_per_second": 4.702,
+      "eval_steps_per_second": 1.176,
       "step": 20
     },
     {
-      "epoch": 0.2494802494802495,
-      "grad_norm": 2.2051281929016113,
-      "learning_rate": 0.00015726495726495727,
-      "loss": 0.406,
-      "mean_token_accuracy": 0.9940585166215896,
-      "num_tokens": 969623.0,
+      "entropy": 4.794859045743943,
+      "epoch": 0.020754064337599448,
+      "grad_norm": 8.58745002746582,
+      "learning_rate": 0.0001318181818181818,
+      "loss": 1.7872,
+      "mean_token_accuracy": 0.7792469739913941,
+      "num_tokens": 134427.0,
       "step": 30
     },
     {
-      "epoch": 0.2494802494802495,
-      "eval_loss": 0.01031240914016962,
-      "eval_mean_token_accuracy": 0.9945538227374737,
-      "eval_num_tokens": 969623.0,
-      "eval_runtime": 32.2752,
-      "eval_samples_per_second": 6.197,
-      "eval_steps_per_second": 0.403,
+      "epoch": 0.020754064337599448,
+      "eval_entropy": 4.026243486404419,
+      "eval_loss": 1.0728627443313599,
+      "eval_mean_token_accuracy": 0.8464114594459534,
+      "eval_num_tokens": 134427.0,
+      "eval_runtime": 42.5437,
+      "eval_samples_per_second": 4.701,
+      "eval_steps_per_second": 1.175,
       "step": 30
     },
     {
-      "epoch": 0.33264033264033266,
-      "grad_norm": 0.7954460382461548,
-      "learning_rate": 0.00014017094017094016,
-      "loss": 0.156,
-      "mean_token_accuracy": 0.9945382237434387,
-      "num_tokens": 1292839.0,
+      "entropy": 2.9154508650302886,
+      "epoch": 0.02767208578346593,
+      "grad_norm": 5.161023139953613,
+      "learning_rate": 0.00017727272727272728,
+      "loss": 0.7894,
+      "mean_token_accuracy": 0.881743885576725,
+      "num_tokens": 179334.0,
       "step": 40
     },
     {
-      "epoch": 0.33264033264033266,
-      "eval_loss": 0.009185228496789932,
-      "eval_mean_token_accuracy": 0.9949805828241202,
-      "eval_num_tokens": 1292839.0,
-      "eval_runtime": 31.808,
-      "eval_samples_per_second": 6.288,
-      "eval_steps_per_second": 0.409,
+      "epoch": 0.02767208578346593,
+      "eval_entropy": 1.3028265857696533,
+      "eval_loss": 0.27193209528923035,
+      "eval_mean_token_accuracy": 0.8934199070930481,
+      "eval_num_tokens": 179334.0,
+      "eval_runtime": 42.5195,
+      "eval_samples_per_second": 4.704,
+      "eval_steps_per_second": 1.176,
       "step": 40
     },
     {
-      "epoch": 0.4158004158004158,
-      "grad_norm": 0.5219862461090088,
-      "learning_rate": 0.0001230769230769231,
-      "loss": 0.1342,
-      "mean_token_accuracy": 0.9951596394181251,
-      "num_tokens": 1615982.0,
+      "entropy": 0.9636951878666877,
+      "epoch": 0.03459010722933241,
+      "grad_norm": 5.691296100616455,
+      "learning_rate": 0.0001992867332382311,
+      "loss": 0.2666,
+      "mean_token_accuracy": 0.8942964091897011,
+      "num_tokens": 224191.0,
       "step": 50
     },
     {
-      "epoch": 0.4158004158004158,
-      "eval_loss": 0.008571554906666279,
-      "eval_mean_token_accuracy": 0.9952132931122413,
-      "eval_num_tokens": 1615982.0,
-      "eval_runtime": 32.1863,
-      "eval_samples_per_second": 6.214,
-      "eval_steps_per_second": 0.404,
+      "epoch": 0.03459010722933241,
+      "eval_entropy": 0.7585571753978729,
+      "eval_loss": 0.2833440601825714,
+      "eval_mean_token_accuracy": 0.9009752857685089,
+      "eval_num_tokens": 224191.0,
+      "eval_runtime": 42.6014,
+      "eval_samples_per_second": 4.695,
+      "eval_steps_per_second": 1.174,
       "step": 50
     },
     {
-      "epoch": 0.498960498960499,
-      "grad_norm": 0.8362743258476257,
-      "learning_rate": 0.000105982905982906,
-      "loss": 0.1291,
-      "mean_token_accuracy": 0.9950484573841095,
-      "num_tokens": 1939204.0,
+      "entropy": 0.7302425026893615,
+      "epoch": 0.041508128675198895,
+      "grad_norm": 2.4715988636016846,
+      "learning_rate": 0.0001978601997146933,
+      "loss": 0.273,
+      "mean_token_accuracy": 0.888365663588047,
+      "num_tokens": 269084.0,
       "step": 60
     },
     {
-      "epoch": 0.498960498960499,
-      "eval_loss": 0.008207487873733044,
-      "eval_mean_token_accuracy": 0.99532917371163,
-      "eval_num_tokens": 1939204.0,
-      "eval_runtime": 32.3309,
-      "eval_samples_per_second": 6.186,
-      "eval_steps_per_second": 0.402,
+      "epoch": 0.041508128675198895,
+      "eval_entropy": 0.8055200433731079,
+      "eval_loss": 0.21239124238491058,
+      "eval_mean_token_accuracy": 0.9023281943798065,
+      "eval_num_tokens": 269084.0,
+      "eval_runtime": 42.5395,
+      "eval_samples_per_second": 4.702,
+      "eval_steps_per_second": 1.175,
       "step": 60
     },
     {
-      "epoch": 0.5821205821205822,
-      "grad_norm": 0.8683068752288818,
-      "learning_rate": 8.888888888888889e-05,
-      "loss": 0.1313,
-      "mean_token_accuracy": 0.9951223149895668,
-      "num_tokens": 2262372.0,
+      "entropy": 0.7857675984501838,
+      "epoch": 0.048426150121065374,
+      "grad_norm": 1.5094635486602783,
+      "learning_rate": 0.0001964336661911555,
+      "loss": 0.1984,
+      "mean_token_accuracy": 0.9075267255306244,
+      "num_tokens": 313857.0,
       "step": 70
     },
     {
-      "epoch": 0.5821205821205822,
-      "eval_loss": 0.008194765076041222,
-      "eval_mean_token_accuracy": 0.9951937015240009,
-      "eval_num_tokens": 2262372.0,
-      "eval_runtime": 32.1376,
-      "eval_samples_per_second": 6.223,
-      "eval_steps_per_second": 0.405,
+      "epoch": 0.048426150121065374,
+      "eval_entropy": 0.7421065926551819,
+      "eval_loss": 0.17795822024345398,
+      "eval_mean_token_accuracy": 0.9092637586593628,
+      "eval_num_tokens": 313857.0,
+      "eval_runtime": 42.718,
+      "eval_samples_per_second": 4.682,
+      "eval_steps_per_second": 1.17,
       "step": 70
     },
     {
-      "epoch": 0.6652806652806653,
-      "grad_norm": 0.401977002620697,
-      "learning_rate": 7.17948717948718e-05,
-      "loss": 0.1151,
-      "mean_token_accuracy": 0.995387016236782,
-      "num_tokens": 2585532.0,
+      "entropy": 0.7267766013741493,
+      "epoch": 0.05534417156693186,
+      "grad_norm": 1.624765157699585,
+      "learning_rate": 0.0001950071326676177,
+      "loss": 0.1844,
+      "mean_token_accuracy": 0.9070346429944038,
+      "num_tokens": 358622.0,
       "step": 80
     },
     {
-      "epoch": 0.6652806652806653,
-      "eval_loss": 0.007898409850895405,
-      "eval_mean_token_accuracy": 0.9954066093151386,
-      "eval_num_tokens": 2585532.0,
-      "eval_runtime": 32.2572,
-      "eval_samples_per_second": 6.2,
-      "eval_steps_per_second": 0.403,
+      "epoch": 0.05534417156693186,
+      "eval_entropy": 0.8165990447998047,
+      "eval_loss": 0.18048767745494843,
+      "eval_mean_token_accuracy": 0.9136576187610627,
+      "eval_num_tokens": 358622.0,
+      "eval_runtime": 42.5831,
+      "eval_samples_per_second": 4.697,
+      "eval_steps_per_second": 1.174,
       "step": 80
     }
   ],
   "logging_steps": 10,
-  "max_steps": 121,
+  "max_steps": 1446,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 40,
@@ -179,7 +195,7 @@
       "attributes": {}
     }
   },
-  "total_flos": 4.292117885846815e+17,
+  "total_flos": 6.082356697673088e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

checkpoint-80/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd81184c4386bdd5320f1754d4cda79540e3bb45d4e9eeffadfdb4c17e09fef2
-size 6353
+oid sha256:c665ff9710ba066622bdc47a0845adeeeb156957d33148906e62f67561245a3f
+size 6481