Training in progress, step 24000, checkpoint

.gitattributes

@@ -34,3 +34,21 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 tokenizer.json filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/.metadata filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__4_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__5_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/optimizer_0/__7_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/.metadata filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__0_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__1_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__2_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__3_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__4_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__5_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__6_0.distcp filter=lfs diff=lfs merge=lfs -text
+checkpoint-24000/pytorch_model_fsdp_0/__7_0.distcp filter=lfs diff=lfs merge=lfs -text

checkpoint-24000/optimizer_0/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60d809dc038d23e6b68cadd99cf7af9077de77cb0f0d6f43b24e979112a1b295
+size 1533608

checkpoint-24000/optimizer_0/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c8f9a30e3c45009d56a5ec0be20b0f09ed563637d5e6a8cc5201678d12b8658
+size 4023049125

checkpoint-24000/optimizer_0/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3433387274e05c71b313cc9135bddf4ed84ae1a1b909be9b0ba841a830fc5391
+size 4023141220

checkpoint-24000/optimizer_0/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85dc86e0b776b0fb8c09ee086e905bd52351bc9e5889312b69e19a5db47530f3
+size 4023141220

checkpoint-24000/optimizer_0/__3_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b24665e27733cfd98badec3d742e975a5d8c02e23fd72f1f736ac7cfb71e1c9a
+size 4023141220

checkpoint-24000/optimizer_0/__4_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7c346cf13a44d2ab2d097f3b070e518ce4dad8cc981c5225455c22ecbed8ac80
+size 4023148068

checkpoint-24000/optimizer_0/__5_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:91681be6b0c1cbea30195add7211810908ee6a1384b8aae0e50dc2bbcf0a49b2
+size 4023151095

checkpoint-24000/optimizer_0/__6_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6a3d476f977cb582007803dacd42839e48f3e44196c6f56d33a2b7fe22dd3640
+size 4023143426

checkpoint-24000/optimizer_0/__7_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:102e798e62fbeab20cf08fdd87c3334c396bbc4487a088c177af16f986e01aa2
+size 4023102466

checkpoint-24000/pytorch_model_fsdp_0/.metadata

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cf8c69c0f61cedf27e006f46d0615e3ed5281c86d998c7c677b6ebe1869011b
+size 619643

checkpoint-24000/pytorch_model_fsdp_0/__0_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:465a3b30a1c5731e2c224c726b23a3944a8c39566d760269b69375ea51cc4dab
+size 2205665511

checkpoint-24000/pytorch_model_fsdp_0/__1_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0302b4d3f3ed79d1c04a299bfa6c6fc308a38890af0fe85d2b270bb7bf09804
+size 2205665511

checkpoint-24000/pytorch_model_fsdp_0/__2_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ffecb5db7d39c047b17d3151906450074df0f4ec3cf755f848bff70c2fd41c0
+size 2205665511

checkpoint-24000/pytorch_model_fsdp_0/__3_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38128be2a377211e27fcd8b0ec30b047beb14f4b78b7ceb09837a560f3cd76eb
+size 2205665511

checkpoint-24000/pytorch_model_fsdp_0/__4_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aad4ba9cb2ee8f45f44fa38d2cf2195451d66108bfa63b134cb4675adad94802
+size 2205665511

checkpoint-24000/pytorch_model_fsdp_0/__5_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acf1405df06d42c235f04aa8c8ca91d8865848be980a528c6a24259fa7818da4
+size 2205665511

checkpoint-24000/pytorch_model_fsdp_0/__6_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87cd769f937531e3be7724e7b09b9868db4fd0ba5b854bf18555e59b5f6e2cbb
+size 2205665511

checkpoint-24000/pytorch_model_fsdp_0/__7_0.distcp

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:755afbcc97d4acb1535168889253dd21d7a099f1be37222ea627139639939976
+size 2205624551

checkpoint-24000/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cddc2b4f381e5ca5a26121bb07a893055ef5cb7d28c6c804c54b7e8d98446e73
+size 16389

checkpoint-24000/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e223bd10aee24162e1caff1370d9823c6846b6780971014a5c65007329cf694
+size 16389

checkpoint-24000/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d892b976cfbd46eba059246c19b4b236eef8897d221e63ff755314090b6c1468
+size 16389

checkpoint-24000/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f995e14936a94c2b4f46193c32d32ed9c674ebe0820b87e0fe38a3f656d62bc5
+size 16389

checkpoint-24000/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fcace709c04c0017b85531964b5aaa9354a122d1139a4cd86c4acf898c0b809b
+size 16389

checkpoint-24000/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e2a0607eaf41f26d8178eb478ff007b4207bfc0f4f183d56c62f253f2884a1f
+size 16389

checkpoint-24000/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aafdec9b603fbcd684a36f5e84626b08d6551bd21559c78bb5220f60b0350c00
+size 16389

checkpoint-24000/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e18e31b5a24e2da5e88b055997bdbce1d4a5b1ca620f28fcbe635ad6bb3da0f
+size 16389

checkpoint-24000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03754a580d7015d8f70235b3c57e7717091a82d6358d70b6add499ae0240195c
+size 1465

checkpoint-24000/trainer_state.json

@@ -0,0 +1,418 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 13.793103448275861,
+  "eval_steps": 500,
+  "global_step": 24000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.28735632183908044,
+      "grad_norm": 0.27726614475250244,
+      "learning_rate": 1.9617624521072798e-05,
+      "loss": 0.237,
+      "mean_token_accuracy": 0.9094978541135788,
+      "step": 500
+    },
+    {
+      "epoch": 0.5747126436781609,
+      "grad_norm": 0.3931587040424347,
+      "learning_rate": 1.9234482758620693e-05,
+      "loss": 0.2304,
+      "mean_token_accuracy": 0.9114951363801956,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8620689655172413,
+      "grad_norm": 0.370857834815979,
+      "learning_rate": 1.8851340996168584e-05,
+      "loss": 0.2126,
+      "mean_token_accuracy": 0.9183068807125092,
+      "step": 1500
+    },
+    {
+      "epoch": 1.1494252873563218,
+      "grad_norm": 0.34739020466804504,
+      "learning_rate": 1.8468199233716476e-05,
+      "loss": 0.2058,
+      "mean_token_accuracy": 0.9208864206075669,
+      "step": 2000
+    },
+    {
+      "epoch": 1.4367816091954024,
+      "grad_norm": 0.44222190976142883,
+      "learning_rate": 1.8085057471264368e-05,
+      "loss": 0.1941,
+      "mean_token_accuracy": 0.9255667915344238,
+      "step": 2500
+    },
+    {
+      "epoch": 1.7241379310344827,
+      "grad_norm": 0.3851265609264374,
+      "learning_rate": 1.7701915708812263e-05,
+      "loss": 0.1864,
+      "mean_token_accuracy": 0.9283954718112946,
+      "step": 3000
+    },
+    {
+      "epoch": 2.0114942528735633,
+      "grad_norm": 0.31711894273757935,
+      "learning_rate": 1.7318773946360154e-05,
+      "loss": 0.1848,
+      "mean_token_accuracy": 0.9288191899061203,
+      "step": 3500
+    },
+    {
+      "epoch": 2.2988505747126435,
+      "grad_norm": 0.5404093265533447,
+      "learning_rate": 1.693563218390805e-05,
+      "loss": 0.1723,
+      "mean_token_accuracy": 0.9336981619596482,
+      "step": 4000
+    },
+    {
+      "epoch": 2.586206896551724,
+      "grad_norm": 0.22831037640571594,
+      "learning_rate": 1.655249042145594e-05,
+      "loss": 0.1645,
+      "mean_token_accuracy": 0.9365380902290344,
+      "step": 4500
+    },
+    {
+      "epoch": 2.873563218390805,
+      "grad_norm": 0.22406116127967834,
+      "learning_rate": 1.6169348659003833e-05,
+      "loss": 0.1699,
+      "mean_token_accuracy": 0.9345249460935593,
+      "step": 5000
+    },
+    {
+      "epoch": 3.160919540229885,
+      "grad_norm": 0.42187097668647766,
+      "learning_rate": 1.5786206896551724e-05,
+      "loss": 0.1599,
+      "mean_token_accuracy": 0.9384886187314987,
+      "step": 5500
+    },
+    {
+      "epoch": 3.4482758620689653,
+      "grad_norm": 0.23665696382522583,
+      "learning_rate": 1.5403065134099616e-05,
+      "loss": 0.1579,
+      "mean_token_accuracy": 0.9392954226732254,
+      "step": 6000
+    },
+    {
+      "epoch": 3.735632183908046,
+      "grad_norm": 0.24368983507156372,
+      "learning_rate": 1.5019923371647511e-05,
+      "loss": 0.1587,
+      "mean_token_accuracy": 0.9390693799257278,
+      "step": 6500
+    },
+    {
+      "epoch": 4.022988505747127,
+      "grad_norm": 0.33987343311309814,
+      "learning_rate": 1.4636781609195403e-05,
+      "loss": 0.1502,
+      "mean_token_accuracy": 0.942336865067482,
+      "step": 7000
+    },
+    {
+      "epoch": 4.310344827586207,
+      "grad_norm": 0.24024857580661774,
+      "learning_rate": 1.4253639846743296e-05,
+      "loss": 0.1372,
+      "mean_token_accuracy": 0.9477886618375778,
+      "step": 7500
+    },
+    {
+      "epoch": 4.597701149425287,
+      "grad_norm": 0.4761509895324707,
+      "learning_rate": 1.3870498084291188e-05,
+      "loss": 0.1422,
+      "mean_token_accuracy": 0.9457951341867447,
+      "step": 8000
+    },
+    {
+      "epoch": 4.885057471264368,
+      "grad_norm": 0.326945424079895,
+      "learning_rate": 1.3487356321839083e-05,
+      "loss": 0.1381,
+      "mean_token_accuracy": 0.9473243873119355,
+      "step": 8500
+    },
+    {
+      "epoch": 5.172413793103448,
+      "grad_norm": 0.9344919919967651,
+      "learning_rate": 1.3104214559386974e-05,
+      "loss": 0.1167,
+      "mean_token_accuracy": 0.9559659276008606,
+      "step": 9000
+    },
+    {
+      "epoch": 5.459770114942529,
+      "grad_norm": 0.5047450065612793,
+      "learning_rate": 1.2721072796934868e-05,
+      "loss": 0.1123,
+      "mean_token_accuracy": 0.957923208117485,
+      "step": 9500
+    },
+    {
+      "epoch": 5.747126436781609,
+      "grad_norm": 0.6128556728363037,
+      "learning_rate": 1.233793103448276e-05,
+      "loss": 0.1172,
+      "mean_token_accuracy": 0.9561160520315171,
+      "step": 10000
+    },
+    {
+      "epoch": 6.0344827586206895,
+      "grad_norm": 0.7909197211265564,
+      "learning_rate": 1.1954789272030651e-05,
+      "loss": 0.109,
+      "mean_token_accuracy": 0.9592113490104676,
+      "step": 10500
+    },
+    {
+      "epoch": 6.32183908045977,
+      "grad_norm": 0.6463600993156433,
+      "learning_rate": 1.1571647509578544e-05,
+      "loss": 0.0783,
+      "mean_token_accuracy": 0.9713092336654663,
+      "step": 11000
+    },
+    {
+      "epoch": 6.609195402298851,
+      "grad_norm": 1.0739723443984985,
+      "learning_rate": 1.1188505747126436e-05,
+      "loss": 0.0806,
+      "mean_token_accuracy": 0.9704036431312562,
+      "step": 11500
+    },
+    {
+      "epoch": 6.896551724137931,
+      "grad_norm": 0.4464154541492462,
+      "learning_rate": 1.0805363984674331e-05,
+      "loss": 0.0828,
+      "mean_token_accuracy": 0.9695826562643051,
+      "step": 12000
+    },
+    {
+      "epoch": 7.183908045977011,
+      "grad_norm": 0.8025017976760864,
+      "learning_rate": 1.0422222222222223e-05,
+      "loss": 0.0621,
+      "mean_token_accuracy": 0.9774264923334122,
+      "step": 12500
+    },
+    {
+      "epoch": 7.471264367816092,
+      "grad_norm": 0.5045526027679443,
+      "learning_rate": 1.0039080459770116e-05,
+      "loss": 0.0498,
+      "mean_token_accuracy": 0.9821003634929657,
+      "step": 13000
+    },
+    {
+      "epoch": 7.758620689655173,
+      "grad_norm": 1.7900103330612183,
+      "learning_rate": 9.655938697318008e-06,
+      "loss": 0.0512,
+      "mean_token_accuracy": 0.9815057872533798,
+      "step": 13500
+    },
+    {
+      "epoch": 8.045977011494253,
+      "grad_norm": 1.2213754653930664,
+      "learning_rate": 9.272796934865901e-06,
+      "loss": 0.0465,
+      "mean_token_accuracy": 0.983227198600769,
+      "step": 14000
+    },
+    {
+      "epoch": 8.333333333333334,
+      "grad_norm": 1.0093538761138916,
+      "learning_rate": 8.889655172413794e-06,
+      "loss": 0.0254,
+      "mean_token_accuracy": 0.9910121623277665,
+      "step": 14500
+    },
+    {
+      "epoch": 8.620689655172415,
+      "grad_norm": 0.4820902347564697,
+      "learning_rate": 8.506513409961686e-06,
+      "loss": 0.0275,
+      "mean_token_accuracy": 0.9902000530958176,
+      "step": 15000
+    },
+    {
+      "epoch": 8.908045977011493,
+      "grad_norm": 1.4471999406814575,
+      "learning_rate": 8.12337164750958e-06,
+      "loss": 0.0279,
+      "mean_token_accuracy": 0.990046201467514,
+      "step": 15500
+    },
+    {
+      "epoch": 9.195402298850574,
+      "grad_norm": 1.0422757863998413,
+      "learning_rate": 7.740229885057473e-06,
+      "loss": 0.0178,
+      "mean_token_accuracy": 0.9936608210802078,
+      "step": 16000
+    },
+    {
+      "epoch": 9.482758620689655,
+      "grad_norm": 0.7313858866691589,
+      "learning_rate": 7.3570881226053644e-06,
+      "loss": 0.0137,
+      "mean_token_accuracy": 0.9951526633501053,
+      "step": 16500
+    },
+    {
+      "epoch": 9.770114942528735,
+      "grad_norm": 1.0549014806747437,
+      "learning_rate": 6.973946360153258e-06,
+      "loss": 0.0151,
+      "mean_token_accuracy": 0.9946176511049271,
+      "step": 17000
+    },
+    {
+      "epoch": 10.057471264367816,
+      "grad_norm": 0.4364485740661621,
+      "learning_rate": 6.59080459770115e-06,
+      "loss": 0.0132,
+      "mean_token_accuracy": 0.9953484191894532,
+      "step": 17500
+    },
+    {
+      "epoch": 10.344827586206897,
+      "grad_norm": 0.621101438999176,
+      "learning_rate": 6.207662835249043e-06,
+      "loss": 0.0068,
+      "mean_token_accuracy": 0.9976246099472046,
+      "step": 18000
+    },
+    {
+      "epoch": 10.632183908045977,
+      "grad_norm": 0.33462387323379517,
+      "learning_rate": 5.8245210727969344e-06,
+      "loss": 0.0077,
+      "mean_token_accuracy": 0.997280099272728,
+      "step": 18500
+    },
+    {
+      "epoch": 10.919540229885058,
+      "grad_norm": 0.539320170879364,
+      "learning_rate": 5.441379310344828e-06,
+      "loss": 0.0079,
+      "mean_token_accuracy": 0.9971861612796783,
+      "step": 19000
+    },
+    {
+      "epoch": 11.206896551724139,
+      "grad_norm": 0.15973538160324097,
+      "learning_rate": 5.05823754789272e-06,
+      "loss": 0.0048,
+      "mean_token_accuracy": 0.9982997093200684,
+      "step": 19500
+    },
+    {
+      "epoch": 11.494252873563218,
+      "grad_norm": 0.41210290789604187,
+      "learning_rate": 4.675095785440613e-06,
+      "loss": 0.0038,
+      "mean_token_accuracy": 0.9986591664552689,
+      "step": 20000
+    },
+    {
+      "epoch": 11.781609195402298,
+      "grad_norm": 0.4738906919956207,
+      "learning_rate": 4.291954022988506e-06,
+      "loss": 0.004,
+      "mean_token_accuracy": 0.9985732196569442,
+      "step": 20500
+    },
+    {
+      "epoch": 12.068965517241379,
+      "grad_norm": 0.08584625273942947,
+      "learning_rate": 3.908812260536399e-06,
+      "loss": 0.0036,
+      "mean_token_accuracy": 0.9987244753837585,
+      "step": 21000
+    },
+    {
+      "epoch": 12.35632183908046,
+      "grad_norm": 0.24842122197151184,
+      "learning_rate": 3.5256704980842915e-06,
+      "loss": 0.0018,
+      "mean_token_accuracy": 0.9993553450107574,
+      "step": 21500
+    },
+    {
+      "epoch": 12.64367816091954,
+      "grad_norm": 0.4292615056037903,
+      "learning_rate": 3.1425287356321844e-06,
+      "loss": 0.002,
+      "mean_token_accuracy": 0.9992790446281433,
+      "step": 22000
+    },
+    {
+      "epoch": 12.931034482758621,
+      "grad_norm": 0.5820701718330383,
+      "learning_rate": 2.759386973180077e-06,
+      "loss": 0.0021,
+      "mean_token_accuracy": 0.9992240278720855,
+      "step": 22500
+    },
+    {
+      "epoch": 13.218390804597702,
+      "grad_norm": 0.1409875601530075,
+      "learning_rate": 2.3762452107279694e-06,
+      "loss": 0.0012,
+      "mean_token_accuracy": 0.9995475882291793,
+      "step": 23000
+    },
+    {
+      "epoch": 13.505747126436782,
+      "grad_norm": 0.20804591476917267,
+      "learning_rate": 1.993103448275862e-06,
+      "loss": 0.0009,
+      "mean_token_accuracy": 0.999629429936409,
+      "step": 23500
+    },
+    {
+      "epoch": 13.793103448275861,
+      "grad_norm": 0.15083467960357666,
+      "learning_rate": 1.6099616858237549e-06,
+      "loss": 0.001,
+      "mean_token_accuracy": 0.9996287471055985,
+      "step": 24000
+    }
+  ],
+  "logging_steps": 500,
+  "max_steps": 26100,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 15,
+  "save_steps": 24000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8.770765095404831e+17,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}