Update AGILLM4 training log tail
Browse files
training/agillm4_floor_v47_v4pro/logs/train_tail.log
CHANGED
|
@@ -1,7 +1,3 @@
|
|
| 1 |
-
[dblock] step=89950 block=3 obj=nat layers=[21, 22, 23, 24, 25, 26, 27] loss=4.282 weighted=4.282 ar=0.000 sat=0.000 nat=4.282 raw_sum=4.282 edm_w=4.169 counts=[19233,15666,15715,39336] ema=[18.10,20.71,21.58,27.49] peak_alloc=13.90GB peak_reserved=18.20GB
|
| 2 |
-
[6.1218%] 3,649,299,456/59,611,622,510 tok | 3398.66 tok/s | loss=7.111 B=6 L=1024 step=89970 eta=190d13h53m elapsed=1d21h10m
|
| 3 |
-
[dblock] step=89975 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.553 weighted=30.566 ar=30.566 sat=0.000 nat=0.000 raw_sum=7.553 edm_w=4.047 counts=[19233,15667,15715,39360] ema=[18.10,21.20,21.58,25.59] peak_alloc=16.16GB peak_reserved=18.20GB
|
| 4 |
-
[dblock] step=90000 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.042 weighted=30.418 ar=30.418 sat=0.000 nat=0.000 raw_sum=7.042 edm_w=4.320 counts=[19233,15668,15715,39384] ema=[18.10,21.23,21.58,26.30] peak_alloc=16.16GB peak_reserved=18.20GB
|
| 5 |
[6.1221%] 3,649,508,352/59,611,622,510 tok | 3398.68 tok/s | loss=7.995 B=6 L=1024 step=90004 eta=190d13h50m elapsed=1d21h11m
|
| 6 |
[dblock] step=90025 block=1 obj=ar layers=[7, 8, 9, 10, 11, 12, 13] loss=4.879 weighted=24.395 ar=24.395 sat=0.000 nat=0.000 raw_sum=4.879 edm_w=5.000 counts=[19233,15671,15715,39406] ema=[18.10,20.92,21.58,27.46] peak_alloc=16.06GB peak_reserved=18.20GB
|
| 7 |
[6.1225%] 3,649,711,104/59,611,622,510 tok | 3398.67 tok/s | loss=6.836 B=6 L=1024 step=90037 eta=190d13h50m elapsed=1d21h12m
|
|
@@ -4998,3 +4994,7 @@ Retrying in 8s [Retry 5/5].
|
|
| 4998 |
[dblock] step=152750 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.494 weighted=31.528 ar=31.528 sat=0.000 nat=0.000 raw_sum=7.494 edm_w=4.207 counts=[33977,29723,30231,58819] ema=[21.13,24.78,24.41,29.62] peak_alloc=16.01GB peak_reserved=18.20GB
|
| 4999 |
[6.7690%] 4,035,093,504/59,611,622,510 tok | 3386.26 tok/s | loss=7.358 B=6 L=1024 step=152762 eta=189d22h59m elapsed=3d4h59m
|
| 5000 |
[dblock] step=152775 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.096 weighted=29.652 ar=29.652 sat=0.000 nat=0.000 raw_sum=7.096 edm_w=4.179 counts=[33977,29724,30231,58843] ema=[21.13,24.04,24.41,28.78] peak_alloc=16.01GB peak_reserved=18.20GB
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
[6.1221%] 3,649,508,352/59,611,622,510 tok | 3398.68 tok/s | loss=7.995 B=6 L=1024 step=90004 eta=190d13h50m elapsed=1d21h11m
|
| 2 |
[dblock] step=90025 block=1 obj=ar layers=[7, 8, 9, 10, 11, 12, 13] loss=4.879 weighted=24.395 ar=24.395 sat=0.000 nat=0.000 raw_sum=4.879 edm_w=5.000 counts=[19233,15671,15715,39406] ema=[18.10,20.92,21.58,27.46] peak_alloc=16.06GB peak_reserved=18.20GB
|
| 3 |
[6.1225%] 3,649,711,104/59,611,622,510 tok | 3398.67 tok/s | loss=6.836 B=6 L=1024 step=90037 eta=190d13h50m elapsed=1d21h12m
|
|
|
|
| 4994 |
[dblock] step=152750 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.494 weighted=31.528 ar=31.528 sat=0.000 nat=0.000 raw_sum=7.494 edm_w=4.207 counts=[33977,29723,30231,58819] ema=[21.13,24.78,24.41,29.62] peak_alloc=16.01GB peak_reserved=18.20GB
|
| 4995 |
[6.7690%] 4,035,093,504/59,611,622,510 tok | 3386.26 tok/s | loss=7.358 B=6 L=1024 step=152762 eta=189d22h59m elapsed=3d4h59m
|
| 4996 |
[dblock] step=152775 block=3 obj=ar layers=[21, 22, 23, 24, 25, 26, 27] loss=7.096 weighted=29.652 ar=29.652 sat=0.000 nat=0.000 raw_sum=7.096 edm_w=4.179 counts=[33977,29724,30231,58843] ema=[21.13,24.04,24.41,28.78] peak_alloc=16.01GB peak_reserved=18.20GB
|
| 4997 |
+
[6.7693%] 4,035,308,544/59,611,622,510 tok | 3386.29 tok/s | loss=6.832 B=6 L=1024 step=152797 eta=189d22h55m elapsed=3d5h0m
|
| 4998 |
+
[dblock] step=152800 block=1 obj=nat layers=[7, 8, 9, 10, 11, 12, 13] loss=6.548 weighted=6.548 ar=0.000 sat=0.000 nat=6.548 raw_sum=6.548 edm_w=5.000 counts=[33977,29730,30231,58862] ema=[21.13,24.38,24.41,24.79] peak_alloc=13.85GB peak_reserved=18.20GB
|
| 4999 |
+
[heartbeat] phase=pretrain pid=1389025 step=1546321 seen_tok=4035419136 loss=6.009 B=6 L=1024 dblock=True structured_masks=True gpu_alloc=4.91GB gpu_reserved=18.20GB gpu_peak=16.06GB
|
| 5000 |
+
[dblock] step=152825 block=1 obj=ar layers=[7, 8, 9, 10, 11, 12, 13] loss=5.989 weighted=29.944 ar=29.944 sat=0.000 nat=0.000 raw_sum=5.989 edm_w=5.000 counts=[33977,29751,30231,58866] ema=[21.13,27.03,24.41,24.00] peak_alloc=16.06GB peak_reserved=18.20GB
|