| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9982851866508377, |
| "eval_steps": 400, |
| "global_step": 473, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.00211053950666139, |
| "grad_norm": 5.643460436957748, |
| "learning_rate": 1.0416666666666666e-08, |
| "logits/chosen": -1.5622574090957642, |
| "logits/rejected": -2.016603946685791, |
| "logps/chosen": -279.929443359375, |
| "logps/rejected": -249.6509552001953, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.0, |
| "rewards/chosen": 0.0, |
| "rewards/margins": 0.0, |
| "rewards/rejected": 0.0, |
| "step": 1 |
| }, |
| { |
| "epoch": 0.010552697533306952, |
| "grad_norm": 4.760670706167096, |
| "learning_rate": 5.208333333333333e-08, |
| "logits/chosen": -1.6410560607910156, |
| "logits/rejected": -1.8854162693023682, |
| "logps/chosen": -306.70123291015625, |
| "logps/rejected": -286.2392883300781, |
| "loss": 0.6934, |
| "rewards/accuracies": 0.3125, |
| "rewards/chosen": -0.0017719048773869872, |
| "rewards/margins": -0.0009851222857832909, |
| "rewards/rejected": -0.0007867825916036963, |
| "step": 5 |
| }, |
| { |
| "epoch": 0.021105395066613904, |
| "grad_norm": 4.237628799563217, |
| "learning_rate": 1.0416666666666667e-07, |
| "logits/chosen": -1.4761555194854736, |
| "logits/rejected": -1.7796385288238525, |
| "logps/chosen": -290.88739013671875, |
| "logps/rejected": -265.3614196777344, |
| "loss": 0.6931, |
| "rewards/accuracies": 0.512499988079071, |
| "rewards/chosen": -0.001178879290819168, |
| "rewards/margins": 0.0011062298435717821, |
| "rewards/rejected": -0.00228510913439095, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.031658092599920855, |
| "grad_norm": 5.575235759782868, |
| "learning_rate": 1.5624999999999999e-07, |
| "logits/chosen": -1.5923292636871338, |
| "logits/rejected": -1.9355911016464233, |
| "logps/chosen": -293.08807373046875, |
| "logps/rejected": -261.4955139160156, |
| "loss": 0.6929, |
| "rewards/accuracies": 0.44999998807907104, |
| "rewards/chosen": -8.010577585082501e-05, |
| "rewards/margins": -0.0003216963086742908, |
| "rewards/rejected": 0.00024159046006388962, |
| "step": 15 |
| }, |
| { |
| "epoch": 0.04221079013322781, |
| "grad_norm": 5.446047742742675, |
| "learning_rate": 2.0833333333333333e-07, |
| "logits/chosen": -1.5667310953140259, |
| "logits/rejected": -2.014115810394287, |
| "logps/chosen": -273.8595275878906, |
| "logps/rejected": -235.01364135742188, |
| "loss": 0.6923, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": 0.0028162619564682245, |
| "rewards/margins": 0.0022979697678238153, |
| "rewards/rejected": 0.0005182920140214264, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.052763487666534756, |
| "grad_norm": 5.373521803666822, |
| "learning_rate": 2.604166666666667e-07, |
| "logits/chosen": -1.6274404525756836, |
| "logits/rejected": -1.875451683998108, |
| "logps/chosen": -279.4980163574219, |
| "logps/rejected": -255.500244140625, |
| "loss": 0.691, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": 0.011563817039132118, |
| "rewards/margins": 0.003255133982747793, |
| "rewards/rejected": 0.008308682590723038, |
| "step": 25 |
| }, |
| { |
| "epoch": 0.06331618519984171, |
| "grad_norm": 6.225478720213553, |
| "learning_rate": 3.1249999999999997e-07, |
| "logits/chosen": -1.3949791193008423, |
| "logits/rejected": -1.7053276300430298, |
| "logps/chosen": -295.1358337402344, |
| "logps/rejected": -266.3870849609375, |
| "loss": 0.6882, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": 0.02798023819923401, |
| "rewards/margins": 0.00888301245868206, |
| "rewards/rejected": 0.0190972238779068, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.07386888273314866, |
| "grad_norm": 5.237638647535282, |
| "learning_rate": 3.645833333333333e-07, |
| "logits/chosen": -1.6196448802947998, |
| "logits/rejected": -1.9479618072509766, |
| "logps/chosen": -296.2655029296875, |
| "logps/rejected": -268.84454345703125, |
| "loss": 0.684, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": 0.05069383978843689, |
| "rewards/margins": 0.019313272088766098, |
| "rewards/rejected": 0.03138056769967079, |
| "step": 35 |
| }, |
| { |
| "epoch": 0.08442158026645562, |
| "grad_norm": 4.876669460762773, |
| "learning_rate": 4.1666666666666667e-07, |
| "logits/chosen": -1.5888957977294922, |
| "logits/rejected": -1.862489938735962, |
| "logps/chosen": -298.8040466308594, |
| "logps/rejected": -281.5601501464844, |
| "loss": 0.681, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": 0.07323700189590454, |
| "rewards/margins": 0.02310621738433838, |
| "rewards/rejected": 0.05013079196214676, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.09497427779976256, |
| "grad_norm": 4.155679871417378, |
| "learning_rate": 4.6874999999999996e-07, |
| "logits/chosen": -1.6918067932128906, |
| "logits/rejected": -2.00124454498291, |
| "logps/chosen": -278.1552734375, |
| "logps/rejected": -257.6329345703125, |
| "loss": 0.6733, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": 0.06014139577746391, |
| "rewards/margins": 0.035790883004665375, |
| "rewards/rejected": 0.024350514635443687, |
| "step": 45 |
| }, |
| { |
| "epoch": 0.10552697533306951, |
| "grad_norm": 4.856245709560566, |
| "learning_rate": 4.999726797933858e-07, |
| "logits/chosen": -1.7646106481552124, |
| "logits/rejected": -1.9858261346817017, |
| "logps/chosen": -278.0591125488281, |
| "logps/rejected": -259.8578186035156, |
| "loss": 0.6681, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": 0.03362422436475754, |
| "rewards/margins": 0.0627993568778038, |
| "rewards/rejected": -0.029175132513046265, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.11607967286637647, |
| "grad_norm": 5.091156338266151, |
| "learning_rate": 4.99665396039775e-07, |
| "logits/chosen": -1.7584812641143799, |
| "logits/rejected": -2.0758414268493652, |
| "logps/chosen": -275.55548095703125, |
| "logps/rejected": -267.1745300292969, |
| "loss": 0.6557, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -0.016241051256656647, |
| "rewards/margins": 0.09382958710193634, |
| "rewards/rejected": -0.11007064580917358, |
| "step": 55 |
| }, |
| { |
| "epoch": 0.12663237039968342, |
| "grad_norm": 7.261447388148616, |
| "learning_rate": 4.99017099386437e-07, |
| "logits/chosen": -1.8382816314697266, |
| "logits/rejected": -2.1405653953552246, |
| "logps/chosen": -281.54827880859375, |
| "logps/rejected": -263.27294921875, |
| "loss": 0.6545, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.11135590076446533, |
| "rewards/margins": 0.09165789932012558, |
| "rewards/rejected": -0.20301377773284912, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.13718506793299037, |
| "grad_norm": 5.4212641641464705, |
| "learning_rate": 4.980286753286194e-07, |
| "logits/chosen": -1.8312028646469116, |
| "logits/rejected": -2.171030282974243, |
| "logps/chosen": -287.714599609375, |
| "logps/rejected": -269.2692565917969, |
| "loss": 0.6518, |
| "rewards/accuracies": 0.6000000238418579, |
| "rewards/chosen": -0.1507767140865326, |
| "rewards/margins": 0.07915514707565308, |
| "rewards/rejected": -0.22993186116218567, |
| "step": 65 |
| }, |
| { |
| "epoch": 0.14773776546629733, |
| "grad_norm": 5.914411454935479, |
| "learning_rate": 4.967014739346915e-07, |
| "logits/chosen": -1.7997725009918213, |
| "logits/rejected": -2.1724910736083984, |
| "logps/chosen": -314.2709045410156, |
| "logps/rejected": -288.6246337890625, |
| "loss": 0.6402, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -0.12115994840860367, |
| "rewards/margins": 0.11401550471782684, |
| "rewards/rejected": -0.23517544567584991, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.15829046299960428, |
| "grad_norm": 6.712529318578845, |
| "learning_rate": 4.950373080021136e-07, |
| "logits/chosen": -1.7687238454818726, |
| "logits/rejected": -2.1885459423065186, |
| "logps/chosen": -325.1200256347656, |
| "logps/rejected": -298.8721008300781, |
| "loss": 0.6297, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.15646745264530182, |
| "rewards/margins": 0.1352117955684662, |
| "rewards/rejected": -0.2916792631149292, |
| "step": 75 |
| }, |
| { |
| "epoch": 0.16884316053291124, |
| "grad_norm": 11.136062496010506, |
| "learning_rate": 4.930384505813737e-07, |
| "logits/chosen": -1.8805389404296875, |
| "logits/rejected": -2.243736982345581, |
| "logps/chosen": -311.088134765625, |
| "logps/rejected": -294.6890563964844, |
| "loss": 0.6359, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.28947150707244873, |
| "rewards/margins": 0.14183922111988068, |
| "rewards/rejected": -0.4313107430934906, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.1793958580662182, |
| "grad_norm": 9.114819712911686, |
| "learning_rate": 4.907076318712738e-07, |
| "logits/chosen": -1.823948621749878, |
| "logits/rejected": -2.289140462875366, |
| "logps/chosen": -310.9908752441406, |
| "logps/rejected": -296.40277099609375, |
| "loss": 0.6303, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.3205372095108032, |
| "rewards/margins": 0.17761529982089996, |
| "rewards/rejected": -0.49815255403518677, |
| "step": 85 |
| }, |
| { |
| "epoch": 0.18994855559952512, |
| "grad_norm": 6.643890919509507, |
| "learning_rate": 4.88048035489807e-07, |
| "logits/chosen": -1.9663282632827759, |
| "logits/rejected": -2.3473124504089355, |
| "logps/chosen": -305.6136779785156, |
| "logps/rejected": -289.89471435546875, |
| "loss": 0.6202, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -0.31997618079185486, |
| "rewards/margins": 0.13845226168632507, |
| "rewards/rejected": -0.4584284722805023, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.20050125313283207, |
| "grad_norm": 8.184472249085363, |
| "learning_rate": 4.85063294125718e-07, |
| "logits/chosen": -1.9098714590072632, |
| "logits/rejected": -2.2084691524505615, |
| "logps/chosen": -316.2163391113281, |
| "logps/rejected": -309.3311462402344, |
| "loss": 0.6176, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.3992615342140198, |
| "rewards/margins": 0.18614216148853302, |
| "rewards/rejected": -0.5854036211967468, |
| "step": 95 |
| }, |
| { |
| "epoch": 0.21105395066613902, |
| "grad_norm": 8.142734346752789, |
| "learning_rate": 4.817574845766874e-07, |
| "logits/chosen": -2.0933427810668945, |
| "logits/rejected": -2.4379730224609375, |
| "logps/chosen": -331.0671691894531, |
| "logps/rejected": -329.8666076660156, |
| "loss": 0.6142, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -0.5073726773262024, |
| "rewards/margins": 0.2520057260990143, |
| "rewards/rejected": -0.7593784332275391, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.22160664819944598, |
| "grad_norm": 7.833427916953237, |
| "learning_rate": 4.781351221809166e-07, |
| "logits/chosen": -2.1106457710266113, |
| "logits/rejected": -2.3915274143218994, |
| "logps/chosen": -346.46337890625, |
| "logps/rejected": -337.7541198730469, |
| "loss": 0.6124, |
| "rewards/accuracies": 0.668749988079071, |
| "rewards/chosen": -0.5495078563690186, |
| "rewards/margins": 0.17171132564544678, |
| "rewards/rejected": -0.7212191820144653, |
| "step": 105 |
| }, |
| { |
| "epoch": 0.23215934573275293, |
| "grad_norm": 7.733643763374119, |
| "learning_rate": 4.742011546497182e-07, |
| "logits/chosen": -1.9331356287002563, |
| "logits/rejected": -2.2653117179870605, |
| "logps/chosen": -344.3125, |
| "logps/rejected": -331.16632080078125, |
| "loss": 0.6061, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -0.4401538372039795, |
| "rewards/margins": 0.23562327027320862, |
| "rewards/rejected": -0.6757770776748657, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.24271204326605988, |
| "grad_norm": 9.878124133877995, |
| "learning_rate": 4.6996095530953875e-07, |
| "logits/chosen": -2.1782004833221436, |
| "logits/rejected": -2.4763283729553223, |
| "logps/chosen": -324.13983154296875, |
| "logps/rejected": -319.3317565917969, |
| "loss": 0.5985, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -0.5510643720626831, |
| "rewards/margins": 0.20593421161174774, |
| "rewards/rejected": -0.7569986581802368, |
| "step": 115 |
| }, |
| { |
| "epoch": 0.25326474079936684, |
| "grad_norm": 10.19310620269605, |
| "learning_rate": 4.654203157626399e-07, |
| "logits/chosen": -1.9924976825714111, |
| "logits/rejected": -2.3102524280548096, |
| "logps/chosen": -377.19183349609375, |
| "logps/rejected": -375.12823486328125, |
| "loss": 0.5964, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -0.7560365796089172, |
| "rewards/margins": 0.3029988408088684, |
| "rewards/rejected": -1.0590355396270752, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.2638174383326738, |
| "grad_norm": 11.001033426114137, |
| "learning_rate": 4.605854379764673e-07, |
| "logits/chosen": -2.199047565460205, |
| "logits/rejected": -2.5438296794891357, |
| "logps/chosen": -374.6500549316406, |
| "logps/rejected": -363.5611572265625, |
| "loss": 0.5867, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -0.9937089681625366, |
| "rewards/margins": 0.28307586908340454, |
| "rewards/rejected": -1.2767850160598755, |
| "step": 125 |
| }, |
| { |
| "epoch": 0.27437013586598075, |
| "grad_norm": 12.37698546731958, |
| "learning_rate": 4.5546292581250857e-07, |
| "logits/chosen": -2.1308746337890625, |
| "logits/rejected": -2.4864110946655273, |
| "logps/chosen": -400.5669860839844, |
| "logps/rejected": -397.34454345703125, |
| "loss": 0.5933, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -1.2083604335784912, |
| "rewards/margins": 0.26188138127326965, |
| "rewards/rejected": -1.4702417850494385, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.2849228333992877, |
| "grad_norm": 11.055505495866356, |
| "learning_rate": 4.5005977600621275e-07, |
| "logits/chosen": -2.0843119621276855, |
| "logits/rejected": -2.539513111114502, |
| "logps/chosen": -385.6966857910156, |
| "logps/rejected": -379.7543029785156, |
| "loss": 0.5772, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -0.9928609728813171, |
| "rewards/margins": 0.3268323540687561, |
| "rewards/rejected": -1.3196933269500732, |
| "step": 135 |
| }, |
| { |
| "epoch": 0.29547553093259465, |
| "grad_norm": 15.017423788636298, |
| "learning_rate": 4.443833686102919e-07, |
| "logits/chosen": -2.218951940536499, |
| "logits/rejected": -2.4617791175842285, |
| "logps/chosen": -422.04803466796875, |
| "logps/rejected": -423.21685791015625, |
| "loss": 0.5756, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.2504339218139648, |
| "rewards/margins": 0.3247791528701782, |
| "rewards/rejected": -1.575213074684143, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.3060282284659016, |
| "grad_norm": 13.104101042453381, |
| "learning_rate": 4.384414569144561e-07, |
| "logits/chosen": -2.239192485809326, |
| "logits/rejected": -2.4994874000549316, |
| "logps/chosen": -423.623046875, |
| "logps/rejected": -425.60546875, |
| "loss": 0.5866, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.5029816627502441, |
| "rewards/margins": 0.3298465609550476, |
| "rewards/rejected": -1.832828164100647, |
| "step": 145 |
| }, |
| { |
| "epoch": 0.31658092599920856, |
| "grad_norm": 9.51654888826691, |
| "learning_rate": 4.3224215685535287e-07, |
| "logits/chosen": -2.0407261848449707, |
| "logits/rejected": -2.337188720703125, |
| "logps/chosen": -426.2940979003906, |
| "logps/rejected": -424.5220642089844, |
| "loss": 0.5817, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.3242921829223633, |
| "rewards/margins": 0.31421297788619995, |
| "rewards/rejected": -1.638505220413208, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.3271336235325155, |
| "grad_norm": 11.744189119101899, |
| "learning_rate": 4.2579393593117364e-07, |
| "logits/chosen": -2.0881667137145996, |
| "logits/rejected": -2.4598240852355957, |
| "logps/chosen": -373.4230041503906, |
| "logps/rejected": -372.4430847167969, |
| "loss": 0.5648, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -1.0188493728637695, |
| "rewards/margins": 0.29292336106300354, |
| "rewards/rejected": -1.3117727041244507, |
| "step": 155 |
| }, |
| { |
| "epoch": 0.33768632106582247, |
| "grad_norm": 11.411495536339306, |
| "learning_rate": 4.191056016360699e-07, |
| "logits/chosen": -2.1164355278015137, |
| "logits/rejected": -2.3749523162841797, |
| "logps/chosen": -452.0877380371094, |
| "logps/rejected": -475.936767578125, |
| "loss": 0.5657, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.5110199451446533, |
| "rewards/margins": 0.45130714774131775, |
| "rewards/rejected": -1.962327241897583, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.3482390185991294, |
| "grad_norm": 13.015188201271227, |
| "learning_rate": 4.121862894301754e-07, |
| "logits/chosen": -2.0862815380096436, |
| "logits/rejected": -2.4722859859466553, |
| "logps/chosen": -415.59368896484375, |
| "logps/rejected": -414.7337951660156, |
| "loss": 0.5574, |
| "rewards/accuracies": 0.737500011920929, |
| "rewards/chosen": -1.2593889236450195, |
| "rewards/margins": 0.3909255266189575, |
| "rewards/rejected": -1.6503145694732666, |
| "step": 165 |
| }, |
| { |
| "epoch": 0.3587917161324364, |
| "grad_norm": 15.106508196254897, |
| "learning_rate": 4.050454502616667e-07, |
| "logits/chosen": -2.120917797088623, |
| "logits/rejected": -2.3543829917907715, |
| "logps/chosen": -464.19622802734375, |
| "logps/rejected": -488.60675048828125, |
| "loss": 0.5484, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -1.7536499500274658, |
| "rewards/margins": 0.46063175797462463, |
| "rewards/rejected": -2.2142815589904785, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.36934441366574333, |
| "grad_norm": 17.660956835556952, |
| "learning_rate": 3.976928376579047e-07, |
| "logits/chosen": -2.117267608642578, |
| "logits/rejected": -2.336695432662964, |
| "logps/chosen": -491.78216552734375, |
| "logps/rejected": -518.7801513671875, |
| "loss": 0.5229, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -1.9079921245574951, |
| "rewards/margins": 0.5516217350959778, |
| "rewards/rejected": -2.459613800048828, |
| "step": 175 |
| }, |
| { |
| "epoch": 0.37989711119905023, |
| "grad_norm": 14.241710823955074, |
| "learning_rate": 3.9013849440328945e-07, |
| "logits/chosen": -2.169321060180664, |
| "logits/rejected": -2.405425786972046, |
| "logps/chosen": -436.4549865722656, |
| "logps/rejected": -458.5728454589844, |
| "loss": 0.5505, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -1.5903228521347046, |
| "rewards/margins": 0.4581179618835449, |
| "rewards/rejected": -2.048440933227539, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.3904498087323572, |
| "grad_norm": 14.232999562557966, |
| "learning_rate": 3.8239273882202473e-07, |
| "logits/chosen": -2.1749088764190674, |
| "logits/rejected": -2.4840614795684814, |
| "logps/chosen": -479.7809143066406, |
| "logps/rejected": -491.2457580566406, |
| "loss": 0.5578, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -1.8379102945327759, |
| "rewards/margins": 0.42885318398475647, |
| "rewards/rejected": -2.266763210296631, |
| "step": 185 |
| }, |
| { |
| "epoch": 0.40100250626566414, |
| "grad_norm": 16.05324813627352, |
| "learning_rate": 3.7446615068452804e-07, |
| "logits/chosen": -2.2167088985443115, |
| "logits/rejected": -2.5488333702087402, |
| "logps/chosen": -488.9418029785156, |
| "logps/rejected": -518.4141845703125, |
| "loss": 0.5337, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -1.9092267751693726, |
| "rewards/margins": 0.5299785137176514, |
| "rewards/rejected": -2.4392056465148926, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.4115552037989711, |
| "grad_norm": 14.147768615324013, |
| "learning_rate": 3.6636955675673743e-07, |
| "logits/chosen": -2.1767070293426514, |
| "logits/rejected": -2.61075496673584, |
| "logps/chosen": -479.6851501464844, |
| "logps/rejected": -483.32501220703125, |
| "loss": 0.5389, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -1.7908236980438232, |
| "rewards/margins": 0.46493005752563477, |
| "rewards/rejected": -2.255753517150879, |
| "step": 195 |
| }, |
| { |
| "epoch": 0.42210790133227805, |
| "grad_norm": 13.882861374739983, |
| "learning_rate": 3.5811401601205093e-07, |
| "logits/chosen": -2.219057321548462, |
| "logits/rejected": -2.5431442260742188, |
| "logps/chosen": -500.2259826660156, |
| "logps/rejected": -521.2952270507812, |
| "loss": 0.5154, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -1.941454529762268, |
| "rewards/margins": 0.5581260919570923, |
| "rewards/rejected": -2.4995803833007812, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.432660598865585, |
| "grad_norm": 13.10437188597977, |
| "learning_rate": 3.497108045260995e-07, |
| "logits/chosen": -2.3422179222106934, |
| "logits/rejected": -2.6200077533721924, |
| "logps/chosen": -444.88433837890625, |
| "logps/rejected": -463.40582275390625, |
| "loss": 0.5479, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -1.721801996231079, |
| "rewards/margins": 0.46254196763038635, |
| "rewards/rejected": -2.1843440532684326, |
| "step": 205 |
| }, |
| { |
| "epoch": 0.44321329639889195, |
| "grad_norm": 17.090042659489537, |
| "learning_rate": 3.411714000749838e-07, |
| "logits/chosen": -2.2252583503723145, |
| "logits/rejected": -2.598954916000366, |
| "logps/chosen": -467.2124938964844, |
| "logps/rejected": -482.96136474609375, |
| "loss": 0.5295, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -1.7041940689086914, |
| "rewards/margins": 0.48477378487586975, |
| "rewards/rejected": -2.1889679431915283, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.4537659939321989, |
| "grad_norm": 27.344254292887783, |
| "learning_rate": 3.3250746645801287e-07, |
| "logits/chosen": -2.346909523010254, |
| "logits/rejected": -2.5004947185516357, |
| "logps/chosen": -492.0323791503906, |
| "logps/rejected": -510.87847900390625, |
| "loss": 0.5491, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -2.0776052474975586, |
| "rewards/margins": 0.3693740963935852, |
| "rewards/rejected": -2.446979284286499, |
| "step": 215 |
| }, |
| { |
| "epoch": 0.46431869146550586, |
| "grad_norm": 19.808222569935815, |
| "learning_rate": 3.237308375663571e-07, |
| "logits/chosen": -2.291229486465454, |
| "logits/rejected": -2.6437947750091553, |
| "logps/chosen": -470.88909912109375, |
| "logps/rejected": -506.59393310546875, |
| "loss": 0.5365, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -1.9697215557098389, |
| "rewards/margins": 0.6469660997390747, |
| "rewards/rejected": -2.616687774658203, |
| "step": 220 |
| }, |
| { |
| "epoch": 0.4748713889988128, |
| "grad_norm": 18.32709921824934, |
| "learning_rate": 3.148535012193767e-07, |
| "logits/chosen": -2.1904757022857666, |
| "logits/rejected": -2.5518805980682373, |
| "logps/chosen": -459.55987548828125, |
| "logps/rejected": -499.54840087890625, |
| "loss": 0.5194, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -1.6650127172470093, |
| "rewards/margins": 0.6420146226882935, |
| "rewards/rejected": -2.3070271015167236, |
| "step": 225 |
| }, |
| { |
| "epoch": 0.48542408653211977, |
| "grad_norm": 14.548363920921867, |
| "learning_rate": 3.0588758279070183e-07, |
| "logits/chosen": -2.1270744800567627, |
| "logits/rejected": -2.476382255554199, |
| "logps/chosen": -442.04400634765625, |
| "logps/rejected": -464.97735595703125, |
| "loss": 0.5326, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -1.7559592723846436, |
| "rewards/margins": 0.4754490852355957, |
| "rewards/rejected": -2.2314083576202393, |
| "step": 230 |
| }, |
| { |
| "epoch": 0.4959767840654267, |
| "grad_norm": 16.709600354788574, |
| "learning_rate": 2.968453286464312e-07, |
| "logits/chosen": -2.354429244995117, |
| "logits/rejected": -2.5410735607147217, |
| "logps/chosen": -514.9906005859375, |
| "logps/rejected": -569.000732421875, |
| "loss": 0.5457, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -2.1175270080566406, |
| "rewards/margins": 0.6777707934379578, |
| "rewards/rejected": -2.795297861099243, |
| "step": 235 |
| }, |
| { |
| "epoch": 0.5065294815987337, |
| "grad_norm": 14.595717161808087, |
| "learning_rate": 2.8773908941806877e-07, |
| "logits/chosen": -2.191709280014038, |
| "logits/rejected": -2.4795994758605957, |
| "logps/chosen": -513.7542724609375, |
| "logps/rejected": -539.9058837890625, |
| "loss": 0.5283, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -2.063544273376465, |
| "rewards/margins": 0.5612505674362183, |
| "rewards/rejected": -2.6247947216033936, |
| "step": 240 |
| }, |
| { |
| "epoch": 0.5170821791320406, |
| "grad_norm": 27.782416609672772, |
| "learning_rate": 2.785813031330473e-07, |
| "logits/chosen": -2.316455602645874, |
| "logits/rejected": -2.5953054428100586, |
| "logps/chosen": -480.8763122558594, |
| "logps/rejected": -517.4465942382812, |
| "loss": 0.5183, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -1.9154212474822998, |
| "rewards/margins": 0.5959927439689636, |
| "rewards/rejected": -2.5114142894744873, |
| "step": 245 |
| }, |
| { |
| "epoch": 0.5276348766653476, |
| "grad_norm": 19.504383146510033, |
| "learning_rate": 2.693844782258779e-07, |
| "logits/chosen": -2.288198947906494, |
| "logits/rejected": -2.663243293762207, |
| "logps/chosen": -504.8866271972656, |
| "logps/rejected": -565.0271606445312, |
| "loss": 0.4927, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -2.188603639602661, |
| "rewards/margins": 0.93329256772995, |
| "rewards/rejected": -3.121896266937256, |
| "step": 250 |
| }, |
| { |
| "epoch": 0.5381875741986545, |
| "grad_norm": 16.174904226348485, |
| "learning_rate": 2.601611764531342e-07, |
| "logits/chosen": -2.3196043968200684, |
| "logits/rejected": -2.615384578704834, |
| "logps/chosen": -520.6978759765625, |
| "logps/rejected": -568.9720458984375, |
| "loss": 0.5158, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -2.302415370941162, |
| "rewards/margins": 0.6759995222091675, |
| "rewards/rejected": -2.978415012359619, |
| "step": 255 |
| }, |
| { |
| "epoch": 0.5487402717319615, |
| "grad_norm": 13.877722437423808, |
| "learning_rate": 2.5092399573560323e-07, |
| "logits/chosen": -2.2904419898986816, |
| "logits/rejected": -2.6239161491394043, |
| "logps/chosen": -469.6702575683594, |
| "logps/rejected": -495.83917236328125, |
| "loss": 0.5271, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -1.8616193532943726, |
| "rewards/margins": 0.5212319493293762, |
| "rewards/rejected": -2.3828511238098145, |
| "step": 260 |
| }, |
| { |
| "epoch": 0.5592929692652684, |
| "grad_norm": 25.387018559215, |
| "learning_rate": 2.4168555295104124e-07, |
| "logits/chosen": -2.3710436820983887, |
| "logits/rejected": -2.6667098999023438, |
| "logps/chosen": -551.4783325195312, |
| "logps/rejected": -584.6627197265625, |
| "loss": 0.5207, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.6731348037719727, |
| "rewards/margins": 0.6126972436904907, |
| "rewards/rejected": -3.285832166671753, |
| "step": 265 |
| }, |
| { |
| "epoch": 0.5698456667985754, |
| "grad_norm": 21.194954670641433, |
| "learning_rate": 2.3245846670103626e-07, |
| "logits/chosen": -2.339695692062378, |
| "logits/rejected": -2.728651285171509, |
| "logps/chosen": -566.4747314453125, |
| "logps/rejected": -614.0745849609375, |
| "loss": 0.4911, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -2.8559041023254395, |
| "rewards/margins": 0.8008524179458618, |
| "rewards/rejected": -3.656756639480591, |
| "step": 270 |
| }, |
| { |
| "epoch": 0.5803983643318823, |
| "grad_norm": 17.02245510623681, |
| "learning_rate": 2.232553400755159e-07, |
| "logits/chosen": -2.462646007537842, |
| "logits/rejected": -2.7295315265655518, |
| "logps/chosen": -520.3306884765625, |
| "logps/rejected": -550.9982299804688, |
| "loss": 0.515, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.492687940597534, |
| "rewards/margins": 0.6085286140441895, |
| "rewards/rejected": -3.1012163162231445, |
| "step": 275 |
| }, |
| { |
| "epoch": 0.5909510618651893, |
| "grad_norm": 18.57311563682423, |
| "learning_rate": 2.1408874343844294e-07, |
| "logits/chosen": -2.46991229057312, |
| "logits/rejected": -2.839108943939209, |
| "logps/chosen": -564.0499877929688, |
| "logps/rejected": -614.0209350585938, |
| "loss": 0.5184, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -2.8387348651885986, |
| "rewards/margins": 0.748325765132904, |
| "rewards/rejected": -3.5870604515075684, |
| "step": 280 |
| }, |
| { |
| "epoch": 0.6015037593984962, |
| "grad_norm": 18.112658192267443, |
| "learning_rate": 2.049711972582101e-07, |
| "logits/chosen": -2.4610495567321777, |
| "logits/rejected": -2.7717814445495605, |
| "logps/chosen": -595.5016479492188, |
| "logps/rejected": -650.4605712890625, |
| "loss": 0.4974, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -3.076230764389038, |
| "rewards/margins": 0.7762446999549866, |
| "rewards/rejected": -3.852475643157959, |
| "step": 285 |
| }, |
| { |
| "epoch": 0.6120564569318032, |
| "grad_norm": 25.977212495196564, |
| "learning_rate": 1.9591515500618588e-07, |
| "logits/chosen": -2.5490634441375732, |
| "logits/rejected": -2.773324728012085, |
| "logps/chosen": -515.6527099609375, |
| "logps/rejected": -566.9156494140625, |
| "loss": 0.5189, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -2.3954033851623535, |
| "rewards/margins": 0.6492033004760742, |
| "rewards/rejected": -3.0446066856384277, |
| "step": 290 |
| }, |
| { |
| "epoch": 0.6226091544651101, |
| "grad_norm": 16.798443756074835, |
| "learning_rate": 1.8693298614677112e-07, |
| "logits/chosen": -2.400968074798584, |
| "logits/rejected": -2.6868553161621094, |
| "logps/chosen": -556.1025390625, |
| "logps/rejected": -596.8348388671875, |
| "loss": 0.4854, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -2.474238872528076, |
| "rewards/margins": 0.6959460377693176, |
| "rewards/rejected": -3.17018461227417, |
| "step": 295 |
| }, |
| { |
| "epoch": 0.6331618519984171, |
| "grad_norm": 23.81678431486218, |
| "learning_rate": 1.7803695924219814e-07, |
| "logits/chosen": -2.479430675506592, |
| "logits/rejected": -2.7782349586486816, |
| "logps/chosen": -578.2171020507812, |
| "logps/rejected": -651.0504150390625, |
| "loss": 0.4662, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.9857161045074463, |
| "rewards/margins": 0.9989351034164429, |
| "rewards/rejected": -3.9846510887145996, |
| "step": 300 |
| }, |
| { |
| "epoch": 0.643714549531724, |
| "grad_norm": 17.09054147088968, |
| "learning_rate": 1.6923922519515067e-07, |
| "logits/chosen": -2.4572558403015137, |
| "logits/rejected": -2.866284132003784, |
| "logps/chosen": -598.88037109375, |
| "logps/rejected": -641.3062744140625, |
| "loss": 0.5055, |
| "rewards/accuracies": 0.762499988079071, |
| "rewards/chosen": -3.1678709983825684, |
| "rewards/margins": 0.8001095056533813, |
| "rewards/rejected": -3.9679806232452393, |
| "step": 305 |
| }, |
| { |
| "epoch": 0.654267247065031, |
| "grad_norm": 14.492251564068122, |
| "learning_rate": 1.605518006520924e-07, |
| "logits/chosen": -2.3932666778564453, |
| "logits/rejected": -2.6719508171081543, |
| "logps/chosen": -501.3484802246094, |
| "logps/rejected": -544.0150146484375, |
| "loss": 0.5221, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -2.2441365718841553, |
| "rewards/margins": 0.6640299558639526, |
| "rewards/rejected": -2.9081664085388184, |
| "step": 310 |
| }, |
| { |
| "epoch": 0.6648199445983379, |
| "grad_norm": 19.243991902749503, |
| "learning_rate": 1.519865515899731e-07, |
| "logits/chosen": -2.444279432296753, |
| "logits/rejected": -2.6949431896209717, |
| "logps/chosen": -506.01519775390625, |
| "logps/rejected": -542.7237548828125, |
| "loss": 0.5115, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -2.281829357147217, |
| "rewards/margins": 0.6259506940841675, |
| "rewards/rejected": -2.907780170440674, |
| "step": 315 |
| }, |
| { |
| "epoch": 0.6753726421316449, |
| "grad_norm": 27.99539224141589, |
| "learning_rate": 1.4355517710873182e-07, |
| "logits/chosen": -2.5953707695007324, |
| "logits/rejected": -2.877714157104492, |
| "logps/chosen": -571.291015625, |
| "logps/rejected": -615.3301391601562, |
| "loss": 0.5011, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -2.8195748329162598, |
| "rewards/margins": 0.6808874607086182, |
| "rewards/rejected": -3.500462293624878, |
| "step": 320 |
| }, |
| { |
| "epoch": 0.6859253396649518, |
| "grad_norm": 23.88018530349238, |
| "learning_rate": 1.3526919345173318e-07, |
| "logits/chosen": -2.5718350410461426, |
| "logits/rejected": -2.88576078414917, |
| "logps/chosen": -595.6961669921875, |
| "logps/rejected": -665.6595458984375, |
| "loss": 0.4992, |
| "rewards/accuracies": 0.8187500238418579, |
| "rewards/chosen": -3.05544376373291, |
| "rewards/margins": 0.9698305130004883, |
| "rewards/rejected": -4.025274753570557, |
| "step": 325 |
| }, |
| { |
| "epoch": 0.6964780371982588, |
| "grad_norm": 23.60330153062859, |
| "learning_rate": 1.2713991827596443e-07, |
| "logits/chosen": -2.614315986633301, |
| "logits/rejected": -2.894726276397705, |
| "logps/chosen": -562.1041259765625, |
| "logps/rejected": -629.8328857421875, |
| "loss": 0.4933, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -2.73535418510437, |
| "rewards/margins": 0.9043378829956055, |
| "rewards/rejected": -3.6396923065185547, |
| "step": 330 |
| }, |
| { |
| "epoch": 0.7070307347315657, |
| "grad_norm": 18.43994758901315, |
| "learning_rate": 1.191784551934773e-07, |
| "logits/chosen": -2.494032144546509, |
| "logits/rejected": -2.8370561599731445, |
| "logps/chosen": -512.5650024414062, |
| "logps/rejected": -558.193115234375, |
| "loss": 0.4919, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -2.298001766204834, |
| "rewards/margins": 0.7504197955131531, |
| "rewards/rejected": -3.048421859741211, |
| "step": 335 |
| }, |
| { |
| "epoch": 0.7175834322648728, |
| "grad_norm": 19.769138328586244, |
| "learning_rate": 1.1139567860518953e-07, |
| "logits/chosen": -2.399077892303467, |
| "logits/rejected": -2.8016879558563232, |
| "logps/chosen": -532.8413696289062, |
| "logps/rejected": -595.0988159179688, |
| "loss": 0.4698, |
| "rewards/accuracies": 0.84375, |
| "rewards/chosen": -2.4740264415740967, |
| "rewards/margins": 0.9756819009780884, |
| "rewards/rejected": -3.4497084617614746, |
| "step": 340 |
| }, |
| { |
| "epoch": 0.7281361297981797, |
| "grad_norm": 29.121484778204135, |
| "learning_rate": 1.0380221884776128e-07, |
| "logits/chosen": -2.504153251647949, |
| "logits/rejected": -2.826664447784424, |
| "logps/chosen": -588.2379150390625, |
| "logps/rejected": -649.8221435546875, |
| "loss": 0.4541, |
| "rewards/accuracies": 0.7749999761581421, |
| "rewards/chosen": -2.9968106746673584, |
| "rewards/margins": 0.9327837824821472, |
| "rewards/rejected": -3.9295945167541504, |
| "step": 345 |
| }, |
| { |
| "epoch": 0.7386888273314867, |
| "grad_norm": 20.917558525767543, |
| "learning_rate": 9.640844767383405e-08, |
| "logits/chosen": -2.4767587184906006, |
| "logits/rejected": -2.815369129180908, |
| "logps/chosen": -636.3276977539062, |
| "logps/rejected": -681.0283203125, |
| "loss": 0.5234, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -3.6004860401153564, |
| "rewards/margins": 0.719234824180603, |
| "rewards/rejected": -4.31972074508667, |
| "step": 350 |
| }, |
| { |
| "epoch": 0.7492415248647936, |
| "grad_norm": 20.761748981239933, |
| "learning_rate": 8.922446408546378e-08, |
| "logits/chosen": -2.4393577575683594, |
| "logits/rejected": -2.7462494373321533, |
| "logps/chosen": -593.6701049804688, |
| "logps/rejected": -662.8970947265625, |
| "loss": 0.4559, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -3.010227918624878, |
| "rewards/margins": 0.9989708662033081, |
| "rewards/rejected": -4.0091986656188965, |
| "step": 355 |
| }, |
| { |
| "epoch": 0.7597942223981005, |
| "grad_norm": 18.501565997520643, |
| "learning_rate": 8.22600805400994e-08, |
| "logits/chosen": -2.382094144821167, |
| "logits/rejected": -2.714757204055786, |
| "logps/chosen": -528.34033203125, |
| "logps/rejected": -588.7109985351562, |
| "loss": 0.4723, |
| "rewards/accuracies": 0.8062499761581421, |
| "rewards/chosen": -2.4561707973480225, |
| "rewards/margins": 0.8588768243789673, |
| "rewards/rejected": -3.3150477409362793, |
| "step": 360 |
| }, |
| { |
| "epoch": 0.7703469199314075, |
| "grad_norm": 22.596420829881406, |
| "learning_rate": 7.552480954794558e-08, |
| "logits/chosen": -2.496333599090576, |
| "logits/rejected": -2.8438127040863037, |
| "logps/chosen": -587.7208862304688, |
| "logps/rejected": -652.3656005859375, |
| "loss": 0.4838, |
| "rewards/accuracies": 0.768750011920929, |
| "rewards/chosen": -3.0826942920684814, |
| "rewards/margins": 0.9047689437866211, |
| "rewards/rejected": -3.9874634742736816, |
| "step": 365 |
| }, |
| { |
| "epoch": 0.7808996174647144, |
| "grad_norm": 17.83641444640056, |
| "learning_rate": 6.902785067901854e-08, |
| "logits/chosen": -2.5392613410949707, |
| "logits/rejected": -2.8968329429626465, |
| "logps/chosen": -596.1561889648438, |
| "logps/rejected": -664.4248046875, |
| "loss": 0.4774, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -3.162071943283081, |
| "rewards/margins": 0.8737271428108215, |
| "rewards/rejected": -4.035799026489258, |
| "step": 370 |
| }, |
| { |
| "epoch": 0.7914523149980214, |
| "grad_norm": 22.105262436574318, |
| "learning_rate": 6.277807799763973e-08, |
| "logits/chosen": -2.464101552963257, |
| "logits/rejected": -2.823216199874878, |
| "logps/chosen": -605.5325317382812, |
| "logps/rejected": -688.7382202148438, |
| "loss": 0.4821, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -3.242568254470825, |
| "rewards/margins": 1.0074737071990967, |
| "rewards/rejected": -4.250041961669922, |
| "step": 375 |
| }, |
| { |
| "epoch": 0.8020050125313283, |
| "grad_norm": 25.73803952489229, |
| "learning_rate": 5.678402794153145e-08, |
| "logits/chosen": -2.5645461082458496, |
| "logits/rejected": -2.8685081005096436, |
| "logps/chosen": -624.9561767578125, |
| "logps/rejected": -682.5247802734375, |
| "loss": 0.4853, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -3.3231148719787598, |
| "rewards/margins": 0.8197879791259766, |
| "rewards/rejected": -4.142902374267578, |
| "step": 380 |
| }, |
| { |
| "epoch": 0.8125577100646353, |
| "grad_norm": 22.783957458664876, |
| "learning_rate": 5.105388766206969e-08, |
| "logits/chosen": -2.611253261566162, |
| "logits/rejected": -2.8708913326263428, |
| "logps/chosen": -601.1683349609375, |
| "logps/rejected": -657.6758422851562, |
| "loss": 0.4961, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -3.23264741897583, |
| "rewards/margins": 0.7812051773071289, |
| "rewards/rejected": -4.013852119445801, |
| "step": 385 |
| }, |
| { |
| "epoch": 0.8231104075979422, |
| "grad_norm": 17.937740756320142, |
| "learning_rate": 4.5595483841620484e-08, |
| "logits/chosen": -2.585615396499634, |
| "logits/rejected": -2.860517978668213, |
| "logps/chosen": -610.9042358398438, |
| "logps/rejected": -673.6950073242188, |
| "loss": 0.5005, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -3.3800010681152344, |
| "rewards/margins": 0.7753348350524902, |
| "rewards/rejected": -4.155335426330566, |
| "step": 390 |
| }, |
| { |
| "epoch": 0.8336631051312492, |
| "grad_norm": 25.048225079070804, |
| "learning_rate": 4.0416272003232526e-08, |
| "logits/chosen": -2.5495500564575195, |
| "logits/rejected": -2.783395290374756, |
| "logps/chosen": -589.7579956054688, |
| "logps/rejected": -651.603515625, |
| "loss": 0.4634, |
| "rewards/accuracies": 0.793749988079071, |
| "rewards/chosen": -3.1502811908721924, |
| "rewards/margins": 0.8453443646430969, |
| "rewards/rejected": -3.9956252574920654, |
| "step": 395 |
| }, |
| { |
| "epoch": 0.8442158026645561, |
| "grad_norm": 27.936816284901383, |
| "learning_rate": 3.552332632729041e-08, |
| "logits/chosen": -2.5146939754486084, |
| "logits/rejected": -2.804884195327759, |
| "logps/chosen": -594.3411865234375, |
| "logps/rejected": -653.3115844726562, |
| "loss": 0.4978, |
| "rewards/accuracies": 0.824999988079071, |
| "rewards/chosen": -3.1166281700134277, |
| "rewards/margins": 0.8442124128341675, |
| "rewards/rejected": -3.9608407020568848, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.8442158026645561, |
| "eval_logits/chosen": -3.206465244293213, |
| "eval_logits/rejected": -3.0895018577575684, |
| "eval_logps/chosen": -606.6761474609375, |
| "eval_logps/rejected": -664.0686645507812, |
| "eval_loss": 0.6230235695838928, |
| "eval_rewards/accuracies": 0.6370967626571655, |
| "eval_rewards/chosen": -3.440429449081421, |
| "eval_rewards/margins": 0.46332982182502747, |
| "eval_rewards/rejected": -3.903759717941284, |
| "eval_runtime": 145.9837, |
| "eval_samples_per_second": 13.536, |
| "eval_steps_per_second": 0.849, |
| "step": 400 |
| }, |
| { |
| "epoch": 0.8547685001978631, |
| "grad_norm": 19.235680562568444, |
| "learning_rate": 3.092332998903416e-08, |
| "logits/chosen": -2.4855546951293945, |
| "logits/rejected": -2.8127689361572266, |
| "logps/chosen": -608.1304931640625, |
| "logps/rejected": -668.380859375, |
| "loss": 0.44, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -3.3122177124023438, |
| "rewards/margins": 0.8978776931762695, |
| "rewards/rejected": -4.210095405578613, |
| "step": 405 |
| }, |
| { |
| "epoch": 0.86532119773117, |
| "grad_norm": 20.802575196574956, |
| "learning_rate": 2.6622566030146455e-08, |
| "logits/chosen": -2.571362018585205, |
| "logits/rejected": -2.8206756114959717, |
| "logps/chosen": -649.1343994140625, |
| "logps/rejected": -712.3201293945312, |
| "loss": 0.4668, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -3.6075432300567627, |
| "rewards/margins": 0.9334003329277039, |
| "rewards/rejected": -4.5409440994262695, |
| "step": 410 |
| }, |
| { |
| "epoch": 0.875873895264477, |
| "grad_norm": 20.49229366313954, |
| "learning_rate": 2.26269087768734e-08, |
| "logits/chosen": -2.5383522510528564, |
| "logits/rejected": -2.862185478210449, |
| "logps/chosen": -622.9601440429688, |
| "logps/rejected": -697.26904296875, |
| "loss": 0.4639, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -3.579232692718506, |
| "rewards/margins": 0.9947258234024048, |
| "rewards/rejected": -4.573958396911621, |
| "step": 415 |
| }, |
| { |
| "epoch": 0.8864265927977839, |
| "grad_norm": 18.435121377291708, |
| "learning_rate": 1.894181581640106e-08, |
| "logits/chosen": -2.4851062297821045, |
| "logits/rejected": -2.818612813949585, |
| "logps/chosen": -691.8073120117188, |
| "logps/rejected": -776.2567138671875, |
| "loss": 0.428, |
| "rewards/accuracies": 0.8374999761581421, |
| "rewards/chosen": -3.87129545211792, |
| "rewards/margins": 1.1842930316925049, |
| "rewards/rejected": -5.055588722229004, |
| "step": 420 |
| }, |
| { |
| "epoch": 0.8969792903310909, |
| "grad_norm": 29.475224627532654, |
| "learning_rate": 1.5572320542448143e-08, |
| "logits/chosen": -2.510409355163574, |
| "logits/rejected": -2.80336594581604, |
| "logps/chosen": -651.34326171875, |
| "logps/rejected": -712.4142456054688, |
| "loss": 0.494, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -3.6103858947753906, |
| "rewards/margins": 0.8933914303779602, |
| "rewards/rejected": -4.503777027130127, |
| "step": 425 |
| }, |
| { |
| "epoch": 0.9075319878643978, |
| "grad_norm": 25.052350185477973, |
| "learning_rate": 1.2523025280255729e-08, |
| "logits/chosen": -2.5651907920837402, |
| "logits/rejected": -2.870457172393799, |
| "logps/chosen": -678.6126708984375, |
| "logps/rejected": -742.5474853515625, |
| "loss": 0.4623, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -3.7764217853546143, |
| "rewards/margins": 1.0084255933761597, |
| "rewards/rejected": -4.784847259521484, |
| "step": 430 |
| }, |
| { |
| "epoch": 0.9180846853977048, |
| "grad_norm": 20.51875255327418, |
| "learning_rate": 9.798095000364214e-09, |
| "logits/chosen": -2.5898213386535645, |
| "logits/rejected": -2.949827194213867, |
| "logps/chosen": -640.5285034179688, |
| "logps/rejected": -698.768798828125, |
| "loss": 0.5011, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -3.686187744140625, |
| "rewards/margins": 0.8774013519287109, |
| "rewards/rejected": -4.563588619232178, |
| "step": 435 |
| }, |
| { |
| "epoch": 0.9286373829310117, |
| "grad_norm": 20.399922495391955, |
| "learning_rate": 7.401251629764876e-09, |
| "logits/chosen": -2.594036817550659, |
| "logits/rejected": -2.882014274597168, |
| "logps/chosen": -671.9681396484375, |
| "logps/rejected": -730.8321533203125, |
| "loss": 0.4861, |
| "rewards/accuracies": 0.8125, |
| "rewards/chosen": -3.8397529125213623, |
| "rewards/margins": 0.9077512621879578, |
| "rewards/rejected": -4.747504234313965, |
| "step": 440 |
| }, |
| { |
| "epoch": 0.9391900804643187, |
| "grad_norm": 22.999851244619595, |
| "learning_rate": 5.335768968195098e-09, |
| "logits/chosen": -2.593620538711548, |
| "logits/rejected": -2.895954132080078, |
| "logps/chosen": -661.6905517578125, |
| "logps/rejected": -731.1881713867188, |
| "loss": 0.4489, |
| "rewards/accuracies": 0.7875000238418579, |
| "rewards/chosen": -3.763947010040283, |
| "rewards/margins": 0.9344717264175415, |
| "rewards/rejected": -4.698418617248535, |
| "step": 445 |
| }, |
| { |
| "epoch": 0.9497427779976256, |
| "grad_norm": 25.929731545060648, |
| "learning_rate": 3.604468216521883e-09, |
| "logits/chosen": -2.6423192024230957, |
| "logits/rejected": -2.9191346168518066, |
| "logps/chosen": -610.2369995117188, |
| "logps/rejected": -673.9677124023438, |
| "loss": 0.4538, |
| "rewards/accuracies": 0.75, |
| "rewards/chosen": -3.2649269104003906, |
| "rewards/margins": 0.8473002314567566, |
| "rewards/rejected": -4.112226963043213, |
| "step": 450 |
| }, |
| { |
| "epoch": 0.9602954755309326, |
| "grad_norm": 30.30149224906545, |
| "learning_rate": 2.2097141233206884e-09, |
| "logits/chosen": -2.479203462600708, |
| "logits/rejected": -2.7860312461853027, |
| "logps/chosen": -680.89453125, |
| "logps/rejected": -742.8502807617188, |
| "loss": 0.4953, |
| "rewards/accuracies": 0.731249988079071, |
| "rewards/chosen": -3.7775790691375732, |
| "rewards/margins": 0.8955792188644409, |
| "rewards/rejected": -4.673158645629883, |
| "step": 455 |
| }, |
| { |
| "epoch": 0.9708481730642395, |
| "grad_norm": 17.27399919597619, |
| "learning_rate": 1.1534117549133472e-09, |
| "logits/chosen": -2.5051302909851074, |
| "logits/rejected": -2.7565226554870605, |
| "logps/chosen": -642.7161254882812, |
| "logps/rejected": -727.14599609375, |
| "loss": 0.4686, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -3.4852137565612793, |
| "rewards/margins": 1.0831372737884521, |
| "rewards/rejected": -4.5683512687683105, |
| "step": 460 |
| }, |
| { |
| "epoch": 0.9814008705975465, |
| "grad_norm": 22.26013302983421, |
| "learning_rate": 4.3700389327672173e-10, |
| "logits/chosen": -2.3914403915405273, |
| "logits/rejected": -2.711667537689209, |
| "logps/chosen": -663.2670288085938, |
| "logps/rejected": -728.1150512695312, |
| "loss": 0.4779, |
| "rewards/accuracies": 0.831250011920929, |
| "rewards/chosen": -3.666525363922119, |
| "rewards/margins": 0.9401981234550476, |
| "rewards/rejected": -4.606723308563232, |
| "step": 465 |
| }, |
| { |
| "epoch": 0.9919535681308534, |
| "grad_norm": 39.07814619267544, |
| "learning_rate": 6.146906537587982e-11, |
| "logits/chosen": -2.5620739459991455, |
| "logits/rejected": -2.8582262992858887, |
| "logps/chosen": -628.55322265625, |
| "logps/rejected": -687.5040283203125, |
| "loss": 0.4882, |
| "rewards/accuracies": 0.800000011920929, |
| "rewards/chosen": -3.4171409606933594, |
| "rewards/margins": 0.8897517919540405, |
| "rewards/rejected": -4.3068928718566895, |
| "step": 470 |
| }, |
| { |
| "epoch": 0.9982851866508377, |
| "step": 473, |
| "total_flos": 0.0, |
| "train_loss": 0.545083115015171, |
| "train_runtime": 9073.2474, |
| "train_samples_per_second": 6.684, |
| "train_steps_per_second": 0.052 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 473, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 1, |
| "save_steps": 1000000, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
