| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9998738011105502, | |
| "eval_steps": 500, | |
| "global_step": 2641, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 862.3935139973959, | |
| "epoch": 0.0003785966683493185, | |
| "grad_norm": 0.29023680090904236, | |
| "kl": 0.0, | |
| "learning_rate": 7.547169811320756e-08, | |
| "loss": -0.0, | |
| "reward": -2.4199581146240234, | |
| "reward_std": 0.6020505428314209, | |
| "rewards/Qwen2-0.5B-Reward": -2.4199581146240234, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 921.5138956705729, | |
| "epoch": 0.003785966683493185, | |
| "grad_norm": 0.5279305577278137, | |
| "kl": 0.00010103649563259548, | |
| "learning_rate": 7.547169811320755e-07, | |
| "loss": 0.0, | |
| "reward": -2.473096079296536, | |
| "reward_std": 0.5592167631343559, | |
| "rewards/Qwen2-0.5B-Reward": -2.473096079296536, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 910.1000081380208, | |
| "epoch": 0.00757193336698637, | |
| "grad_norm": 0.2346569150686264, | |
| "kl": 0.00012467702229817707, | |
| "learning_rate": 1.509433962264151e-06, | |
| "loss": 0.0, | |
| "reward": -2.422696002324422, | |
| "reward_std": 0.5530827701091766, | |
| "rewards/Qwen2-0.5B-Reward": -2.422696002324422, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 911.8717631022135, | |
| "epoch": 0.011357900050479555, | |
| "grad_norm": 0.21592262387275696, | |
| "kl": 0.0003096898396809896, | |
| "learning_rate": 2.2641509433962266e-06, | |
| "loss": 0.0, | |
| "reward": -2.41411194006602, | |
| "reward_std": 0.5578982929388682, | |
| "rewards/Qwen2-0.5B-Reward": -2.41411194006602, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 893.8527893066406, | |
| "epoch": 0.01514386673397274, | |
| "grad_norm": 0.26798614859580994, | |
| "kl": 0.0015757242838541667, | |
| "learning_rate": 3.018867924528302e-06, | |
| "loss": 0.0001, | |
| "reward": -2.297330105304718, | |
| "reward_std": 0.528485847512881, | |
| "rewards/Qwen2-0.5B-Reward": -2.297330105304718, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 875.3588053385416, | |
| "epoch": 0.018929833417465926, | |
| "grad_norm": 0.27687838673591614, | |
| "kl": 0.0064605712890625, | |
| "learning_rate": 3.7735849056603777e-06, | |
| "loss": 0.0003, | |
| "reward": -2.010285266240438, | |
| "reward_std": 0.5279872556527455, | |
| "rewards/Qwen2-0.5B-Reward": -2.010285266240438, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 859.5717692057292, | |
| "epoch": 0.02271580010095911, | |
| "grad_norm": 0.2615930736064911, | |
| "kl": 0.018147786458333332, | |
| "learning_rate": 4.528301886792453e-06, | |
| "loss": 0.0007, | |
| "reward": -1.795549988746643, | |
| "reward_std": 0.49105457464853924, | |
| "rewards/Qwen2-0.5B-Reward": -1.795549988746643, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 765.7564921061198, | |
| "epoch": 0.026501766784452298, | |
| "grad_norm": 0.28387993574142456, | |
| "kl": 0.028316243489583334, | |
| "learning_rate": 5.283018867924529e-06, | |
| "loss": 0.0011, | |
| "reward": -1.4461613575617471, | |
| "reward_std": 0.47599050005277, | |
| "rewards/Qwen2-0.5B-Reward": -1.4461613575617471, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 783.0041768391927, | |
| "epoch": 0.03028773346794548, | |
| "grad_norm": 0.25285565853118896, | |
| "kl": 0.040238444010416666, | |
| "learning_rate": 6.037735849056604e-06, | |
| "loss": 0.0016, | |
| "reward": -1.193545683224996, | |
| "reward_std": 0.4799055278301239, | |
| "rewards/Qwen2-0.5B-Reward": -1.193545683224996, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 836.7032450358073, | |
| "epoch": 0.034073700151438666, | |
| "grad_norm": 0.25005629658699036, | |
| "kl": 0.05516764322916667, | |
| "learning_rate": 6.792452830188679e-06, | |
| "loss": 0.0022, | |
| "reward": -1.0326486746470134, | |
| "reward_std": 0.5193435788154602, | |
| "rewards/Qwen2-0.5B-Reward": -1.0326486746470134, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 853.6324157714844, | |
| "epoch": 0.03785966683493185, | |
| "grad_norm": 0.3665623664855957, | |
| "kl": 0.10475260416666667, | |
| "learning_rate": 7.5471698113207555e-06, | |
| "loss": 0.0042, | |
| "reward": -0.9854023973147075, | |
| "reward_std": 0.6189069559176763, | |
| "rewards/Qwen2-0.5B-Reward": -0.9854023973147075, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 815.8801005045573, | |
| "epoch": 0.04164563351842504, | |
| "grad_norm": 0.8496055006980896, | |
| "kl": 0.41549479166666664, | |
| "learning_rate": 8.301886792452832e-06, | |
| "loss": 0.0166, | |
| "reward": -1.463372488816579, | |
| "reward_std": 1.0357649803161622, | |
| "rewards/Qwen2-0.5B-Reward": -1.463372488816579, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 813.9148213704427, | |
| "epoch": 0.04543160020191822, | |
| "grad_norm": 0.3407374918460846, | |
| "kl": 0.40042317708333336, | |
| "learning_rate": 9.056603773584907e-06, | |
| "loss": 0.016, | |
| "reward": -1.8741844495137532, | |
| "reward_std": 1.722016990184784, | |
| "rewards/Qwen2-0.5B-Reward": -1.8741844495137532, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 646.899545288086, | |
| "epoch": 0.04921756688541141, | |
| "grad_norm": 0.4906499981880188, | |
| "kl": 0.2925618489583333, | |
| "learning_rate": 9.811320754716981e-06, | |
| "loss": 0.0117, | |
| "reward": -1.3685388286908469, | |
| "reward_std": 1.243816477060318, | |
| "rewards/Qwen2-0.5B-Reward": -1.3685388286908469, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 547.943989054362, | |
| "epoch": 0.053003533568904596, | |
| "grad_norm": 1.3670618534088135, | |
| "kl": 1.1440104166666667, | |
| "learning_rate": 1.0566037735849058e-05, | |
| "loss": 0.0458, | |
| "reward": -2.530128773053487, | |
| "reward_std": 2.079139538606008, | |
| "rewards/Qwen2-0.5B-Reward": -2.530128773053487, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 482.1189860026042, | |
| "epoch": 0.056789500252397776, | |
| "grad_norm": 2.426543712615967, | |
| "kl": 2.05546875, | |
| "learning_rate": 1.1320754716981132e-05, | |
| "loss": 0.0822, | |
| "reward": -3.6518485943476358, | |
| "reward_std": 2.5313418904940286, | |
| "rewards/Qwen2-0.5B-Reward": -3.6518485943476358, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 583.3393595377604, | |
| "epoch": 0.06057546693589096, | |
| "grad_norm": 7.087838649749756, | |
| "kl": 1.375, | |
| "learning_rate": 1.2075471698113209e-05, | |
| "loss": 0.055, | |
| "reward": -2.7873202482859294, | |
| "reward_std": 2.297369889418284, | |
| "rewards/Qwen2-0.5B-Reward": -2.7873202482859294, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 581.331483968099, | |
| "epoch": 0.06436143361938415, | |
| "grad_norm": 0.3393622636795044, | |
| "kl": 0.7234049479166667, | |
| "learning_rate": 1.2830188679245283e-05, | |
| "loss": 0.029, | |
| "reward": -1.6760946492354074, | |
| "reward_std": 1.530751649538676, | |
| "rewards/Qwen2-0.5B-Reward": -1.6760946492354074, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 551.2680633544921, | |
| "epoch": 0.06814740030287733, | |
| "grad_norm": 0.5040144920349121, | |
| "kl": 0.5955729166666667, | |
| "learning_rate": 1.3584905660377358e-05, | |
| "loss": 0.0238, | |
| "reward": -1.6978328824043274, | |
| "reward_std": 1.5175378421942394, | |
| "rewards/Qwen2-0.5B-Reward": -1.6978328824043274, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 498.8856536865234, | |
| "epoch": 0.07193336698637053, | |
| "grad_norm": 0.6744162440299988, | |
| "kl": 1.2225260416666666, | |
| "learning_rate": 1.4339622641509435e-05, | |
| "loss": 0.0489, | |
| "reward": -2.619816021124522, | |
| "reward_std": 2.05845144589742, | |
| "rewards/Qwen2-0.5B-Reward": -2.619816021124522, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 651.0615783691406, | |
| "epoch": 0.0757193336698637, | |
| "grad_norm": 0.47306591272354126, | |
| "kl": 0.9920572916666667, | |
| "learning_rate": 1.5094339622641511e-05, | |
| "loss": 0.0397, | |
| "reward": -2.183918062845866, | |
| "reward_std": 2.0014989256858824, | |
| "rewards/Qwen2-0.5B-Reward": -2.183918062845866, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 723.8722351074218, | |
| "epoch": 0.07950530035335689, | |
| "grad_norm": 0.8630687594413757, | |
| "kl": 1.132421875, | |
| "learning_rate": 1.5849056603773586e-05, | |
| "loss": 0.0453, | |
| "reward": -2.3433102289835612, | |
| "reward_std": 2.1008309284845987, | |
| "rewards/Qwen2-0.5B-Reward": -2.3433102289835612, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 633.7060221354167, | |
| "epoch": 0.08329126703685008, | |
| "grad_norm": 0.33421555161476135, | |
| "kl": 1.11015625, | |
| "learning_rate": 1.6603773584905664e-05, | |
| "loss": 0.0444, | |
| "reward": -1.7007139801979065, | |
| "reward_std": 1.8362650871276855, | |
| "rewards/Qwen2-0.5B-Reward": -1.7007139801979065, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 801.9037150065104, | |
| "epoch": 0.08707723372034326, | |
| "grad_norm": 0.2525235116481781, | |
| "kl": 0.492578125, | |
| "learning_rate": 1.735849056603774e-05, | |
| "loss": 0.0197, | |
| "reward": -0.9283350398143132, | |
| "reward_std": 1.1985284070173898, | |
| "rewards/Qwen2-0.5B-Reward": -0.9283350398143132, | |
| "step": 230 | |
| }, | |
| { | |
| "completion_length": 757.5694498697917, | |
| "epoch": 0.09086320040383644, | |
| "grad_norm": 0.23876462876796722, | |
| "kl": 0.5396484375, | |
| "learning_rate": 1.8113207547169813e-05, | |
| "loss": 0.0216, | |
| "reward": -1.083450937271118, | |
| "reward_std": 1.4640587449073792, | |
| "rewards/Qwen2-0.5B-Reward": -1.083450937271118, | |
| "step": 240 | |
| }, | |
| { | |
| "completion_length": 691.9120452880859, | |
| "epoch": 0.09464916708732964, | |
| "grad_norm": 0.5013711452484131, | |
| "kl": 0.6822265625, | |
| "learning_rate": 1.8867924528301888e-05, | |
| "loss": 0.0273, | |
| "reward": -1.4570284724235534, | |
| "reward_std": 1.626520773768425, | |
| "rewards/Qwen2-0.5B-Reward": -1.4570284724235534, | |
| "step": 250 | |
| }, | |
| { | |
| "completion_length": 573.8166727701823, | |
| "epoch": 0.09843513377082282, | |
| "grad_norm": 0.6115075349807739, | |
| "kl": 1.6170572916666666, | |
| "learning_rate": 1.9622641509433963e-05, | |
| "loss": 0.0647, | |
| "reward": -2.9246065855026244, | |
| "reward_std": 2.250337036450704, | |
| "rewards/Qwen2-0.5B-Reward": -2.9246065855026244, | |
| "step": 260 | |
| }, | |
| { | |
| "completion_length": 773.3074178059895, | |
| "epoch": 0.102221100454316, | |
| "grad_norm": 0.31634387373924255, | |
| "kl": 0.733984375, | |
| "learning_rate": 1.999980332108064e-05, | |
| "loss": 0.0294, | |
| "reward": -1.5115862051645914, | |
| "reward_std": 1.5288376450538634, | |
| "rewards/Qwen2-0.5B-Reward": -1.5115862051645914, | |
| "step": 270 | |
| }, | |
| { | |
| "completion_length": 829.3129659016927, | |
| "epoch": 0.10600706713780919, | |
| "grad_norm": 0.25395989418029785, | |
| "kl": 0.4763671875, | |
| "learning_rate": 1.9998229941302175e-05, | |
| "loss": 0.0191, | |
| "reward": -0.9474448690811793, | |
| "reward_std": 1.09269377887249, | |
| "rewards/Qwen2-0.5B-Reward": -0.9474448690811793, | |
| "step": 280 | |
| }, | |
| { | |
| "completion_length": 999.9018575032552, | |
| "epoch": 0.10979303382130237, | |
| "grad_norm": 0.3452966809272766, | |
| "kl": 0.6498046875, | |
| "learning_rate": 1.9995083456809467e-05, | |
| "loss": 0.026, | |
| "reward": -1.7776759227116903, | |
| "reward_std": 1.8762857417265575, | |
| "rewards/Qwen2-0.5B-Reward": -1.7776759227116903, | |
| "step": 290 | |
| }, | |
| { | |
| "completion_length": 867.9375081380208, | |
| "epoch": 0.11357900050479555, | |
| "grad_norm": 0.4680746793746948, | |
| "kl": 0.98203125, | |
| "learning_rate": 1.9990364417682882e-05, | |
| "loss": 0.0393, | |
| "reward": -2.6815950234731036, | |
| "reward_std": 2.1339449683825173, | |
| "rewards/Qwen2-0.5B-Reward": -2.6815950234731036, | |
| "step": 300 | |
| }, | |
| { | |
| "completion_length": 841.0263916015625, | |
| "epoch": 0.11736496718828875, | |
| "grad_norm": 0.4002317786216736, | |
| "kl": 1.4239583333333334, | |
| "learning_rate": 1.9984073648922753e-05, | |
| "loss": 0.057, | |
| "reward": -2.9415343125661213, | |
| "reward_std": 2.4492496887842816, | |
| "rewards/Qwen2-0.5B-Reward": -2.9415343125661213, | |
| "step": 310 | |
| }, | |
| { | |
| "completion_length": 845.747226969401, | |
| "epoch": 0.12115093387178193, | |
| "grad_norm": 0.7248504757881165, | |
| "kl": 1.8854166666666667, | |
| "learning_rate": 1.997621225030515e-05, | |
| "loss": 0.0754, | |
| "reward": -3.695864470799764, | |
| "reward_std": 2.583825929959615, | |
| "rewards/Qwen2-0.5B-Reward": -3.695864470799764, | |
| "step": 320 | |
| }, | |
| { | |
| "completion_length": 1439.778253173828, | |
| "epoch": 0.12493690055527511, | |
| "grad_norm": 0.602688193321228, | |
| "kl": 1.6190104166666666, | |
| "learning_rate": 1.9966781596189623e-05, | |
| "loss": 0.0648, | |
| "reward": -3.7271327575047812, | |
| "reward_std": 1.9023333628972372, | |
| "rewards/Qwen2-0.5B-Reward": -3.7271327575047812, | |
| "step": 330 | |
| }, | |
| { | |
| "completion_length": 1148.3754781087239, | |
| "epoch": 0.1287228672387683, | |
| "grad_norm": 0.6237585544586182, | |
| "kl": 1.2373697916666666, | |
| "learning_rate": 1.9955783335278924e-05, | |
| "loss": 0.0495, | |
| "reward": -3.0485590934753417, | |
| "reward_std": 1.7622671604156495, | |
| "rewards/Qwen2-0.5B-Reward": -3.0485590934753417, | |
| "step": 340 | |
| }, | |
| { | |
| "completion_length": 871.3398213704427, | |
| "epoch": 0.13250883392226148, | |
| "grad_norm": 0.7727285623550415, | |
| "kl": 0.8674479166666667, | |
| "learning_rate": 1.9943219390330767e-05, | |
| "loss": 0.0347, | |
| "reward": -2.604492497444153, | |
| "reward_std": 1.5047667543093364, | |
| "rewards/Qwen2-0.5B-Reward": -2.604492497444153, | |
| "step": 350 | |
| }, | |
| { | |
| "completion_length": 731.1259358723959, | |
| "epoch": 0.13629480060575466, | |
| "grad_norm": 0.2828836441040039, | |
| "kl": 0.725, | |
| "learning_rate": 1.9929091957821703e-05, | |
| "loss": 0.029, | |
| "reward": -1.8863240122795104, | |
| "reward_std": 1.1796027421951294, | |
| "rewards/Qwen2-0.5B-Reward": -1.8863240122795104, | |
| "step": 360 | |
| }, | |
| { | |
| "completion_length": 593.0722300211588, | |
| "epoch": 0.14008076728924784, | |
| "grad_norm": 0.34980472922325134, | |
| "kl": 0.5994791666666667, | |
| "learning_rate": 1.9913403507563104e-05, | |
| "loss": 0.024, | |
| "reward": -1.5030529995759327, | |
| "reward_std": 0.9978658020496368, | |
| "rewards/Qwen2-0.5B-Reward": -1.5030529995759327, | |
| "step": 370 | |
| }, | |
| { | |
| "completion_length": 574.3398223876953, | |
| "epoch": 0.14386673397274105, | |
| "grad_norm": 0.44906890392303467, | |
| "kl": 0.98515625, | |
| "learning_rate": 1.9896156782269405e-05, | |
| "loss": 0.0394, | |
| "reward": -2.0863842129707337, | |
| "reward_std": 1.7282814304033915, | |
| "rewards/Qwen2-0.5B-Reward": -2.0863842129707337, | |
| "step": 380 | |
| }, | |
| { | |
| "completion_length": 598.7032470703125, | |
| "epoch": 0.14765270065623423, | |
| "grad_norm": 0.42980390787124634, | |
| "kl": 1.1180989583333334, | |
| "learning_rate": 1.9877354797078577e-05, | |
| "loss": 0.0448, | |
| "reward": -1.7375385125478109, | |
| "reward_std": 1.6146510203679403, | |
| "rewards/Qwen2-0.5B-Reward": -1.7375385125478109, | |
| "step": 390 | |
| }, | |
| { | |
| "completion_length": 522.9648213704427, | |
| "epoch": 0.1514386673397274, | |
| "grad_norm": 0.4960784614086151, | |
| "kl": 0.8545572916666667, | |
| "learning_rate": 1.9857000839025043e-05, | |
| "loss": 0.0342, | |
| "reward": -1.5845000902811686, | |
| "reward_std": 1.6743145366509755, | |
| "rewards/Qwen2-0.5B-Reward": -1.5845000902811686, | |
| "step": 400 | |
| }, | |
| { | |
| "completion_length": 650.5805603027344, | |
| "epoch": 0.1552246340232206, | |
| "grad_norm": 3.5647335052490234, | |
| "kl": 0.7625, | |
| "learning_rate": 1.983509846646502e-05, | |
| "loss": 0.0305, | |
| "reward": -1.3111775855223338, | |
| "reward_std": 1.5136757413546245, | |
| "rewards/Qwen2-0.5B-Reward": -1.3111775855223338, | |
| "step": 410 | |
| }, | |
| { | |
| "completion_length": 608.793061319987, | |
| "epoch": 0.15901060070671377, | |
| "grad_norm": 0.7843858599662781, | |
| "kl": 1.0794270833333333, | |
| "learning_rate": 1.9811651508454405e-05, | |
| "loss": 0.0432, | |
| "reward": -1.030318695306778, | |
| "reward_std": 1.3256585756937662, | |
| "rewards/Qwen2-0.5B-Reward": -1.030318695306778, | |
| "step": 420 | |
| }, | |
| { | |
| "completion_length": 717.2574086507161, | |
| "epoch": 0.16279656739020695, | |
| "grad_norm": 1.8327181339263916, | |
| "kl": 1.7510416666666666, | |
| "learning_rate": 1.97866640640794e-05, | |
| "loss": 0.07, | |
| "reward": -1.3865876078605652, | |
| "reward_std": 1.5873213092486063, | |
| "rewards/Qwen2-0.5B-Reward": -1.3865876078605652, | |
| "step": 430 | |
| }, | |
| { | |
| "completion_length": 886.4629699707032, | |
| "epoch": 0.16658253407370016, | |
| "grad_norm": 1.820997714996338, | |
| "kl": 2.492317708333333, | |
| "learning_rate": 1.9760140501739885e-05, | |
| "loss": 0.0997, | |
| "reward": -2.041487044095993, | |
| "reward_std": 1.9687125941117605, | |
| "rewards/Qwen2-0.5B-Reward": -2.041487044095993, | |
| "step": 440 | |
| }, | |
| { | |
| "completion_length": 761.4412068684895, | |
| "epoch": 0.17036850075719334, | |
| "grad_norm": 2.2856929302215576, | |
| "kl": 2.5984375, | |
| "learning_rate": 1.9732085458385706e-05, | |
| "loss": 0.104, | |
| "reward": -1.7194086611270905, | |
| "reward_std": 1.6787285923957824, | |
| "rewards/Qwen2-0.5B-Reward": -1.7194086611270905, | |
| "step": 450 | |
| }, | |
| { | |
| "completion_length": 800.9458414713541, | |
| "epoch": 0.17415446744068652, | |
| "grad_norm": 1.5514496564865112, | |
| "kl": 3.582291666666667, | |
| "learning_rate": 1.9702503838706032e-05, | |
| "loss": 0.1433, | |
| "reward": -1.918267943461736, | |
| "reward_std": 1.798914521932602, | |
| "rewards/Qwen2-0.5B-Reward": -1.918267943461736, | |
| "step": 460 | |
| }, | |
| { | |
| "completion_length": 766.8731547037761, | |
| "epoch": 0.1779404341241797, | |
| "grad_norm": 2.541083812713623, | |
| "kl": 2.43359375, | |
| "learning_rate": 1.9671400814271904e-05, | |
| "loss": 0.0973, | |
| "reward": -1.5191373944282531, | |
| "reward_std": 1.7267815709114074, | |
| "rewards/Qwen2-0.5B-Reward": -1.5191373944282531, | |
| "step": 470 | |
| }, | |
| { | |
| "completion_length": 730.6893575032552, | |
| "epoch": 0.18172640080767288, | |
| "grad_norm": 2.0177841186523438, | |
| "kl": 1.9341145833333333, | |
| "learning_rate": 1.9638781822632117e-05, | |
| "loss": 0.0774, | |
| "reward": -0.9346473336219787, | |
| "reward_std": 1.284997742374738, | |
| "rewards/Qwen2-0.5B-Reward": -0.9346473336219787, | |
| "step": 480 | |
| }, | |
| { | |
| "completion_length": 714.4213033040364, | |
| "epoch": 0.1855123674911661, | |
| "grad_norm": 1.7401411533355713, | |
| "kl": 2.3013020833333333, | |
| "learning_rate": 1.9604652566362604e-05, | |
| "loss": 0.092, | |
| "reward": -0.9350511769453684, | |
| "reward_std": 1.2963847279548646, | |
| "rewards/Qwen2-0.5B-Reward": -0.9350511769453684, | |
| "step": 490 | |
| }, | |
| { | |
| "completion_length": 718.9740814208984, | |
| "epoch": 0.18929833417465927, | |
| "grad_norm": 5.835248947143555, | |
| "kl": 2.794661458333333, | |
| "learning_rate": 1.95690190120695e-05, | |
| "loss": 0.1118, | |
| "reward": -1.0198218444983165, | |
| "reward_std": 1.4708388864994049, | |
| "rewards/Qwen2-0.5B-Reward": -1.0198218444983165, | |
| "step": 500 | |
| }, | |
| { | |
| "completion_length": 626.8421396891276, | |
| "epoch": 0.19308430085815245, | |
| "grad_norm": 2.837407112121582, | |
| "kl": 1.6803385416666667, | |
| "learning_rate": 1.9531887389346016e-05, | |
| "loss": 0.0672, | |
| "reward": -0.6805184543132782, | |
| "reward_std": 1.061421944697698, | |
| "rewards/Qwen2-0.5B-Reward": -0.6805184543132782, | |
| "step": 510 | |
| }, | |
| { | |
| "completion_length": 709.4504659016927, | |
| "epoch": 0.19687026754164563, | |
| "grad_norm": 1.3921163082122803, | |
| "kl": 2.5, | |
| "learning_rate": 1.9493264189683393e-05, | |
| "loss": 0.1, | |
| "reward": -1.0162009666363399, | |
| "reward_std": 1.3732348203659057, | |
| "rewards/Qwen2-0.5B-Reward": -1.0162009666363399, | |
| "step": 520 | |
| }, | |
| { | |
| "completion_length": 734.6662150065105, | |
| "epoch": 0.2006562342251388, | |
| "grad_norm": 0.9199353456497192, | |
| "kl": 2.3385416666666665, | |
| "learning_rate": 1.9453156165336e-05, | |
| "loss": 0.0936, | |
| "reward": -0.9758850524822871, | |
| "reward_std": 1.4469304541746775, | |
| "rewards/Qwen2-0.5B-Reward": -0.9758850524822871, | |
| "step": 530 | |
| }, | |
| { | |
| "completion_length": 685.4245402018229, | |
| "epoch": 0.204442200908632, | |
| "grad_norm": 1.9519050121307373, | |
| "kl": 1.9833333333333334, | |
| "learning_rate": 1.94115703281409e-05, | |
| "loss": 0.0793, | |
| "reward": -0.7073126316070557, | |
| "reward_std": 1.1466901183128357, | |
| "rewards/Qwen2-0.5B-Reward": -0.7073126316070557, | |
| "step": 540 | |
| }, | |
| { | |
| "completion_length": 755.8652852376302, | |
| "epoch": 0.2082281675921252, | |
| "grad_norm": 1.136602520942688, | |
| "kl": 2.7690104166666667, | |
| "learning_rate": 1.9368513948291997e-05, | |
| "loss": 0.1108, | |
| "reward": -1.0485609819491704, | |
| "reward_std": 1.5164429823557535, | |
| "rewards/Qwen2-0.5B-Reward": -1.0485609819491704, | |
| "step": 550 | |
| }, | |
| { | |
| "completion_length": 691.9601928710938, | |
| "epoch": 0.21201413427561838, | |
| "grad_norm": 1.1344228982925415, | |
| "kl": 1.9572916666666667, | |
| "learning_rate": 1.932399455306906e-05, | |
| "loss": 0.0783, | |
| "reward": -0.7905913976331552, | |
| "reward_std": 1.2394062995910644, | |
| "rewards/Qwen2-0.5B-Reward": -0.7905913976331552, | |
| "step": 560 | |
| }, | |
| { | |
| "completion_length": 774.0166727701823, | |
| "epoch": 0.21580010095911156, | |
| "grad_norm": 4.68572998046875, | |
| "kl": 2.8236979166666667, | |
| "learning_rate": 1.9278019925521744e-05, | |
| "loss": 0.113, | |
| "reward": -0.9427557557821273, | |
| "reward_std": 1.4873551627000172, | |
| "rewards/Qwen2-0.5B-Reward": -0.9427557557821273, | |
| "step": 570 | |
| }, | |
| { | |
| "completion_length": 782.1125061035157, | |
| "epoch": 0.21958606764260474, | |
| "grad_norm": 3.2951087951660156, | |
| "kl": 2.81875, | |
| "learning_rate": 1.9230598103108958e-05, | |
| "loss": 0.1127, | |
| "reward": -0.9920766482750575, | |
| "reward_std": 1.5032208581765494, | |
| "rewards/Qwen2-0.5B-Reward": -0.9920766482750575, | |
| "step": 580 | |
| }, | |
| { | |
| "completion_length": 764.4046447753906, | |
| "epoch": 0.22337203432609792, | |
| "grad_norm": 0.7878244519233704, | |
| "kl": 2.4518229166666665, | |
| "learning_rate": 1.9181737376293693e-05, | |
| "loss": 0.0981, | |
| "reward": -0.8713747123877208, | |
| "reward_std": 1.4777807037035624, | |
| "rewards/Qwen2-0.5B-Reward": -0.8713747123877208, | |
| "step": 590 | |
| }, | |
| { | |
| "completion_length": 787.2912129720052, | |
| "epoch": 0.2271580010095911, | |
| "grad_norm": 1.002245306968689, | |
| "kl": 2.3055989583333334, | |
| "learning_rate": 1.9131446287093683e-05, | |
| "loss": 0.0922, | |
| "reward": -0.914855935672919, | |
| "reward_std": 1.4097402195135753, | |
| "rewards/Qwen2-0.5B-Reward": -0.914855935672919, | |
| "step": 600 | |
| }, | |
| { | |
| "completion_length": 853.8175984700521, | |
| "epoch": 0.2309439676930843, | |
| "grad_norm": 1.2771328687667847, | |
| "kl": 3.388802083333333, | |
| "learning_rate": 1.9079733627588042e-05, | |
| "loss": 0.1356, | |
| "reward": -1.4452542603015899, | |
| "reward_std": 1.7754655241966248, | |
| "rewards/Qwen2-0.5B-Reward": -1.4452542603015899, | |
| "step": 610 | |
| }, | |
| { | |
| "completion_length": 742.8106587727865, | |
| "epoch": 0.2347299343765775, | |
| "grad_norm": 1.4351956844329834, | |
| "kl": 2.539322916666667, | |
| "learning_rate": 1.9026608438380195e-05, | |
| "loss": 0.1016, | |
| "reward": -1.0814687182505927, | |
| "reward_std": 1.450120480855306, | |
| "rewards/Qwen2-0.5B-Reward": -1.0814687182505927, | |
| "step": 620 | |
| }, | |
| { | |
| "completion_length": 683.3101959228516, | |
| "epoch": 0.23851590106007067, | |
| "grad_norm": 1.352857232093811, | |
| "kl": 1.84296875, | |
| "learning_rate": 1.897208000701737e-05, | |
| "loss": 0.0737, | |
| "reward": -0.6815965756773948, | |
| "reward_std": 1.0737029949824015, | |
| "rewards/Qwen2-0.5B-Reward": -0.6815965756773948, | |
| "step": 630 | |
| }, | |
| { | |
| "completion_length": 792.8847290039063, | |
| "epoch": 0.24230186774356385, | |
| "grad_norm": 0.9555492997169495, | |
| "kl": 2.507552083333333, | |
| "learning_rate": 1.8916157866366928e-05, | |
| "loss": 0.1003, | |
| "reward": -0.9711129138867061, | |
| "reward_std": 1.5443729062875111, | |
| "rewards/Qwen2-0.5B-Reward": -0.9711129138867061, | |
| "step": 640 | |
| }, | |
| { | |
| "completion_length": 771.8388916015625, | |
| "epoch": 0.24608783442705703, | |
| "grad_norm": 1.100447177886963, | |
| "kl": 2.43984375, | |
| "learning_rate": 1.8858851792949764e-05, | |
| "loss": 0.0976, | |
| "reward": -1.0383977095286052, | |
| "reward_std": 1.4934775571028391, | |
| "rewards/Qwen2-0.5B-Reward": -1.0383977095286052, | |
| "step": 650 | |
| }, | |
| { | |
| "completion_length": 719.6476888020833, | |
| "epoch": 0.24987380111055021, | |
| "grad_norm": 1.0007727146148682, | |
| "kl": 1.8555989583333334, | |
| "learning_rate": 1.880017180523116e-05, | |
| "loss": 0.0742, | |
| "reward": -0.844773971537749, | |
| "reward_std": 1.3539518495400746, | |
| "rewards/Qwen2-0.5B-Reward": -0.844773971537749, | |
| "step": 660 | |
| }, | |
| { | |
| "completion_length": 751.7523234049479, | |
| "epoch": 0.2536597677940434, | |
| "grad_norm": 0.8904104232788086, | |
| "kl": 1.8032552083333333, | |
| "learning_rate": 1.8740128161869308e-05, | |
| "loss": 0.0721, | |
| "reward": -0.6786784966786702, | |
| "reward_std": 1.2231530169645946, | |
| "rewards/Qwen2-0.5B-Reward": -0.6786784966786702, | |
| "step": 670 | |
| }, | |
| { | |
| "completion_length": 732.9828735351563, | |
| "epoch": 0.2574457344775366, | |
| "grad_norm": 0.9944930672645569, | |
| "kl": 2.1743489583333333, | |
| "learning_rate": 1.8678731359921856e-05, | |
| "loss": 0.087, | |
| "reward": -0.6016703399519125, | |
| "reward_std": 1.2204503317674, | |
| "rewards/Qwen2-0.5B-Reward": -0.6016703399519125, | |
| "step": 680 | |
| }, | |
| { | |
| "completion_length": 769.8699157714843, | |
| "epoch": 0.2612317011610298, | |
| "grad_norm": 1.7161378860473633, | |
| "kl": 2.68125, | |
| "learning_rate": 1.8615992133010777e-05, | |
| "loss": 0.1073, | |
| "reward": -0.9773722817500432, | |
| "reward_std": 1.5413507958253225, | |
| "rewards/Qwen2-0.5B-Reward": -0.9773722817500432, | |
| "step": 690 | |
| }, | |
| { | |
| "completion_length": 723.2902893066406, | |
| "epoch": 0.26501766784452296, | |
| "grad_norm": 1.2049915790557861, | |
| "kl": 2.252083333333333, | |
| "learning_rate": 1.855192144944586e-05, | |
| "loss": 0.0901, | |
| "reward": -0.6862340954442819, | |
| "reward_std": 1.3176872313022614, | |
| "rewards/Qwen2-0.5B-Reward": -0.6862340954442819, | |
| "step": 700 | |
| }, | |
| { | |
| "completion_length": 730.842598470052, | |
| "epoch": 0.26880363452801614, | |
| "grad_norm": 1.1353825330734253, | |
| "kl": 2.6614583333333335, | |
| "learning_rate": 1.8486530510307222e-05, | |
| "loss": 0.1064, | |
| "reward": -0.8512504202624162, | |
| "reward_std": 1.4152730743090312, | |
| "rewards/Qwen2-0.5B-Reward": -0.8512504202624162, | |
| "step": 710 | |
| }, | |
| { | |
| "completion_length": 784.4833374023438, | |
| "epoch": 0.2725896012115093, | |
| "grad_norm": 1.1364926099777222, | |
| "kl": 2.8877604166666666, | |
| "learning_rate": 1.8419830747487045e-05, | |
| "loss": 0.1155, | |
| "reward": -1.4028477271397908, | |
| "reward_std": 1.6338281035423279, | |
| "rewards/Qwen2-0.5B-Reward": -1.4028477271397908, | |
| "step": 720 | |
| }, | |
| { | |
| "completion_length": 786.0379679361979, | |
| "epoch": 0.2763755678950025, | |
| "grad_norm": 1.4071515798568726, | |
| "kl": 3.0088541666666666, | |
| "learning_rate": 1.8351833821691053e-05, | |
| "loss": 0.1204, | |
| "reward": -1.2512944350639978, | |
| "reward_std": 1.6677428344885508, | |
| "rewards/Qwen2-0.5B-Reward": -1.2512944350639978, | |
| "step": 730 | |
| }, | |
| { | |
| "completion_length": 807.7277893066406, | |
| "epoch": 0.2801615345784957, | |
| "grad_norm": 1.4424173831939697, | |
| "kl": 3.021354166666667, | |
| "learning_rate": 1.8282551620399917e-05, | |
| "loss": 0.1208, | |
| "reward": -1.225243662794431, | |
| "reward_std": 1.7895207107067108, | |
| "rewards/Qwen2-0.5B-Reward": -1.225243662794431, | |
| "step": 740 | |
| }, | |
| { | |
| "completion_length": 728.170839436849, | |
| "epoch": 0.2839475012619889, | |
| "grad_norm": 0.6950631141662598, | |
| "kl": 2.519661458333333, | |
| "learning_rate": 1.821199625579105e-05, | |
| "loss": 0.1008, | |
| "reward": -0.8639134142082184, | |
| "reward_std": 1.4788370271523794, | |
| "rewards/Qwen2-0.5B-Reward": -0.8639134142082184, | |
| "step": 750 | |
| }, | |
| { | |
| "completion_length": 679.6050984700521, | |
| "epoch": 0.2877334679454821, | |
| "grad_norm": 1.6717815399169922, | |
| "kl": 1.7360677083333333, | |
| "learning_rate": 1.8140180062621117e-05, | |
| "loss": 0.0695, | |
| "reward": -0.46732902062746384, | |
| "reward_std": 0.9378261427084605, | |
| "rewards/Qwen2-0.5B-Reward": -0.46732902062746384, | |
| "step": 760 | |
| }, | |
| { | |
| "completion_length": 783.3986165364583, | |
| "epoch": 0.2915194346289753, | |
| "grad_norm": 1.3388867378234863, | |
| "kl": 2.79609375, | |
| "learning_rate": 1.8067115596069607e-05, | |
| "loss": 0.1118, | |
| "reward": -0.9435359309117, | |
| "reward_std": 1.6089221199353536, | |
| "rewards/Qwen2-0.5B-Reward": -0.9435359309117, | |
| "step": 770 | |
| }, | |
| { | |
| "completion_length": 713.3592651367187, | |
| "epoch": 0.29530540131246846, | |
| "grad_norm": 1.2017817497253418, | |
| "kl": 2.4661458333333335, | |
| "learning_rate": 1.79928156295439e-05, | |
| "loss": 0.0986, | |
| "reward": -0.7846424505114555, | |
| "reward_std": 1.4175224483013154, | |
| "rewards/Qwen2-0.5B-Reward": -0.7846424505114555, | |
| "step": 780 | |
| }, | |
| { | |
| "completion_length": 813.8467631022136, | |
| "epoch": 0.29909136799596164, | |
| "grad_norm": 2.2606418132781982, | |
| "kl": 3.955208333333333, | |
| "learning_rate": 1.7917293152446184e-05, | |
| "loss": 0.1583, | |
| "reward": -1.4304717580477397, | |
| "reward_std": 2.023730218410492, | |
| "rewards/Qwen2-0.5B-Reward": -1.4304717580477397, | |
| "step": 790 | |
| }, | |
| { | |
| "completion_length": 701.6078796386719, | |
| "epoch": 0.3028773346794548, | |
| "grad_norm": 1.5273058414459229, | |
| "kl": 2.3984375, | |
| "learning_rate": 1.784056136790257e-05, | |
| "loss": 0.096, | |
| "reward": -0.7075912684202195, | |
| "reward_std": 1.3393534004688263, | |
| "rewards/Qwen2-0.5B-Reward": -0.7075912684202195, | |
| "step": 800 | |
| }, | |
| { | |
| "completion_length": 709.4273193359375, | |
| "epoch": 0.306663301362948, | |
| "grad_norm": 1.1304354667663574, | |
| "kl": 2.4188802083333334, | |
| "learning_rate": 1.7762633690454897e-05, | |
| "loss": 0.0968, | |
| "reward": -0.6373326261838277, | |
| "reward_std": 1.289098753531774, | |
| "rewards/Qwen2-0.5B-Reward": -0.6373326261838277, | |
| "step": 810 | |
| }, | |
| { | |
| "completion_length": 757.3930643717448, | |
| "epoch": 0.3104492680464412, | |
| "grad_norm": 1.3200254440307617, | |
| "kl": 2.48046875, | |
| "learning_rate": 1.7683523743715538e-05, | |
| "loss": 0.0993, | |
| "reward": -0.8247589614242316, | |
| "reward_std": 1.4155633012453714, | |
| "rewards/Qwen2-0.5B-Reward": -0.8247589614242316, | |
| "step": 820 | |
| }, | |
| { | |
| "completion_length": 697.1213033040365, | |
| "epoch": 0.31423523472993437, | |
| "grad_norm": 0.8467837572097778, | |
| "kl": 2.003515625, | |
| "learning_rate": 1.760324535798567e-05, | |
| "loss": 0.0802, | |
| "reward": -0.4532388661056757, | |
| "reward_std": 1.0981567233800889, | |
| "rewards/Qwen2-0.5B-Reward": -0.4532388661056757, | |
| "step": 830 | |
| }, | |
| { | |
| "completion_length": 780.1504659016927, | |
| "epoch": 0.31802120141342755, | |
| "grad_norm": 596676.75, | |
| "kl": 3364.5799479166667, | |
| "learning_rate": 1.752181256783741e-05, | |
| "loss": 134.4873, | |
| "reward": -0.9652832999825478, | |
| "reward_std": 1.6040133237838745, | |
| "rewards/Qwen2-0.5B-Reward": -0.9652832999825478, | |
| "step": 840 | |
| }, | |
| { | |
| "completion_length": 695.2185282389323, | |
| "epoch": 0.3218071680969207, | |
| "grad_norm": 1.3249794244766235, | |
| "kl": 2.296744791666667, | |
| "learning_rate": 1.7439239609660238e-05, | |
| "loss": 0.0919, | |
| "reward": -0.49953351405759655, | |
| "reward_std": 1.100526017944018, | |
| "rewards/Qwen2-0.5B-Reward": -0.49953351405759655, | |
| "step": 850 | |
| }, | |
| { | |
| "completion_length": 705.1893636067708, | |
| "epoch": 0.3255931347804139, | |
| "grad_norm": 2.2733891010284424, | |
| "kl": 2.476302083333333, | |
| "learning_rate": 1.735554091917214e-05, | |
| "loss": 0.0991, | |
| "reward": -0.7803226565321286, | |
| "reward_std": 1.427873319387436, | |
| "rewards/Qwen2-0.5B-Reward": -0.7803226565321286, | |
| "step": 860 | |
| }, | |
| { | |
| "completion_length": 724.2041748046875, | |
| "epoch": 0.32937910146390714, | |
| "grad_norm": 1.2122727632522583, | |
| "kl": 2.6158854166666665, | |
| "learning_rate": 1.7270731128895896e-05, | |
| "loss": 0.1046, | |
| "reward": -0.9140092690785726, | |
| "reward_std": 1.5725321372350056, | |
| "rewards/Qwen2-0.5B-Reward": -0.9140092690785726, | |
| "step": 870 | |
| }, | |
| { | |
| "completion_length": 736.9453796386719, | |
| "epoch": 0.3331650681474003, | |
| "grad_norm": 0.9501739740371704, | |
| "kl": 2.3080729166666667, | |
| "learning_rate": 1.7184825065600964e-05, | |
| "loss": 0.0923, | |
| "reward": -0.7457656829307476, | |
| "reward_std": 1.343357914686203, | |
| "rewards/Qwen2-0.5B-Reward": -0.7457656829307476, | |
| "step": 880 | |
| }, | |
| { | |
| "completion_length": 785.8842651367188, | |
| "epoch": 0.3369510348308935, | |
| "grad_norm": 0.9159669280052185, | |
| "kl": 2.5815104166666667, | |
| "learning_rate": 1.709783774771141e-05, | |
| "loss": 0.1033, | |
| "reward": -0.7225840290387472, | |
| "reward_std": 1.4536415020624796, | |
| "rewards/Qwen2-0.5B-Reward": -0.7225840290387472, | |
| "step": 890 | |
| }, | |
| { | |
| "completion_length": 824.7926045735677, | |
| "epoch": 0.3407370015143867, | |
| "grad_norm": 3.830165147781372, | |
| "kl": 2.72265625, | |
| "learning_rate": 1.7009784382680345e-05, | |
| "loss": 0.1089, | |
| "reward": -0.9060644646485646, | |
| "reward_std": 1.5053735852241517, | |
| "rewards/Qwen2-0.5B-Reward": -0.9060644646485646, | |
| "step": 900 | |
| }, | |
| { | |
| "completion_length": 788.6736124674479, | |
| "epoch": 0.34452296819787986, | |
| "grad_norm": 1.977720022201538, | |
| "kl": 2.8015625, | |
| "learning_rate": 1.692068036433128e-05, | |
| "loss": 0.1121, | |
| "reward": -0.7987352999548117, | |
| "reward_std": 1.52867697874705, | |
| "rewards/Qwen2-0.5B-Reward": -0.7987352999548117, | |
| "step": 910 | |
| }, | |
| { | |
| "completion_length": 740.3379699707032, | |
| "epoch": 0.34830893488137304, | |
| "grad_norm": 170.5952911376953, | |
| "kl": 2.7221354166666667, | |
| "learning_rate": 1.6830541270166928e-05, | |
| "loss": 0.1088, | |
| "reward": -0.9519633074601491, | |
| "reward_std": 1.5265244921048482, | |
| "rewards/Qwen2-0.5B-Reward": -0.9519633074601491, | |
| "step": 920 | |
| }, | |
| { | |
| "completion_length": 715.4027811686198, | |
| "epoch": 0.3520949015648662, | |
| "grad_norm": 1.736777663230896, | |
| "kl": 2.2143229166666667, | |
| "learning_rate": 1.673938285864588e-05, | |
| "loss": 0.0886, | |
| "reward": -0.5707902121047179, | |
| "reward_std": 1.127177753051122, | |
| "rewards/Qwen2-0.5B-Reward": -0.5707902121047179, | |
| "step": 930 | |
| }, | |
| { | |
| "completion_length": 812.7449137369791, | |
| "epoch": 0.3558808682483594, | |
| "grad_norm": 2.1414971351623535, | |
| "kl": 2.667708333333333, | |
| "learning_rate": 1.664722106642767e-05, | |
| "loss": 0.1066, | |
| "reward": -0.9589705864588419, | |
| "reward_std": 1.527525293827057, | |
| "rewards/Qwen2-0.5B-Reward": -0.9589705864588419, | |
| "step": 940 | |
| }, | |
| { | |
| "completion_length": 769.1416849772136, | |
| "epoch": 0.3596668349318526, | |
| "grad_norm": 4.496264457702637, | |
| "kl": 2.38359375, | |
| "learning_rate": 1.6554072005586638e-05, | |
| "loss": 0.0953, | |
| "reward": -0.5887288892020782, | |
| "reward_std": 1.23007483681043, | |
| "rewards/Qwen2-0.5B-Reward": -0.5887288892020782, | |
| "step": 950 | |
| }, | |
| { | |
| "completion_length": 766.1638936360677, | |
| "epoch": 0.36345280161534577, | |
| "grad_norm": 1.2970997095108032, | |
| "kl": 2.5834635416666667, | |
| "learning_rate": 1.6459951960795185e-05, | |
| "loss": 0.1033, | |
| "reward": -0.7721572608997425, | |
| "reward_std": 1.4835912009080252, | |
| "rewards/Qwen2-0.5B-Reward": -0.7721572608997425, | |
| "step": 960 | |
| }, | |
| { | |
| "completion_length": 751.2708374023438, | |
| "epoch": 0.36723876829883895, | |
| "grad_norm": 2.702152967453003, | |
| "kl": 2.40390625, | |
| "learning_rate": 1.6364877386476804e-05, | |
| "loss": 0.0961, | |
| "reward": -0.7570990284283956, | |
| "reward_std": 1.4351972460746765, | |
| "rewards/Qwen2-0.5B-Reward": -0.7570990284283956, | |
| "step": 970 | |
| }, | |
| { | |
| "completion_length": 730.1676005045573, | |
| "epoch": 0.3710247349823322, | |
| "grad_norm": 1.0240263938903809, | |
| "kl": 2.5716145833333335, | |
| "learning_rate": 1.6268864903929466e-05, | |
| "loss": 0.1029, | |
| "reward": -0.6520452598730723, | |
| "reward_std": 1.3328065713246664, | |
| "rewards/Qwen2-0.5B-Reward": -0.6520452598730723, | |
| "step": 980 | |
| }, | |
| { | |
| "completion_length": 738.0222351074219, | |
| "epoch": 0.37481070166582536, | |
| "grad_norm": 0.9893134832382202, | |
| "kl": 2.990104166666667, | |
| "learning_rate": 1.617193129841982e-05, | |
| "loss": 0.1196, | |
| "reward": -0.973382901151975, | |
| "reward_std": 1.5284679671128591, | |
| "rewards/Qwen2-0.5B-Reward": -0.973382901151975, | |
| "step": 990 | |
| }, | |
| { | |
| "completion_length": 767.7051005045573, | |
| "epoch": 0.37859666834931854, | |
| "grad_norm": 1.4028962850570679, | |
| "kl": 3.0208333333333335, | |
| "learning_rate": 1.6074093516248726e-05, | |
| "loss": 0.1208, | |
| "reward": -0.8820533196131388, | |
| "reward_std": 1.5515558183193208, | |
| "rewards/Qwen2-0.5B-Reward": -0.8820533196131388, | |
| "step": 1000 | |
| }, | |
| { | |
| "completion_length": 729.3245463053386, | |
| "epoch": 0.3823826350328117, | |
| "grad_norm": 1.1494252681732178, | |
| "kl": 2.1536458333333335, | |
| "learning_rate": 1.5975368661788636e-05, | |
| "loss": 0.0861, | |
| "reward": -0.617452886607498, | |
| "reward_std": 1.2075418949127197, | |
| "rewards/Qwen2-0.5B-Reward": -0.617452886607498, | |
| "step": 1010 | |
| }, | |
| { | |
| "completion_length": 711.6662150065105, | |
| "epoch": 0.3861686017163049, | |
| "grad_norm": 0.9261192083358765, | |
| "kl": 2.349739583333333, | |
| "learning_rate": 1.587577399449336e-05, | |
| "loss": 0.094, | |
| "reward": -0.6707314955691497, | |
| "reward_std": 1.2855535586675009, | |
| "rewards/Qwen2-0.5B-Reward": -0.6707314955691497, | |
| "step": 1020 | |
| }, | |
| { | |
| "completion_length": 751.3074096679687, | |
| "epoch": 0.3899545683997981, | |
| "grad_norm": 2.042595148086548, | |
| "kl": 2.3372395833333335, | |
| "learning_rate": 1.5775326925880675e-05, | |
| "loss": 0.0935, | |
| "reward": -0.6637267053127289, | |
| "reward_std": 1.3381904661655426, | |
| "rewards/Qwen2-0.5B-Reward": -0.6637267053127289, | |
| "step": 1030 | |
| }, | |
| { | |
| "completion_length": 776.1870402018229, | |
| "epoch": 0.39374053508329127, | |
| "grad_norm": 1.2383322715759277, | |
| "kl": 5.3609375, | |
| "learning_rate": 1.5674045016488397e-05, | |
| "loss": 0.2142, | |
| "reward": -0.6239150881767273, | |
| "reward_std": 1.3248741805553437, | |
| "rewards/Qwen2-0.5B-Reward": -0.6239150881767273, | |
| "step": 1040 | |
| }, | |
| { | |
| "completion_length": 718.260194905599, | |
| "epoch": 0.39752650176678445, | |
| "grad_norm": 1.5840164422988892, | |
| "kl": 2.3580729166666665, | |
| "learning_rate": 1.5571945972804376e-05, | |
| "loss": 0.0943, | |
| "reward": -0.5199564640720685, | |
| "reward_std": 1.2036932865778605, | |
| "rewards/Qwen2-0.5B-Reward": -0.5199564640720685, | |
| "step": 1050 | |
| }, | |
| { | |
| "completion_length": 809.6148213704427, | |
| "epoch": 0.4013124684502776, | |
| "grad_norm": 1.5066214799880981, | |
| "kl": 3.0403645833333335, | |
| "learning_rate": 1.546904764417098e-05, | |
| "loss": 0.1216, | |
| "reward": -0.9776304622491201, | |
| "reward_std": 1.6650471250216166, | |
| "rewards/Qwen2-0.5B-Reward": -0.9776304622491201, | |
| "step": 1060 | |
| }, | |
| { | |
| "completion_length": 766.8726867675781, | |
| "epoch": 0.4050984351337708, | |
| "grad_norm": 1.4285918474197388, | |
| "kl": 2.3622395833333334, | |
| "learning_rate": 1.5365368019664618e-05, | |
| "loss": 0.0945, | |
| "reward": -0.650248110294342, | |
| "reward_std": 1.3134302516778311, | |
| "rewards/Qwen2-0.5B-Reward": -0.650248110294342, | |
| "step": 1070 | |
| }, | |
| { | |
| "completion_length": 778.1921325683594, | |
| "epoch": 0.408884401817264, | |
| "grad_norm": 1.9540224075317383, | |
| "kl": 2.269270833333333, | |
| "learning_rate": 1.5260925224950785e-05, | |
| "loss": 0.0908, | |
| "reward": -0.5108215274910132, | |
| "reward_std": 1.1806359807650249, | |
| "rewards/Qwen2-0.5B-Reward": -0.5108215274910132, | |
| "step": 1080 | |
| }, | |
| { | |
| "completion_length": 806.8787089029948, | |
| "epoch": 0.41267036850075717, | |
| "grad_norm": 0.9543392062187195, | |
| "kl": 2.60859375, | |
| "learning_rate": 1.5155737519115308e-05, | |
| "loss": 0.1043, | |
| "reward": -0.8536549975474675, | |
| "reward_std": 1.4883501867453257, | |
| "rewards/Qwen2-0.5B-Reward": -0.8536549975474675, | |
| "step": 1090 | |
| }, | |
| { | |
| "completion_length": 790.8611124674479, | |
| "epoch": 0.4164563351842504, | |
| "grad_norm": 1.6240158081054688, | |
| "kl": 2.213671875, | |
| "learning_rate": 1.5049823291472195e-05, | |
| "loss": 0.0885, | |
| "reward": -0.5210499677807092, | |
| "reward_std": 1.2201600551605225, | |
| "rewards/Qwen2-0.5B-Reward": -0.5210499677807092, | |
| "step": 1100 | |
| }, | |
| { | |
| "completion_length": 844.4666748046875, | |
| "epoch": 0.4202423018677436, | |
| "grad_norm": 0.7703062891960144, | |
| "kl": 3.1419270833333335, | |
| "learning_rate": 1.494320105834876e-05, | |
| "loss": 0.1257, | |
| "reward": -1.1577677488327027, | |
| "reward_std": 1.7909785747528075, | |
| "rewards/Qwen2-0.5B-Reward": -1.1577677488327027, | |
| "step": 1110 | |
| }, | |
| { | |
| "completion_length": 873.2398213704427, | |
| "epoch": 0.42402826855123676, | |
| "grad_norm": 1.8059611320495605, | |
| "kl": 3.25859375, | |
| "learning_rate": 1.4835889459848517e-05, | |
| "loss": 0.1304, | |
| "reward": -0.9918207342425982, | |
| "reward_std": 1.6435052702824275, | |
| "rewards/Qwen2-0.5B-Reward": -0.9918207342425982, | |
| "step": 1120 | |
| }, | |
| { | |
| "completion_length": 883.4926025390625, | |
| "epoch": 0.42781423523472994, | |
| "grad_norm": 1.4837961196899414, | |
| "kl": 2.7075520833333333, | |
| "learning_rate": 1.472790725659245e-05, | |
| "loss": 0.1083, | |
| "reward": -0.7034151526788871, | |
| "reward_std": 1.3653341392676035, | |
| "rewards/Qwen2-0.5B-Reward": -0.7034151526788871, | |
| "step": 1130 | |
| }, | |
| { | |
| "completion_length": 779.6824117024739, | |
| "epoch": 0.4316002019182231, | |
| "grad_norm": 1.1727573871612549, | |
| "kl": 2.1869791666666667, | |
| "learning_rate": 1.4619273326439229e-05, | |
| "loss": 0.0875, | |
| "reward": -0.6506599500775337, | |
| "reward_std": 1.3229804019133249, | |
| "rewards/Qwen2-0.5B-Reward": -0.6506599500775337, | |
| "step": 1140 | |
| }, | |
| { | |
| "completion_length": 829.1185241699219, | |
| "epoch": 0.4353861686017163, | |
| "grad_norm": 0.974542498588562, | |
| "kl": 2.659375, | |
| "learning_rate": 1.4510006661184867e-05, | |
| "loss": 0.1064, | |
| "reward": -0.7578525463740031, | |
| "reward_std": 1.531895116964976, | |
| "rewards/Qwen2-0.5B-Reward": -0.7578525463740031, | |
| "step": 1150 | |
| }, | |
| { | |
| "completion_length": 796.5884318033854, | |
| "epoch": 0.4391721352852095, | |
| "grad_norm": 1.2544572353363037, | |
| "kl": 2.2998697916666666, | |
| "learning_rate": 1.440012636324255e-05, | |
| "loss": 0.092, | |
| "reward": -0.6453255646862089, | |
| "reward_std": 1.2682056347529094, | |
| "rewards/Qwen2-0.5B-Reward": -0.6453255646862089, | |
| "step": 1160 | |
| }, | |
| { | |
| "completion_length": 656.2717681884766, | |
| "epoch": 0.44295810196870267, | |
| "grad_norm": 1.7041164636611938, | |
| "kl": 1.9328125, | |
| "learning_rate": 1.4289651642303055e-05, | |
| "loss": 0.0773, | |
| "reward": -0.3406788529828191, | |
| "reward_std": 1.0103827198346456, | |
| "rewards/Qwen2-0.5B-Reward": -0.3406788529828191, | |
| "step": 1170 | |
| }, | |
| { | |
| "completion_length": 733.9171315511068, | |
| "epoch": 0.44674406865219585, | |
| "grad_norm": 0.846507728099823, | |
| "kl": 2.328125, | |
| "learning_rate": 1.4178601811976435e-05, | |
| "loss": 0.0931, | |
| "reward": -0.4902394848565261, | |
| "reward_std": 1.2676184395949046, | |
| "rewards/Qwen2-0.5B-Reward": -0.4902394848565261, | |
| "step": 1180 | |
| }, | |
| { | |
| "completion_length": 768.9398234049479, | |
| "epoch": 0.450530035335689, | |
| "grad_norm": 0.7115055322647095, | |
| "kl": 2.668229166666667, | |
| "learning_rate": 1.4066996286415562e-05, | |
| "loss": 0.1068, | |
| "reward": -0.7519384076197942, | |
| "reward_std": 1.4289092858632406, | |
| "rewards/Qwen2-0.5B-Reward": -0.7519384076197942, | |
| "step": 1190 | |
| }, | |
| { | |
| "completion_length": 825.0884297688802, | |
| "epoch": 0.4543160020191822, | |
| "grad_norm": 1.200706958770752, | |
| "kl": 3.144791666666667, | |
| "learning_rate": 1.3954854576922052e-05, | |
| "loss": 0.1258, | |
| "reward": -1.0908042828241984, | |
| "reward_std": 1.6707689007123312, | |
| "rewards/Qwen2-0.5B-Reward": -1.0908042828241984, | |
| "step": 1200 | |
| }, | |
| { | |
| "completion_length": 705.2310302734375, | |
| "epoch": 0.45810196870267544, | |
| "grad_norm": 1.3045536279678345, | |
| "kl": 2.23046875, | |
| "learning_rate": 1.3842196288535226e-05, | |
| "loss": 0.0893, | |
| "reward": -0.5541289503375689, | |
| "reward_std": 1.264378293355306, | |
| "rewards/Qwen2-0.5B-Reward": -0.5541289503375689, | |
| "step": 1210 | |
| }, | |
| { | |
| "completion_length": 662.7287109375, | |
| "epoch": 0.4618879353861686, | |
| "grad_norm": 1.1240729093551636, | |
| "kl": 1.7548177083333334, | |
| "learning_rate": 1.3729041116604697e-05, | |
| "loss": 0.0702, | |
| "reward": -0.33847450762987136, | |
| "reward_std": 1.030816239118576, | |
| "rewards/Qwen2-0.5B-Reward": -0.33847450762987136, | |
| "step": 1220 | |
| }, | |
| { | |
| "completion_length": 723.0296376546224, | |
| "epoch": 0.4656739020696618, | |
| "grad_norm": 2.3360471725463867, | |
| "kl": 2.3111979166666665, | |
| "learning_rate": 1.3615408843347141e-05, | |
| "loss": 0.0924, | |
| "reward": -0.5807175462444624, | |
| "reward_std": 1.3384559114774068, | |
| "rewards/Qwen2-0.5B-Reward": -0.5807175462444624, | |
| "step": 1230 | |
| }, | |
| { | |
| "completion_length": 751.2430704752604, | |
| "epoch": 0.469459868753155, | |
| "grad_norm": 2.823309898376465, | |
| "kl": 2.7513020833333335, | |
| "learning_rate": 1.3501319334387902e-05, | |
| "loss": 0.1101, | |
| "reward": -0.8531121673683325, | |
| "reward_std": 1.5220951795578004, | |
| "rewards/Qwen2-0.5B-Reward": -0.8531121673683325, | |
| "step": 1240 | |
| }, | |
| { | |
| "completion_length": 790.4842651367187, | |
| "epoch": 0.47324583543664817, | |
| "grad_norm": 1.8123273849487305, | |
| "kl": 3.0010416666666666, | |
| "learning_rate": 1.3386792535287997e-05, | |
| "loss": 0.1201, | |
| "reward": -0.9698835199077924, | |
| "reward_std": 1.6139462788899739, | |
| "rewards/Qwen2-0.5B-Reward": -0.9698835199077924, | |
| "step": 1250 | |
| }, | |
| { | |
| "completion_length": 734.1004720052083, | |
| "epoch": 0.47703180212014135, | |
| "grad_norm": 0.6924867033958435, | |
| "kl": 2.5669270833333333, | |
| "learning_rate": 1.3271848468057176e-05, | |
| "loss": 0.1027, | |
| "reward": -0.6089021896322568, | |
| "reward_std": 1.2572330633799236, | |
| "rewards/Qwen2-0.5B-Reward": -0.6089021896322568, | |
| "step": 1260 | |
| }, | |
| { | |
| "completion_length": 740.8490763346355, | |
| "epoch": 0.4808177688036345, | |
| "grad_norm": 1.0355186462402344, | |
| "kl": 2.7315104166666666, | |
| "learning_rate": 1.3156507227653582e-05, | |
| "loss": 0.1093, | |
| "reward": -0.5665054028232892, | |
| "reward_std": 1.3232530683279038, | |
| "rewards/Qwen2-0.5B-Reward": -0.5665054028232892, | |
| "step": 1270 | |
| }, | |
| { | |
| "completion_length": 721.3925944010417, | |
| "epoch": 0.4846037354871277, | |
| "grad_norm": 1.0751088857650757, | |
| "kl": 2.77890625, | |
| "learning_rate": 1.3040788978470678e-05, | |
| "loss": 0.1111, | |
| "reward": -0.617917682370171, | |
| "reward_std": 1.3952182014783223, | |
| "rewards/Qwen2-0.5B-Reward": -0.617917682370171, | |
| "step": 1280 | |
| }, | |
| { | |
| "completion_length": 743.2884348551432, | |
| "epoch": 0.4883897021706209, | |
| "grad_norm": 1.7289220094680786, | |
| "kl": 2.8721354166666666, | |
| "learning_rate": 1.2924713950812033e-05, | |
| "loss": 0.1148, | |
| "reward": -0.6107141558701793, | |
| "reward_std": 1.3133805135885874, | |
| "rewards/Qwen2-0.5B-Reward": -0.6107141558701793, | |
| "step": 1290 | |
| }, | |
| { | |
| "completion_length": 744.837967936198, | |
| "epoch": 0.49217566885411407, | |
| "grad_norm": 0.9980621337890625, | |
| "kl": 2.6927083333333335, | |
| "learning_rate": 1.280830243735459e-05, | |
| "loss": 0.1077, | |
| "reward": -0.6816005217532317, | |
| "reward_std": 1.3647177835305533, | |
| "rewards/Qwen2-0.5B-Reward": -0.6816005217532317, | |
| "step": 1300 | |
| }, | |
| { | |
| "completion_length": 765.5287109375, | |
| "epoch": 0.49596163553760725, | |
| "grad_norm": 1.5100042819976807, | |
| "kl": 3.23359375, | |
| "learning_rate": 1.2691574789601006e-05, | |
| "loss": 0.1293, | |
| "reward": -0.7456285426393151, | |
| "reward_std": 1.504830890893936, | |
| "rewards/Qwen2-0.5B-Reward": -0.7456285426393151, | |
| "step": 1310 | |
| }, | |
| { | |
| "completion_length": 776.5162089029948, | |
| "epoch": 0.49974760222110043, | |
| "grad_norm": 3.0420119762420654, | |
| "kl": 2.664322916666667, | |
| "learning_rate": 1.2574551414321749e-05, | |
| "loss": 0.1066, | |
| "reward": -0.6133380237966776, | |
| "reward_std": 1.4030099928379058, | |
| "rewards/Qwen2-0.5B-Reward": -0.6133380237966776, | |
| "step": 1320 | |
| }, | |
| { | |
| "completion_length": 756.2375101725261, | |
| "epoch": 0.5035335689045937, | |
| "grad_norm": 1.2776826620101929, | |
| "kl": 2.5111979166666667, | |
| "learning_rate": 1.2457252769987485e-05, | |
| "loss": 0.1005, | |
| "reward": -0.4735676831565797, | |
| "reward_std": 1.2207833151022593, | |
| "rewards/Qwen2-0.5B-Reward": -0.4735676831565797, | |
| "step": 1330 | |
| }, | |
| { | |
| "completion_length": 780.6055643717448, | |
| "epoch": 0.5073195355880868, | |
| "grad_norm": 1.277037262916565, | |
| "kl": 2.29453125, | |
| "learning_rate": 1.2339699363192461e-05, | |
| "loss": 0.0918, | |
| "reward": -0.41186855093886454, | |
| "reward_std": 1.1698833445707957, | |
| "rewards/Qwen2-0.5B-Reward": -0.41186855093886454, | |
| "step": 1340 | |
| }, | |
| { | |
| "completion_length": 814.8995402018229, | |
| "epoch": 0.51110550227158, | |
| "grad_norm": 1.1098392009735107, | |
| "kl": 2.9515625, | |
| "learning_rate": 1.2221911745069473e-05, | |
| "loss": 0.118, | |
| "reward": -0.7255906278888384, | |
| "reward_std": 1.5052427490552267, | |
| "rewards/Qwen2-0.5B-Reward": -0.7255906278888384, | |
| "step": 1350 | |
| }, | |
| { | |
| "completion_length": 800.6180623372396, | |
| "epoch": 0.5148914689550732, | |
| "grad_norm": 1.5379681587219238, | |
| "kl": 3.078385416666667, | |
| "learning_rate": 1.210391050769702e-05, | |
| "loss": 0.1231, | |
| "reward": -0.9011206914981206, | |
| "reward_std": 1.5988249023755392, | |
| "rewards/Qwen2-0.5B-Reward": -0.9011206914981206, | |
| "step": 1360 | |
| }, | |
| { | |
| "completion_length": 787.152783203125, | |
| "epoch": 0.5186774356385664, | |
| "grad_norm": 1.421747088432312, | |
| "kl": 2.74453125, | |
| "learning_rate": 1.1985716280499338e-05, | |
| "loss": 0.1098, | |
| "reward": -0.7614536421994368, | |
| "reward_std": 1.4081373771031698, | |
| "rewards/Qwen2-0.5B-Reward": -0.7614536421994368, | |
| "step": 1370 | |
| }, | |
| { | |
| "completion_length": 842.5861185709635, | |
| "epoch": 0.5224634023220596, | |
| "grad_norm": 2.403327226638794, | |
| "kl": 3.16171875, | |
| "learning_rate": 1.1867349726639868e-05, | |
| "loss": 0.1266, | |
| "reward": -0.8059929932157198, | |
| "reward_std": 1.487107406059901, | |
| "rewards/Qwen2-0.5B-Reward": -0.8059929932157198, | |
| "step": 1380 | |
| }, | |
| { | |
| "completion_length": 793.2569539388021, | |
| "epoch": 0.5262493690055527, | |
| "grad_norm": 1.0243574380874634, | |
| "kl": 3.30625, | |
| "learning_rate": 1.1748831539408863e-05, | |
| "loss": 0.1323, | |
| "reward": -0.9990609556436538, | |
| "reward_std": 1.641613002618154, | |
| "rewards/Qwen2-0.5B-Reward": -0.9990609556436538, | |
| "step": 1390 | |
| }, | |
| { | |
| "completion_length": 781.2513997395833, | |
| "epoch": 0.5300353356890459, | |
| "grad_norm": 1.4023561477661133, | |
| "kl": 2.6401041666666667, | |
| "learning_rate": 1.1630182438605688e-05, | |
| "loss": 0.1056, | |
| "reward": -0.73541273077329, | |
| "reward_std": 1.391848737001419, | |
| "rewards/Qwen2-0.5B-Reward": -0.73541273077329, | |
| "step": 1400 | |
| }, | |
| { | |
| "completion_length": 777.1314880371094, | |
| "epoch": 0.5338213023725391, | |
| "grad_norm": 1.4984385967254639, | |
| "kl": 3.1786458333333334, | |
| "learning_rate": 1.151142316691652e-05, | |
| "loss": 0.1273, | |
| "reward": -0.9620630964636803, | |
| "reward_std": 1.6180862605571746, | |
| "rewards/Qwen2-0.5B-Reward": -0.9620630964636803, | |
| "step": 1410 | |
| }, | |
| { | |
| "completion_length": 753.6296305338542, | |
| "epoch": 0.5376072690560323, | |
| "grad_norm": 0.6080305576324463, | |
| "kl": 2.93046875, | |
| "learning_rate": 1.1392574486288026e-05, | |
| "loss": 0.1172, | |
| "reward": -0.6871781093068421, | |
| "reward_std": 1.4368105371793112, | |
| "rewards/Qwen2-0.5B-Reward": -0.6871781093068421, | |
| "step": 1420 | |
| }, | |
| { | |
| "completion_length": 755.0680684407552, | |
| "epoch": 0.5413932357395255, | |
| "grad_norm": 0.9181307554244995, | |
| "kl": 2.5361979166666666, | |
| "learning_rate": 1.1273657174297687e-05, | |
| "loss": 0.1016, | |
| "reward": -0.41866928230350214, | |
| "reward_std": 1.193355711301168, | |
| "rewards/Qwen2-0.5B-Reward": -0.41866928230350214, | |
| "step": 1430 | |
| }, | |
| { | |
| "completion_length": 739.3643595377604, | |
| "epoch": 0.5451792024230186, | |
| "grad_norm": 1.3852412700653076, | |
| "kl": 2.4328125, | |
| "learning_rate": 1.1154692020521379e-05, | |
| "loss": 0.0973, | |
| "reward": -0.42044620849192144, | |
| "reward_std": 1.1699665983517964, | |
| "rewards/Qwen2-0.5B-Reward": -0.42044620849192144, | |
| "step": 1440 | |
| }, | |
| { | |
| "completion_length": 787.2263997395834, | |
| "epoch": 0.5489651691065118, | |
| "grad_norm": 1.2610223293304443, | |
| "kl": 2.7135416666666665, | |
| "learning_rate": 1.1035699822898852e-05, | |
| "loss": 0.1085, | |
| "reward": -0.5719452144578099, | |
| "reward_std": 1.3674102127552032, | |
| "rewards/Qwen2-0.5B-Reward": -0.5719452144578099, | |
| "step": 1450 | |
| }, | |
| { | |
| "completion_length": 794.1407450358073, | |
| "epoch": 0.552751135790005, | |
| "grad_norm": 3.987548351287842, | |
| "kl": 3.580208333333333, | |
| "learning_rate": 1.091670138409778e-05, | |
| "loss": 0.1432, | |
| "reward": -0.913334188858668, | |
| "reward_std": 1.654043678442637, | |
| "rewards/Qwen2-0.5B-Reward": -0.913334188858668, | |
| "step": 1460 | |
| }, | |
| { | |
| "completion_length": 755.6018575032552, | |
| "epoch": 0.5565371024734982, | |
| "grad_norm": 1.312009334564209, | |
| "kl": 2.115364583333333, | |
| "learning_rate": 1.0797717507876926e-05, | |
| "loss": 0.0846, | |
| "reward": -0.605161217538019, | |
| "reward_std": 1.230643669764201, | |
| "rewards/Qwen2-0.5B-Reward": -0.605161217538019, | |
| "step": 1470 | |
| }, | |
| { | |
| "completion_length": 745.5069519042969, | |
| "epoch": 0.5603230691569914, | |
| "grad_norm": 1.5958776473999023, | |
| "kl": 2.8216145833333335, | |
| "learning_rate": 1.0678768995449179e-05, | |
| "loss": 0.1129, | |
| "reward": -0.5114948400606711, | |
| "reward_std": 1.213375515739123, | |
| "rewards/Qwen2-0.5B-Reward": -0.5114948400606711, | |
| "step": 1480 | |
| }, | |
| { | |
| "completion_length": 790.2676025390625, | |
| "epoch": 0.5641090358404846, | |
| "grad_norm": 1.0892456769943237, | |
| "kl": 2.9091145833333334, | |
| "learning_rate": 1.055987664184499e-05, | |
| "loss": 0.1164, | |
| "reward": -0.6985714793205261, | |
| "reward_std": 1.441979839404424, | |
| "rewards/Qwen2-0.5B-Reward": -0.6985714793205261, | |
| "step": 1490 | |
| }, | |
| { | |
| "completion_length": 768.1861206054688, | |
| "epoch": 0.5678950025239778, | |
| "grad_norm": 1.5841772556304932, | |
| "kl": 2.4014322916666666, | |
| "learning_rate": 1.0441061232276914e-05, | |
| "loss": 0.096, | |
| "reward": -0.5361925270253172, | |
| "reward_std": 1.2279207597176234, | |
| "rewards/Qwen2-0.5B-Reward": -0.5361925270253172, | |
| "step": 1500 | |
| }, | |
| { | |
| "completion_length": 790.4763977050782, | |
| "epoch": 0.571680969207471, | |
| "grad_norm": 1.4919512271881104, | |
| "kl": 2.945052083333333, | |
| "learning_rate": 1.0322343538505859e-05, | |
| "loss": 0.1178, | |
| "reward": -0.6917820642391841, | |
| "reward_std": 1.41629096865654, | |
| "rewards/Qwen2-0.5B-Reward": -0.6917820642391841, | |
| "step": 1510 | |
| }, | |
| { | |
| "completion_length": 732.1388956705729, | |
| "epoch": 0.5754669358909642, | |
| "grad_norm": 1.3066332340240479, | |
| "kl": 2.5669270833333333, | |
| "learning_rate": 1.0203744315209683e-05, | |
| "loss": 0.1026, | |
| "reward": -0.4832228126314779, | |
| "reward_std": 1.21365185379982, | |
| "rewards/Qwen2-0.5B-Reward": -0.4832228126314779, | |
| "step": 1520 | |
| }, | |
| { | |
| "completion_length": 777.1152811686198, | |
| "epoch": 0.5792529025744574, | |
| "grad_norm": 2.0675883293151855, | |
| "kl": 3.0052083333333335, | |
| "learning_rate": 1.0085284296354784e-05, | |
| "loss": 0.1202, | |
| "reward": -0.7202197993795078, | |
| "reward_std": 1.480885813633601, | |
| "rewards/Qwen2-0.5B-Reward": -0.7202197993795078, | |
| "step": 1530 | |
| }, | |
| { | |
| "completion_length": 810.0231577555338, | |
| "epoch": 0.5830388692579506, | |
| "grad_norm": 1.1669964790344238, | |
| "kl": 3.351822916666667, | |
| "learning_rate": 9.966984191571318e-06, | |
| "loss": 0.1341, | |
| "reward": -0.9308211114102354, | |
| "reward_std": 1.5289963026841482, | |
| "rewards/Qwen2-0.5B-Reward": -0.9308211114102354, | |
| "step": 1540 | |
| }, | |
| { | |
| "completion_length": 803.5157470703125, | |
| "epoch": 0.5868248359414437, | |
| "grad_norm": 1.2970937490463257, | |
| "kl": 2.9263020833333333, | |
| "learning_rate": 9.848864682532654e-06, | |
| "loss": 0.1171, | |
| "reward": -0.897743321955204, | |
| "reward_std": 1.4250325242678323, | |
| "rewards/Qwen2-0.5B-Reward": -0.897743321955204, | |
| "step": 1550 | |
| }, | |
| { | |
| "completion_length": 774.900467936198, | |
| "epoch": 0.5906108026249369, | |
| "grad_norm": 1.5224976539611816, | |
| "kl": 3.23203125, | |
| "learning_rate": 9.730946419339721e-06, | |
| "loss": 0.1293, | |
| "reward": -0.8313487897316615, | |
| "reward_std": 1.4089517414569854, | |
| "rewards/Qwen2-0.5B-Reward": -0.8313487897316615, | |
| "step": 1560 | |
| }, | |
| { | |
| "completion_length": 814.4111185709636, | |
| "epoch": 0.5943967693084301, | |
| "grad_norm": 1.5672080516815186, | |
| "kl": 2.9359375, | |
| "learning_rate": 9.613250016910894e-06, | |
| "loss": 0.1174, | |
| "reward": -0.7221511860688528, | |
| "reward_std": 1.3432387212912242, | |
| "rewards/Qwen2-0.5B-Reward": -0.7221511860688528, | |
| "step": 1570 | |
| }, | |
| { | |
| "completion_length": 776.6129638671875, | |
| "epoch": 0.5981827359919233, | |
| "grad_norm": 1.8100062608718872, | |
| "kl": 2.7890625, | |
| "learning_rate": 9.495796051377997e-06, | |
| "loss": 0.1115, | |
| "reward": -0.8584653136630853, | |
| "reward_std": 1.3234432935714722, | |
| "rewards/Qwen2-0.5B-Reward": -0.8584653136630853, | |
| "step": 1580 | |
| }, | |
| { | |
| "completion_length": 825.8180603027344, | |
| "epoch": 0.6019687026754165, | |
| "grad_norm": 1.6404787302017212, | |
| "kl": 3.863541666666667, | |
| "learning_rate": 9.378605056489128e-06, | |
| "loss": 0.1545, | |
| "reward": -1.263607233762741, | |
| "reward_std": 1.8019790093104044, | |
| "rewards/Qwen2-0.5B-Reward": -1.263607233762741, | |
| "step": 1590 | |
| }, | |
| { | |
| "completion_length": 728.1912089029948, | |
| "epoch": 0.6057546693589096, | |
| "grad_norm": 0.8878143429756165, | |
| "kl": 2.6088541666666667, | |
| "learning_rate": 9.261697520018849e-06, | |
| "loss": 0.1044, | |
| "reward": -0.42785762051741283, | |
| "reward_std": 1.0820347189903259, | |
| "rewards/Qwen2-0.5B-Reward": -0.42785762051741283, | |
| "step": 1600 | |
| }, | |
| { | |
| "completion_length": 747.9509358723958, | |
| "epoch": 0.6095406360424028, | |
| "grad_norm": 1.613976240158081, | |
| "kl": 2.468489583333333, | |
| "learning_rate": 9.145093880186451e-06, | |
| "loss": 0.0988, | |
| "reward": -0.41555683029194673, | |
| "reward_std": 1.179705987373988, | |
| "rewards/Qwen2-0.5B-Reward": -0.41555683029194673, | |
| "step": 1610 | |
| }, | |
| { | |
| "completion_length": 787.43056640625, | |
| "epoch": 0.613326602725896, | |
| "grad_norm": 0.5864226818084717, | |
| "kl": 2.894270833333333, | |
| "learning_rate": 9.028814522082857e-06, | |
| "loss": 0.1157, | |
| "reward": -0.6661467840274174, | |
| "reward_std": 1.412223219871521, | |
| "rewards/Qwen2-0.5B-Reward": -0.6661467840274174, | |
| "step": 1620 | |
| }, | |
| { | |
| "completion_length": 742.3319498697916, | |
| "epoch": 0.6171125694093892, | |
| "grad_norm": 1.7149267196655273, | |
| "kl": 2.7528645833333334, | |
| "learning_rate": 8.912879774106832e-06, | |
| "loss": 0.1101, | |
| "reward": -0.560060964524746, | |
| "reward_std": 1.2752733170986175, | |
| "rewards/Qwen2-0.5B-Reward": -0.560060964524746, | |
| "step": 1630 | |
| }, | |
| { | |
| "completion_length": 750.7245422363281, | |
| "epoch": 0.6208985360928824, | |
| "grad_norm": 2.106180191040039, | |
| "kl": 2.40546875, | |
| "learning_rate": 8.797309904411087e-06, | |
| "loss": 0.0962, | |
| "reward": -0.416633996165668, | |
| "reward_std": 1.1659721612930298, | |
| "rewards/Qwen2-0.5B-Reward": -0.416633996165668, | |
| "step": 1640 | |
| }, | |
| { | |
| "completion_length": 778.6088033040364, | |
| "epoch": 0.6246845027763755, | |
| "grad_norm": 1.4638694524765015, | |
| "kl": 2.676041666666667, | |
| "learning_rate": 8.682125117358927e-06, | |
| "loss": 0.1071, | |
| "reward": -0.6446437170108159, | |
| "reward_std": 1.3279209415117899, | |
| "rewards/Qwen2-0.5B-Reward": -0.6446437170108159, | |
| "step": 1650 | |
| }, | |
| { | |
| "completion_length": 808.040283203125, | |
| "epoch": 0.6284704694598687, | |
| "grad_norm": 1.1022939682006836, | |
| "kl": 3.4580729166666666, | |
| "learning_rate": 8.567345549992045e-06, | |
| "loss": 0.1383, | |
| "reward": -0.7954719786842664, | |
| "reward_std": 1.4967798054218293, | |
| "rewards/Qwen2-0.5B-Reward": -0.7954719786842664, | |
| "step": 1660 | |
| }, | |
| { | |
| "completion_length": 757.4060282389323, | |
| "epoch": 0.6322564361433619, | |
| "grad_norm": 2.4723708629608154, | |
| "kl": 2.792708333333333, | |
| "learning_rate": 8.4529912685101e-06, | |
| "loss": 0.1117, | |
| "reward": -0.5523949672778448, | |
| "reward_std": 1.3249893307685852, | |
| "rewards/Qwen2-0.5B-Reward": -0.5523949672778448, | |
| "step": 1670 | |
| }, | |
| { | |
| "completion_length": 762.1398213704427, | |
| "epoch": 0.6360424028268551, | |
| "grad_norm": 0.8709607720375061, | |
| "kl": 2.8286458333333333, | |
| "learning_rate": 8.33908226476265e-06, | |
| "loss": 0.1132, | |
| "reward": -0.5545504409819841, | |
| "reward_std": 1.3114221652348836, | |
| "rewards/Qwen2-0.5B-Reward": -0.5545504409819841, | |
| "step": 1680 | |
| }, | |
| { | |
| "completion_length": 823.0356526692708, | |
| "epoch": 0.6398283695103483, | |
| "grad_norm": 0.969098687171936, | |
| "kl": 2.855729166666667, | |
| "learning_rate": 8.22563845275411e-06, | |
| "loss": 0.1142, | |
| "reward": -0.7070573056737582, | |
| "reward_std": 1.3873663266499838, | |
| "rewards/Qwen2-0.5B-Reward": -0.7070573056737582, | |
| "step": 1690 | |
| }, | |
| { | |
| "completion_length": 810.1981506347656, | |
| "epoch": 0.6436143361938415, | |
| "grad_norm": 1.2305635213851929, | |
| "kl": 3.793229166666667, | |
| "learning_rate": 8.11267966516231e-06, | |
| "loss": 0.1518, | |
| "reward": -1.061463608344396, | |
| "reward_std": 1.7348846475283304, | |
| "rewards/Qwen2-0.5B-Reward": -1.061463608344396, | |
| "step": 1700 | |
| }, | |
| { | |
| "completion_length": 776.243983968099, | |
| "epoch": 0.6474003028773346, | |
| "grad_norm": 1.6688897609710693, | |
| "kl": 2.94375, | |
| "learning_rate": 8.000225649871272e-06, | |
| "loss": 0.1177, | |
| "reward": -0.7328139250477155, | |
| "reward_std": 1.4019733607769012, | |
| "rewards/Qwen2-0.5B-Reward": -0.7328139250477155, | |
| "step": 1710 | |
| }, | |
| { | |
| "completion_length": 782.6092692057292, | |
| "epoch": 0.6511862695608278, | |
| "grad_norm": 2.184279680252075, | |
| "kl": 3.275260416666667, | |
| "learning_rate": 7.888296066518806e-06, | |
| "loss": 0.131, | |
| "reward": -0.826190093656381, | |
| "reward_std": 1.539618053038915, | |
| "rewards/Qwen2-0.5B-Reward": -0.826190093656381, | |
| "step": 1720 | |
| }, | |
| { | |
| "completion_length": 707.3263997395833, | |
| "epoch": 0.6549722362443211, | |
| "grad_norm": 2.3973989486694336, | |
| "kl": 2.400260416666667, | |
| "learning_rate": 7.776910483059543e-06, | |
| "loss": 0.096, | |
| "reward": -0.5184978457788626, | |
| "reward_std": 1.1560731967290243, | |
| "rewards/Qwen2-0.5B-Reward": -0.5184978457788626, | |
| "step": 1730 | |
| }, | |
| { | |
| "completion_length": 734.9152872721354, | |
| "epoch": 0.6587582029278143, | |
| "grad_norm": 1.8029112815856934, | |
| "kl": 2.9859375, | |
| "learning_rate": 7.666088372343984e-06, | |
| "loss": 0.1194, | |
| "reward": -0.5925529218278826, | |
| "reward_std": 1.267720968524615, | |
| "rewards/Qwen2-0.5B-Reward": -0.5925529218278826, | |
| "step": 1740 | |
| }, | |
| { | |
| "completion_length": 807.6726928710938, | |
| "epoch": 0.6625441696113075, | |
| "grad_norm": 1.5247033834457397, | |
| "kl": 3.3872395833333333, | |
| "learning_rate": 7.555849108714192e-06, | |
| "loss": 0.1355, | |
| "reward": -0.7715960969527562, | |
| "reward_std": 1.4897764484087626, | |
| "rewards/Qwen2-0.5B-Reward": -0.7715960969527562, | |
| "step": 1750 | |
| }, | |
| { | |
| "completion_length": 776.8838073730469, | |
| "epoch": 0.6663301362948006, | |
| "grad_norm": 1.9940361976623535, | |
| "kl": 2.837760416666667, | |
| "learning_rate": 7.4462119646166855e-06, | |
| "loss": 0.1136, | |
| "reward": -0.7241511250535647, | |
| "reward_std": 1.4011840164661407, | |
| "rewards/Qwen2-0.5B-Reward": -0.7241511250535647, | |
| "step": 1760 | |
| }, | |
| { | |
| "completion_length": 767.8162129720052, | |
| "epoch": 0.6701161029782938, | |
| "grad_norm": 1.5367672443389893, | |
| "kl": 3.5140625, | |
| "learning_rate": 7.337196107233155e-06, | |
| "loss": 0.1407, | |
| "reward": -0.7663616319497426, | |
| "reward_std": 1.5210982898871104, | |
| "rewards/Qwen2-0.5B-Reward": -0.7663616319497426, | |
| "step": 1770 | |
| }, | |
| { | |
| "completion_length": 721.7675944010417, | |
| "epoch": 0.673902069661787, | |
| "grad_norm": 1.302241563796997, | |
| "kl": 2.931510416666667, | |
| "learning_rate": 7.228820595129604e-06, | |
| "loss": 0.1172, | |
| "reward": -0.725257391979297, | |
| "reward_std": 1.334197594722112, | |
| "rewards/Qwen2-0.5B-Reward": -0.725257391979297, | |
| "step": 1780 | |
| }, | |
| { | |
| "completion_length": 720.3171468098958, | |
| "epoch": 0.6776880363452802, | |
| "grad_norm": 0.8652080297470093, | |
| "kl": 3.028125, | |
| "learning_rate": 7.12110437492443e-06, | |
| "loss": 0.1211, | |
| "reward": -0.753487682590882, | |
| "reward_std": 1.4118338882923127, | |
| "rewards/Qwen2-0.5B-Reward": -0.753487682590882, | |
| "step": 1790 | |
| }, | |
| { | |
| "completion_length": 744.602783203125, | |
| "epoch": 0.6814740030287734, | |
| "grad_norm": 0.6850081086158752, | |
| "kl": 3.18046875, | |
| "learning_rate": 7.014066277976128e-06, | |
| "loss": 0.1272, | |
| "reward": -0.6332276176661253, | |
| "reward_std": 1.3656011939048767, | |
| "rewards/Qwen2-0.5B-Reward": -0.6332276176661253, | |
| "step": 1800 | |
| }, | |
| { | |
| "completion_length": 759.4481526692708, | |
| "epoch": 0.6852599697122665, | |
| "grad_norm": 2.0515530109405518, | |
| "kl": 3.3453125, | |
| "learning_rate": 6.9077250170911005e-06, | |
| "loss": 0.1338, | |
| "reward": -0.8095526337623596, | |
| "reward_std": 1.5075600425402322, | |
| "rewards/Qwen2-0.5B-Reward": -0.8095526337623596, | |
| "step": 1810 | |
| }, | |
| { | |
| "completion_length": 723.5777811686198, | |
| "epoch": 0.6890459363957597, | |
| "grad_norm": 0.7833884358406067, | |
| "kl": 2.9953125, | |
| "learning_rate": 6.802099183252235e-06, | |
| "loss": 0.1198, | |
| "reward": -0.7537414369483789, | |
| "reward_std": 1.383406792084376, | |
| "rewards/Qwen2-0.5B-Reward": -0.7537414369483789, | |
| "step": 1820 | |
| }, | |
| { | |
| "completion_length": 724.8837972005208, | |
| "epoch": 0.6928319030792529, | |
| "grad_norm": 0.9831650853157043, | |
| "kl": 2.5338541666666665, | |
| "learning_rate": 6.697207242368742e-06, | |
| "loss": 0.1013, | |
| "reward": -0.43006037194281815, | |
| "reward_std": 1.1635287086168924, | |
| "rewards/Qwen2-0.5B-Reward": -0.43006037194281815, | |
| "step": 1830 | |
| }, | |
| { | |
| "completion_length": 760.3333374023438, | |
| "epoch": 0.6966178697627461, | |
| "grad_norm": 1.1536668539047241, | |
| "kl": 2.6203125, | |
| "learning_rate": 6.593067532047882e-06, | |
| "loss": 0.1049, | |
| "reward": -0.4441113060961167, | |
| "reward_std": 1.1987637420495352, | |
| "rewards/Qwen2-0.5B-Reward": -0.4441113060961167, | |
| "step": 1840 | |
| }, | |
| { | |
| "completion_length": 749.903251139323, | |
| "epoch": 0.7004038364462393, | |
| "grad_norm": 0.8368715643882751, | |
| "kl": 2.5341145833333334, | |
| "learning_rate": 6.489698258389107e-06, | |
| "loss": 0.1013, | |
| "reward": -0.5944258317351341, | |
| "reward_std": 1.3474121958017349, | |
| "rewards/Qwen2-0.5B-Reward": -0.5944258317351341, | |
| "step": 1850 | |
| }, | |
| { | |
| "completion_length": 745.5365783691407, | |
| "epoch": 0.7041898031297325, | |
| "grad_norm": 1.029958724975586, | |
| "kl": 2.90078125, | |
| "learning_rate": 6.387117492801213e-06, | |
| "loss": 0.1161, | |
| "reward": -0.6068828483422597, | |
| "reward_std": 1.321648943424225, | |
| "rewards/Qwen2-0.5B-Reward": -0.6068828483422597, | |
| "step": 1860 | |
| }, | |
| { | |
| "completion_length": 755.6328796386719, | |
| "epoch": 0.7079757698132256, | |
| "grad_norm": 5.108635425567627, | |
| "kl": 2.9171875, | |
| "learning_rate": 6.285343168843028e-06, | |
| "loss": 0.1167, | |
| "reward": -0.6523237491647402, | |
| "reward_std": 1.3444733719031017, | |
| "rewards/Qwen2-0.5B-Reward": -0.6523237491647402, | |
| "step": 1870 | |
| }, | |
| { | |
| "completion_length": 787.0935241699219, | |
| "epoch": 0.7117617364967188, | |
| "grad_norm": 1.3548846244812012, | |
| "kl": 3.0869791666666666, | |
| "learning_rate": 6.1843930790881766e-06, | |
| "loss": 0.1235, | |
| "reward": -0.6537054566045603, | |
| "reward_std": 1.4838234384854634, | |
| "rewards/Qwen2-0.5B-Reward": -0.6537054566045603, | |
| "step": 1880 | |
| }, | |
| { | |
| "completion_length": 773.563895670573, | |
| "epoch": 0.715547703180212, | |
| "grad_norm": 0.8410789966583252, | |
| "kl": 2.837760416666667, | |
| "learning_rate": 6.084284872014545e-06, | |
| "loss": 0.1136, | |
| "reward": -0.5507580937196811, | |
| "reward_std": 1.2756544808546701, | |
| "rewards/Qwen2-0.5B-Reward": -0.5507580937196811, | |
| "step": 1890 | |
| }, | |
| { | |
| "completion_length": 760.8699096679687, | |
| "epoch": 0.7193336698637052, | |
| "grad_norm": 1.5116900205612183, | |
| "kl": 2.6723958333333333, | |
| "learning_rate": 5.985036048918894e-06, | |
| "loss": 0.1069, | |
| "reward": -0.46427804150929053, | |
| "reward_std": 1.1952710588773092, | |
| "rewards/Qwen2-0.5B-Reward": -0.46427804150929053, | |
| "step": 1900 | |
| }, | |
| { | |
| "completion_length": 763.8004699707031, | |
| "epoch": 0.7231196365471984, | |
| "grad_norm": 1.1645935773849487, | |
| "kl": 3.13828125, | |
| "learning_rate": 5.886663960857202e-06, | |
| "loss": 0.1255, | |
| "reward": -0.7973003094395001, | |
| "reward_std": 1.4403738955656686, | |
| "rewards/Qwen2-0.5B-Reward": -0.7973003094395001, | |
| "step": 1910 | |
| }, | |
| { | |
| "completion_length": 746.5444559733073, | |
| "epoch": 0.7269056032306915, | |
| "grad_norm": 1.8314180374145508, | |
| "kl": 3.378125, | |
| "learning_rate": 5.789185805611313e-06, | |
| "loss": 0.1351, | |
| "reward": -0.6777333706617356, | |
| "reward_std": 1.452496987581253, | |
| "rewards/Qwen2-0.5B-Reward": -0.6777333706617356, | |
| "step": 1920 | |
| }, | |
| { | |
| "completion_length": 743.2513977050781, | |
| "epoch": 0.7306915699141847, | |
| "grad_norm": 1.8599276542663574, | |
| "kl": 2.6572916666666666, | |
| "learning_rate": 5.692618624682342e-06, | |
| "loss": 0.1063, | |
| "reward": -0.5468713939189911, | |
| "reward_std": 1.203757886091868, | |
| "rewards/Qwen2-0.5B-Reward": -0.5468713939189911, | |
| "step": 1930 | |
| }, | |
| { | |
| "completion_length": 715.9157470703125, | |
| "epoch": 0.7344775365976779, | |
| "grad_norm": 3.749554395675659, | |
| "kl": 3.373177083333333, | |
| "learning_rate": 5.596979300311408e-06, | |
| "loss": 0.1351, | |
| "reward": -0.42453126634160676, | |
| "reward_std": 1.129069878657659, | |
| "rewards/Qwen2-0.5B-Reward": -0.42453126634160676, | |
| "step": 1940 | |
| }, | |
| { | |
| "completion_length": 707.4583414713542, | |
| "epoch": 0.7382635032811711, | |
| "grad_norm": 1.2406065464019775, | |
| "kl": 2.40546875, | |
| "learning_rate": 5.502284552528236e-06, | |
| "loss": 0.0962, | |
| "reward": -0.3166978692635894, | |
| "reward_std": 1.0220210254192352, | |
| "rewards/Qwen2-0.5B-Reward": -0.3166978692635894, | |
| "step": 1950 | |
| }, | |
| { | |
| "completion_length": 730.9064880371094, | |
| "epoch": 0.7420494699646644, | |
| "grad_norm": 0.894660472869873, | |
| "kl": 3.0755208333333335, | |
| "learning_rate": 5.408550936228072e-06, | |
| "loss": 0.1231, | |
| "reward": -0.6020015890399615, | |
| "reward_std": 1.3233680129051208, | |
| "rewards/Qwen2-0.5B-Reward": -0.6020015890399615, | |
| "step": 1960 | |
| }, | |
| { | |
| "completion_length": 784.6120402018229, | |
| "epoch": 0.7458354366481575, | |
| "grad_norm": 0.9947274923324585, | |
| "kl": 3.3036458333333334, | |
| "learning_rate": 5.315794838277524e-06, | |
| "loss": 0.1321, | |
| "reward": -0.8605576127767562, | |
| "reward_std": 1.5929324706395467, | |
| "rewards/Qwen2-0.5B-Reward": -0.8605576127767562, | |
| "step": 1970 | |
| }, | |
| { | |
| "completion_length": 761.7782409667968, | |
| "epoch": 0.7496214033316507, | |
| "grad_norm": 0.8357589244842529, | |
| "kl": 3.126822916666667, | |
| "learning_rate": 5.2240324746497185e-06, | |
| "loss": 0.1251, | |
| "reward": -0.6573333943883578, | |
| "reward_std": 1.3803256154060364, | |
| "rewards/Qwen2-0.5B-Reward": -0.6573333943883578, | |
| "step": 1980 | |
| }, | |
| { | |
| "completion_length": 751.271309407552, | |
| "epoch": 0.7534073700151439, | |
| "grad_norm": 0.9635012149810791, | |
| "kl": 2.846875, | |
| "learning_rate": 5.133279887589381e-06, | |
| "loss": 0.114, | |
| "reward": -0.5246660086015861, | |
| "reward_std": 1.2728915989398957, | |
| "rewards/Qwen2-0.5B-Reward": -0.5246660086015861, | |
| "step": 1990 | |
| }, | |
| { | |
| "completion_length": 721.8902760823568, | |
| "epoch": 0.7571933366986371, | |
| "grad_norm": 1.915734887123108, | |
| "kl": 2.886588541666667, | |
| "learning_rate": 5.043552942808269e-06, | |
| "loss": 0.1155, | |
| "reward": -0.4225703233232101, | |
| "reward_std": 1.1504804422458013, | |
| "rewards/Qwen2-0.5B-Reward": -0.4225703233232101, | |
| "step": 2000 | |
| }, | |
| { | |
| "completion_length": 747.6074157714844, | |
| "epoch": 0.7609793033821303, | |
| "grad_norm": 1.7324910163879395, | |
| "kl": 2.849739583333333, | |
| "learning_rate": 4.9548673267114535e-06, | |
| "loss": 0.114, | |
| "reward": -0.4868051894629995, | |
| "reward_std": 1.2382884542147319, | |
| "rewards/Qwen2-0.5B-Reward": -0.4868051894629995, | |
| "step": 2010 | |
| }, | |
| { | |
| "completion_length": 723.2888916015625, | |
| "epoch": 0.7647652700656234, | |
| "grad_norm": 1.870195984840393, | |
| "kl": 3.38125, | |
| "learning_rate": 4.86723854365498e-06, | |
| "loss": 0.1353, | |
| "reward": -0.6813056563337644, | |
| "reward_std": 1.4171151260534922, | |
| "rewards/Qwen2-0.5B-Reward": -0.6813056563337644, | |
| "step": 2020 | |
| }, | |
| { | |
| "completion_length": 739.2546325683594, | |
| "epoch": 0.7685512367491166, | |
| "grad_norm": 0.6563529968261719, | |
| "kl": 2.7765625, | |
| "learning_rate": 4.78068191323533e-06, | |
| "loss": 0.111, | |
| "reward": -0.6810662182668845, | |
| "reward_std": 1.3699560364087422, | |
| "rewards/Qwen2-0.5B-Reward": -0.6810662182668845, | |
| "step": 2030 | |
| }, | |
| { | |
| "completion_length": 723.6301025390625, | |
| "epoch": 0.7723372034326098, | |
| "grad_norm": 0.845397412776947, | |
| "kl": 3.3549479166666667, | |
| "learning_rate": 4.695212567611183e-06, | |
| "loss": 0.1343, | |
| "reward": -0.6839562758803368, | |
| "reward_std": 1.3764802972475687, | |
| "rewards/Qwen2-0.5B-Reward": -0.6839562758803368, | |
| "step": 2040 | |
| }, | |
| { | |
| "completion_length": 707.3944529215495, | |
| "epoch": 0.776123170116103, | |
| "grad_norm": 0.8297199606895447, | |
| "kl": 2.2606770833333334, | |
| "learning_rate": 4.6108454488579754e-06, | |
| "loss": 0.0904, | |
| "reward": -0.32430495528969916, | |
| "reward_std": 1.0496096114317577, | |
| "rewards/Qwen2-0.5B-Reward": -0.32430495528969916, | |
| "step": 2050 | |
| }, | |
| { | |
| "completion_length": 728.9092631022136, | |
| "epoch": 0.7799091367995962, | |
| "grad_norm": 0.8965924382209778, | |
| "kl": 2.7317708333333335, | |
| "learning_rate": 4.5275953063556515e-06, | |
| "loss": 0.1092, | |
| "reward": -0.49890854886422553, | |
| "reward_std": 1.1908490220705668, | |
| "rewards/Qwen2-0.5B-Reward": -0.49890854886422553, | |
| "step": 2060 | |
| }, | |
| { | |
| "completion_length": 787.4157389322917, | |
| "epoch": 0.7836951034830894, | |
| "grad_norm": 1.6908742189407349, | |
| "kl": 3.14453125, | |
| "learning_rate": 4.445476694210125e-06, | |
| "loss": 0.1258, | |
| "reward": -0.6872879594564438, | |
| "reward_std": 1.5059267342090608, | |
| "rewards/Qwen2-0.5B-Reward": -0.6872879594564438, | |
| "step": 2070 | |
| }, | |
| { | |
| "completion_length": 724.3907409667969, | |
| "epoch": 0.7874810701665825, | |
| "grad_norm": 0.5646480917930603, | |
| "kl": 2.5322916666666666, | |
| "learning_rate": 4.364503968708885e-06, | |
| "loss": 0.1013, | |
| "reward": -0.4010113532965382, | |
| "reward_std": 1.1661198248465856, | |
| "rewards/Qwen2-0.5B-Reward": -0.4010113532965382, | |
| "step": 2080 | |
| }, | |
| { | |
| "completion_length": 762.3759338378907, | |
| "epoch": 0.7912670368500757, | |
| "grad_norm": 0.7707305550575256, | |
| "kl": 3.08828125, | |
| "learning_rate": 4.284691285811162e-06, | |
| "loss": 0.1235, | |
| "reward": -0.6063117478042841, | |
| "reward_std": 1.4541340112686156, | |
| "rewards/Qwen2-0.5B-Reward": -0.6063117478042841, | |
| "step": 2090 | |
| }, | |
| { | |
| "completion_length": 757.8597249348958, | |
| "epoch": 0.7950530035335689, | |
| "grad_norm": 0.609060525894165, | |
| "kl": 2.7552083333333335, | |
| "learning_rate": 4.206052598673134e-06, | |
| "loss": 0.1102, | |
| "reward": -0.5107901314894359, | |
| "reward_std": 1.2742640137672425, | |
| "rewards/Qwen2-0.5B-Reward": -0.5107901314894359, | |
| "step": 2100 | |
| }, | |
| { | |
| "completion_length": 714.1713012695312, | |
| "epoch": 0.7988389702170621, | |
| "grad_norm": 1.5023508071899414, | |
| "kl": 2.7880208333333334, | |
| "learning_rate": 4.128601655208588e-06, | |
| "loss": 0.1115, | |
| "reward": -0.4477219473881026, | |
| "reward_std": 1.2109043717384338, | |
| "rewards/Qwen2-0.5B-Reward": -0.4477219473881026, | |
| "step": 2110 | |
| }, | |
| { | |
| "completion_length": 742.9495381673177, | |
| "epoch": 0.8026249369005553, | |
| "grad_norm": 1.4843252897262573, | |
| "kl": 2.490104166666667, | |
| "learning_rate": 4.052351995685459e-06, | |
| "loss": 0.0996, | |
| "reward": -0.40210790758331616, | |
| "reward_std": 1.1073905199766159, | |
| "rewards/Qwen2-0.5B-Reward": -0.40210790758331616, | |
| "step": 2120 | |
| }, | |
| { | |
| "completion_length": 758.4166687011718, | |
| "epoch": 0.8064109035840484, | |
| "grad_norm": 0.8346318006515503, | |
| "kl": 3.2510416666666666, | |
| "learning_rate": 3.977316950358647e-06, | |
| "loss": 0.1301, | |
| "reward": -0.744351115822792, | |
| "reward_std": 1.4400279184182485, | |
| "rewards/Qwen2-0.5B-Reward": -0.744351115822792, | |
| "step": 2130 | |
| }, | |
| { | |
| "completion_length": 711.5217651367187, | |
| "epoch": 0.8101968702675416, | |
| "grad_norm": 3.075549840927124, | |
| "kl": 2.4575520833333333, | |
| "learning_rate": 3.903509637139604e-06, | |
| "loss": 0.0983, | |
| "reward": -0.4195836258431276, | |
| "reward_std": 1.1368374347686767, | |
| "rewards/Qwen2-0.5B-Reward": -0.4195836258431276, | |
| "step": 2140 | |
| }, | |
| { | |
| "completion_length": 667.8574137369792, | |
| "epoch": 0.8139828369510348, | |
| "grad_norm": 1.288053035736084, | |
| "kl": 2.64140625, | |
| "learning_rate": 3.830942959302988e-06, | |
| "loss": 0.1056, | |
| "reward": -0.25947842622796696, | |
| "reward_std": 1.0453672617673875, | |
| "rewards/Qwen2-0.5B-Reward": -0.25947842622796696, | |
| "step": 2150 | |
| }, | |
| { | |
| "completion_length": 713.2092692057291, | |
| "epoch": 0.817768803634528, | |
| "grad_norm": 1.47870934009552, | |
| "kl": 3.060677083333333, | |
| "learning_rate": 3.7596296032308655e-06, | |
| "loss": 0.1224, | |
| "reward": -0.5742474019527435, | |
| "reward_std": 1.2993368287881215, | |
| "rewards/Qwen2-0.5B-Reward": -0.5742474019527435, | |
| "step": 2160 | |
| }, | |
| { | |
| "completion_length": 756.1185282389323, | |
| "epoch": 0.8215547703180212, | |
| "grad_norm": 1.0809710025787354, | |
| "kl": 3.0234375, | |
| "learning_rate": 3.689582036194844e-06, | |
| "loss": 0.121, | |
| "reward": -0.6388996203740438, | |
| "reward_std": 1.3941177546977996, | |
| "rewards/Qwen2-0.5B-Reward": -0.6388996203740438, | |
| "step": 2170 | |
| }, | |
| { | |
| "completion_length": 689.1287068684895, | |
| "epoch": 0.8253407370015143, | |
| "grad_norm": 0.8256644606590271, | |
| "kl": 2.6302083333333335, | |
| "learning_rate": 3.620812504176483e-06, | |
| "loss": 0.1052, | |
| "reward": -0.3896134149283171, | |
| "reward_std": 1.1061949849128723, | |
| "rewards/Qwen2-0.5B-Reward": -0.3896134149283171, | |
| "step": 2180 | |
| }, | |
| { | |
| "completion_length": 747.3708435058594, | |
| "epoch": 0.8291267036850076, | |
| "grad_norm": 1.2586473226547241, | |
| "kl": 2.8255208333333335, | |
| "learning_rate": 3.5533330297264055e-06, | |
| "loss": 0.113, | |
| "reward": -0.47125562417010464, | |
| "reward_std": 1.3159513572851818, | |
| "rewards/Qwen2-0.5B-Reward": -0.47125562417010464, | |
| "step": 2190 | |
| }, | |
| { | |
| "completion_length": 718.9842681884766, | |
| "epoch": 0.8329126703685008, | |
| "grad_norm": 0.7325953841209412, | |
| "kl": 2.89453125, | |
| "learning_rate": 3.4871554098624783e-06, | |
| "loss": 0.1159, | |
| "reward": -0.515640505651633, | |
| "reward_std": 1.2894119222958882, | |
| "rewards/Qwen2-0.5B-Reward": -0.515640505651633, | |
| "step": 2200 | |
| }, | |
| { | |
| "completion_length": 730.6486206054688, | |
| "epoch": 0.836698637051994, | |
| "grad_norm": 1.3458070755004883, | |
| "kl": 2.746354166666667, | |
| "learning_rate": 3.4222912140074072e-06, | |
| "loss": 0.1099, | |
| "reward": -0.43878471093873184, | |
| "reward_std": 1.1841597487529119, | |
| "rewards/Qwen2-0.5B-Reward": -0.43878471093873184, | |
| "step": 2210 | |
| }, | |
| { | |
| "completion_length": 728.4597218831381, | |
| "epoch": 0.8404846037354872, | |
| "grad_norm": 2.082460880279541, | |
| "kl": 3.025520833333333, | |
| "learning_rate": 3.358751781966125e-06, | |
| "loss": 0.121, | |
| "reward": -0.5120975616077582, | |
| "reward_std": 1.399947702884674, | |
| "rewards/Qwen2-0.5B-Reward": -0.5120975616077582, | |
| "step": 2220 | |
| }, | |
| { | |
| "completion_length": 702.8384338378906, | |
| "epoch": 0.8442705704189803, | |
| "grad_norm": 0.7987167239189148, | |
| "kl": 2.9817708333333335, | |
| "learning_rate": 3.2965482219433266e-06, | |
| "loss": 0.1193, | |
| "reward": -0.5346707743903001, | |
| "reward_std": 1.298090636730194, | |
| "rewards/Qwen2-0.5B-Reward": -0.5346707743903001, | |
| "step": 2230 | |
| }, | |
| { | |
| "completion_length": 743.4412068684895, | |
| "epoch": 0.8480565371024735, | |
| "grad_norm": 1.0572713613510132, | |
| "kl": 2.8296875, | |
| "learning_rate": 3.2356914086014895e-06, | |
| "loss": 0.1132, | |
| "reward": -0.45420979845027126, | |
| "reward_std": 1.2626650591691335, | |
| "rewards/Qwen2-0.5B-Reward": -0.45420979845027126, | |
| "step": 2240 | |
| }, | |
| { | |
| "completion_length": 751.9037150065104, | |
| "epoch": 0.8518425037859667, | |
| "grad_norm": 1.2263774871826172, | |
| "kl": 2.789322916666667, | |
| "learning_rate": 3.1761919811597286e-06, | |
| "loss": 0.1116, | |
| "reward": -0.41814162402103344, | |
| "reward_std": 1.254759935537974, | |
| "rewards/Qwen2-0.5B-Reward": -0.41814162402103344, | |
| "step": 2250 | |
| }, | |
| { | |
| "completion_length": 735.0213012695312, | |
| "epoch": 0.8556284704694599, | |
| "grad_norm": 1.536089539527893, | |
| "kl": 2.711197916666667, | |
| "learning_rate": 3.118060341533795e-06, | |
| "loss": 0.1084, | |
| "reward": -0.3957721870703002, | |
| "reward_std": 1.215382601817449, | |
| "rewards/Qwen2-0.5B-Reward": -0.3957721870703002, | |
| "step": 2260 | |
| }, | |
| { | |
| "completion_length": 739.1541676839192, | |
| "epoch": 0.8594144371529531, | |
| "grad_norm": 2.2628087997436523, | |
| "kl": 3.322135416666667, | |
| "learning_rate": 3.0613066525175916e-06, | |
| "loss": 0.1328, | |
| "reward": -0.5474292345655462, | |
| "reward_std": 1.3296300649642945, | |
| "rewards/Qwen2-0.5B-Reward": -0.5474292345655462, | |
| "step": 2270 | |
| }, | |
| { | |
| "completion_length": 753.1319498697917, | |
| "epoch": 0.8632004038364463, | |
| "grad_norm": 1.759981393814087, | |
| "kl": 2.53984375, | |
| "learning_rate": 3.00594083600646e-06, | |
| "loss": 0.1016, | |
| "reward": -0.4004799094672004, | |
| "reward_std": 1.2508702536424001, | |
| "rewards/Qwen2-0.5B-Reward": -0.4004799094672004, | |
| "step": 2280 | |
| }, | |
| { | |
| "completion_length": 765.2296366373698, | |
| "epoch": 0.8669863705199394, | |
| "grad_norm": 1.7521519660949707, | |
| "kl": 3.2877604166666665, | |
| "learning_rate": 2.9519725712625993e-06, | |
| "loss": 0.1315, | |
| "reward": -0.5632258212814728, | |
| "reward_std": 1.3489103774229685, | |
| "rewards/Qwen2-0.5B-Reward": -0.5632258212814728, | |
| "step": 2290 | |
| }, | |
| { | |
| "completion_length": 728.2092671712239, | |
| "epoch": 0.8707723372034326, | |
| "grad_norm": 1.1282004117965698, | |
| "kl": 2.808333333333333, | |
| "learning_rate": 2.89941129322291e-06, | |
| "loss": 0.1123, | |
| "reward": -0.4623491804425915, | |
| "reward_std": 1.2616208771864572, | |
| "rewards/Qwen2-0.5B-Reward": -0.4623491804425915, | |
| "step": 2300 | |
| }, | |
| { | |
| "completion_length": 763.8801025390625, | |
| "epoch": 0.8745583038869258, | |
| "grad_norm": 1.6411226987838745, | |
| "kl": 2.96328125, | |
| "learning_rate": 2.848266190849534e-06, | |
| "loss": 0.1186, | |
| "reward": -0.47133560677369435, | |
| "reward_std": 1.3187556425730387, | |
| "rewards/Qwen2-0.5B-Reward": -0.47133560677369435, | |
| "step": 2310 | |
| }, | |
| { | |
| "completion_length": 767.462967936198, | |
| "epoch": 0.878344270570419, | |
| "grad_norm": 1.238519310951233, | |
| "kl": 2.96015625, | |
| "learning_rate": 2.798546205523405e-06, | |
| "loss": 0.1184, | |
| "reward": -0.553766346598665, | |
| "reward_std": 1.3190133293469748, | |
| "rewards/Qwen2-0.5B-Reward": -0.553766346598665, | |
| "step": 2320 | |
| }, | |
| { | |
| "completion_length": 738.1370381673177, | |
| "epoch": 0.8821302372539122, | |
| "grad_norm": 1.9779850244522095, | |
| "kl": 2.7606770833333334, | |
| "learning_rate": 2.7502600294810888e-06, | |
| "loss": 0.1104, | |
| "reward": -0.48763653316224614, | |
| "reward_std": 1.276737904548645, | |
| "rewards/Qwen2-0.5B-Reward": -0.48763653316224614, | |
| "step": 2330 | |
| }, | |
| { | |
| "completion_length": 772.7652852376302, | |
| "epoch": 0.8859162039374053, | |
| "grad_norm": 0.9569075107574463, | |
| "kl": 3.640625, | |
| "learning_rate": 2.7034161042951696e-06, | |
| "loss": 0.1457, | |
| "reward": -0.752403491238753, | |
| "reward_std": 1.5029548863569895, | |
| "rewards/Qwen2-0.5B-Reward": -0.752403491238753, | |
| "step": 2340 | |
| }, | |
| { | |
| "completion_length": 748.1027770996094, | |
| "epoch": 0.8897021706208985, | |
| "grad_norm": 1.2532896995544434, | |
| "kl": 2.788802083333333, | |
| "learning_rate": 2.658022619398459e-06, | |
| "loss": 0.1115, | |
| "reward": -0.5759354960018148, | |
| "reward_std": 1.252836243311564, | |
| "rewards/Qwen2-0.5B-Reward": -0.5759354960018148, | |
| "step": 2350 | |
| }, | |
| { | |
| "completion_length": 756.765283203125, | |
| "epoch": 0.8934881373043917, | |
| "grad_norm": 1.243710994720459, | |
| "kl": 3.470572916666667, | |
| "learning_rate": 2.6140875106522906e-06, | |
| "loss": 0.1388, | |
| "reward": -0.7527099266648293, | |
| "reward_std": 1.5181720991929373, | |
| "rewards/Qwen2-0.5B-Reward": -0.7527099266648293, | |
| "step": 2360 | |
| }, | |
| { | |
| "completion_length": 731.5490844726562, | |
| "epoch": 0.8972741039878849, | |
| "grad_norm": 0.8256412744522095, | |
| "kl": 2.8911458333333333, | |
| "learning_rate": 2.5716184589591504e-06, | |
| "loss": 0.1156, | |
| "reward": -0.4917602331067125, | |
| "reward_std": 1.3739383776982625, | |
| "rewards/Qwen2-0.5B-Reward": -0.4917602331067125, | |
| "step": 2370 | |
| }, | |
| { | |
| "completion_length": 762.2222249348958, | |
| "epoch": 0.901060070671378, | |
| "grad_norm": 0.976091206073761, | |
| "kl": 3.2059895833333334, | |
| "learning_rate": 2.5306228889198595e-06, | |
| "loss": 0.1282, | |
| "reward": -0.492262601479888, | |
| "reward_std": 1.3222837885220846, | |
| "rewards/Qwen2-0.5B-Reward": -0.492262601479888, | |
| "step": 2380 | |
| }, | |
| { | |
| "completion_length": 752.6963033040364, | |
| "epoch": 0.9048460373548712, | |
| "grad_norm": 0.8627796769142151, | |
| "kl": 3.144270833333333, | |
| "learning_rate": 2.4911079675355852e-06, | |
| "loss": 0.1258, | |
| "reward": -0.5920792824278275, | |
| "reward_std": 1.4338179051876068, | |
| "rewards/Qwen2-0.5B-Reward": -0.5920792824278275, | |
| "step": 2390 | |
| }, | |
| { | |
| "completion_length": 729.3250172932943, | |
| "epoch": 0.9086320040383644, | |
| "grad_norm": 2.569244384765625, | |
| "kl": 3.0598958333333335, | |
| "learning_rate": 2.453080602954878e-06, | |
| "loss": 0.1224, | |
| "reward": -0.5552944198250771, | |
| "reward_std": 1.259453280766805, | |
| "rewards/Qwen2-0.5B-Reward": -0.5552944198250771, | |
| "step": 2400 | |
| }, | |
| { | |
| "completion_length": 769.4680562337239, | |
| "epoch": 0.9124179707218576, | |
| "grad_norm": 1.9891189336776733, | |
| "kl": 3.347395833333333, | |
| "learning_rate": 2.416547443265959e-06, | |
| "loss": 0.134, | |
| "reward": -0.7994883202016354, | |
| "reward_std": 1.5337923685709636, | |
| "rewards/Qwen2-0.5B-Reward": -0.7994883202016354, | |
| "step": 2410 | |
| }, | |
| { | |
| "completion_length": 711.1333435058593, | |
| "epoch": 0.9162039374053509, | |
| "grad_norm": 1.2348560094833374, | |
| "kl": 2.6640625, | |
| "learning_rate": 2.381514875334478e-06, | |
| "loss": 0.1066, | |
| "reward": -0.4012350387871265, | |
| "reward_std": 1.1682847638924916, | |
| "rewards/Qwen2-0.5B-Reward": -0.4012350387871265, | |
| "step": 2420 | |
| }, | |
| { | |
| "completion_length": 728.6402770996094, | |
| "epoch": 0.9199899040888441, | |
| "grad_norm": 1.0510834455490112, | |
| "kl": 2.4625, | |
| "learning_rate": 2.34798902368694e-06, | |
| "loss": 0.0985, | |
| "reward": -0.255227384219567, | |
| "reward_std": 1.0641139527161916, | |
| "rewards/Qwen2-0.5B-Reward": -0.255227384219567, | |
| "step": 2430 | |
| }, | |
| { | |
| "completion_length": 742.7620483398438, | |
| "epoch": 0.9237758707723372, | |
| "grad_norm": 0.6936110854148865, | |
| "kl": 2.7760416666666665, | |
| "learning_rate": 2.31597574943999e-06, | |
| "loss": 0.1111, | |
| "reward": -0.32442100283806213, | |
| "reward_std": 1.1662549694379172, | |
| "rewards/Qwen2-0.5B-Reward": -0.32442100283806213, | |
| "step": 2440 | |
| }, | |
| { | |
| "completion_length": 709.3500081380208, | |
| "epoch": 0.9275618374558304, | |
| "grad_norm": 0.6553380489349365, | |
| "kl": 2.91484375, | |
| "learning_rate": 2.2854806492757473e-06, | |
| "loss": 0.1166, | |
| "reward": -0.4610091609259446, | |
| "reward_std": 1.2611193935076395, | |
| "rewards/Qwen2-0.5B-Reward": -0.4610091609259446, | |
| "step": 2450 | |
| }, | |
| { | |
| "completion_length": 751.5171427408854, | |
| "epoch": 0.9313478041393236, | |
| "grad_norm": 1.1703935861587524, | |
| "kl": 3.030989583333333, | |
| "learning_rate": 2.256509054463379e-06, | |
| "loss": 0.1212, | |
| "reward": -0.47760866036017735, | |
| "reward_std": 1.3580244441827138, | |
| "rewards/Qwen2-0.5B-Reward": -0.47760866036017735, | |
| "step": 2460 | |
| }, | |
| { | |
| "completion_length": 734.3888997395833, | |
| "epoch": 0.9351337708228168, | |
| "grad_norm": 1.4841110706329346, | |
| "kl": 2.941666666666667, | |
| "learning_rate": 2.2290660299270626e-06, | |
| "loss": 0.1176, | |
| "reward": -0.5363880881418784, | |
| "reward_std": 1.320775838692983, | |
| "rewards/Qwen2-0.5B-Reward": -0.5363880881418784, | |
| "step": 2470 | |
| }, | |
| { | |
| "completion_length": 792.8597361246744, | |
| "epoch": 0.93891973750631, | |
| "grad_norm": 0.9216225743293762, | |
| "kl": 3.2005208333333335, | |
| "learning_rate": 2.2031563733605154e-06, | |
| "loss": 0.128, | |
| "reward": -0.6734383806586266, | |
| "reward_std": 1.5115692138671875, | |
| "rewards/Qwen2-0.5B-Reward": -0.6734383806586266, | |
| "step": 2480 | |
| }, | |
| { | |
| "completion_length": 742.3555603027344, | |
| "epoch": 0.9427057041898032, | |
| "grad_norm": 0.8652907013893127, | |
| "kl": 2.9796875, | |
| "learning_rate": 2.178784614388247e-06, | |
| "loss": 0.1192, | |
| "reward": -0.5235640426476796, | |
| "reward_std": 1.2792722801367442, | |
| "rewards/Qwen2-0.5B-Reward": -0.5235640426476796, | |
| "step": 2490 | |
| }, | |
| { | |
| "completion_length": 731.887510172526, | |
| "epoch": 0.9464916708732963, | |
| "grad_norm": 0.900198221206665, | |
| "kl": 2.758072916666667, | |
| "learning_rate": 2.155955013773674e-06, | |
| "loss": 0.1102, | |
| "reward": -0.427229492738843, | |
| "reward_std": 1.2093970189491907, | |
| "rewards/Qwen2-0.5B-Reward": -0.427229492738843, | |
| "step": 2500 | |
| }, | |
| { | |
| "completion_length": 742.8615783691406, | |
| "epoch": 0.9502776375567895, | |
| "grad_norm": 1.4608945846557617, | |
| "kl": 2.884635416666667, | |
| "learning_rate": 2.134671562674233e-06, | |
| "loss": 0.1154, | |
| "reward": -0.40613490512090117, | |
| "reward_std": 1.2598043183485668, | |
| "rewards/Qwen2-0.5B-Reward": -0.40613490512090117, | |
| "step": 2510 | |
| }, | |
| { | |
| "completion_length": 732.2143575032552, | |
| "epoch": 0.9540636042402827, | |
| "grad_norm": 0.861190676689148, | |
| "kl": 3.107291666666667, | |
| "learning_rate": 2.114937981943634e-06, | |
| "loss": 0.1243, | |
| "reward": -0.4464622031897306, | |
| "reward_std": 1.2578558444976806, | |
| "rewards/Qwen2-0.5B-Reward": -0.4464622031897306, | |
| "step": 2520 | |
| }, | |
| { | |
| "completion_length": 796.321767171224, | |
| "epoch": 0.9578495709237759, | |
| "grad_norm": 2.202199697494507, | |
| "kl": 3.068489583333333, | |
| "learning_rate": 2.096757721481365e-06, | |
| "loss": 0.1228, | |
| "reward": -0.6399494647979737, | |
| "reward_std": 1.4180189092954, | |
| "rewards/Qwen2-0.5B-Reward": -0.6399494647979737, | |
| "step": 2530 | |
| }, | |
| { | |
| "completion_length": 744.4689880371094, | |
| "epoch": 0.961635537607269, | |
| "grad_norm": 0.9193338751792908, | |
| "kl": 2.9799479166666667, | |
| "learning_rate": 2.0801339596295706e-06, | |
| "loss": 0.1192, | |
| "reward": -0.5712469642050564, | |
| "reward_std": 1.3389502465724945, | |
| "rewards/Qwen2-0.5B-Reward": -0.5712469642050564, | |
| "step": 2540 | |
| }, | |
| { | |
| "completion_length": 779.9819458007812, | |
| "epoch": 0.9654215042907622, | |
| "grad_norm": 1.811191439628601, | |
| "kl": 3.6411458333333333, | |
| "learning_rate": 2.0650696026173993e-06, | |
| "loss": 0.1456, | |
| "reward": -0.7589993777374426, | |
| "reward_std": 1.5557840009530386, | |
| "rewards/Qwen2-0.5B-Reward": -0.7589993777374426, | |
| "step": 2550 | |
| }, | |
| { | |
| "completion_length": 782.5782531738281, | |
| "epoch": 0.9692074709742554, | |
| "grad_norm": 0.9566059112548828, | |
| "kl": 3.095572916666667, | |
| "learning_rate": 2.051567284052924e-06, | |
| "loss": 0.1238, | |
| "reward": -0.6302419572137297, | |
| "reward_std": 1.4944741606712342, | |
| "rewards/Qwen2-0.5B-Reward": -0.6302419572137297, | |
| "step": 2560 | |
| }, | |
| { | |
| "completion_length": 718.9152872721354, | |
| "epoch": 0.9729934376577486, | |
| "grad_norm": 0.9425510168075562, | |
| "kl": 2.82890625, | |
| "learning_rate": 2.0396293644627313e-06, | |
| "loss": 0.1132, | |
| "reward": -0.32908876914686214, | |
| "reward_std": 1.2080858111381532, | |
| "rewards/Qwen2-0.5B-Reward": -0.32908876914686214, | |
| "step": 2570 | |
| }, | |
| { | |
| "completion_length": 732.7824157714844, | |
| "epoch": 0.9767794043412418, | |
| "grad_norm": 0.9575442671775818, | |
| "kl": 3.121875, | |
| "learning_rate": 2.0292579308792374e-06, | |
| "loss": 0.125, | |
| "reward": -0.47131281966964406, | |
| "reward_std": 1.3826497634251913, | |
| "rewards/Qwen2-0.5B-Reward": -0.47131281966964406, | |
| "step": 2580 | |
| }, | |
| { | |
| "completion_length": 761.6578694661458, | |
| "epoch": 0.980565371024735, | |
| "grad_norm": 1.0160202980041504, | |
| "kl": 2.9203125, | |
| "learning_rate": 2.020454796475829e-06, | |
| "loss": 0.1168, | |
| "reward": -0.47771473427613576, | |
| "reward_std": 1.2897698918978373, | |
| "rewards/Qwen2-0.5B-Reward": -0.47771473427613576, | |
| "step": 2590 | |
| }, | |
| { | |
| "completion_length": 781.389815266927, | |
| "epoch": 0.9843513377082281, | |
| "grad_norm": 2.1385881900787354, | |
| "kl": 2.9213541666666667, | |
| "learning_rate": 2.013221500249879e-06, | |
| "loss": 0.1168, | |
| "reward": -0.4969511273006598, | |
| "reward_std": 1.3705980678399403, | |
| "rewards/Qwen2-0.5B-Reward": -0.4969511273006598, | |
| "step": 2600 | |
| }, | |
| { | |
| "completion_length": 748.4037109375, | |
| "epoch": 0.9881373043917213, | |
| "grad_norm": 1.3061258792877197, | |
| "kl": 3.1223958333333335, | |
| "learning_rate": 2.0075593067536895e-06, | |
| "loss": 0.1249, | |
| "reward": -0.511777646218737, | |
| "reward_std": 1.338163250684738, | |
| "rewards/Qwen2-0.5B-Reward": -0.511777646218737, | |
| "step": 2610 | |
| }, | |
| { | |
| "completion_length": 730.8513916015625, | |
| "epoch": 0.9919232710752145, | |
| "grad_norm": 1.0508885383605957, | |
| "kl": 2.6411458333333333, | |
| "learning_rate": 2.0034692058734197e-06, | |
| "loss": 0.1056, | |
| "reward": -0.3765604312221209, | |
| "reward_std": 1.2147092600663503, | |
| "rewards/Qwen2-0.5B-Reward": -0.3765604312221209, | |
| "step": 2620 | |
| }, | |
| { | |
| "completion_length": 783.2115763346354, | |
| "epoch": 0.9957092377587077, | |
| "grad_norm": 1.1735745668411255, | |
| "kl": 3.4817708333333335, | |
| "learning_rate": 2.000951912656033e-06, | |
| "loss": 0.1392, | |
| "reward": -0.6186425998806954, | |
| "reward_std": 1.4974812746047974, | |
| "rewards/Qwen2-0.5B-Reward": -0.6186425998806954, | |
| "step": 2630 | |
| }, | |
| { | |
| "completion_length": 786.5088033040364, | |
| "epoch": 0.9994952044422009, | |
| "grad_norm": 1.1970211267471313, | |
| "kl": 3.134375, | |
| "learning_rate": 2.0000078671842824e-06, | |
| "loss": 0.1254, | |
| "reward": -0.662852063588798, | |
| "reward_std": 1.5238366266091665, | |
| "rewards/Qwen2-0.5B-Reward": -0.662852063588798, | |
| "step": 2640 | |
| }, | |
| { | |
| "completion_length": 728.1759236653646, | |
| "epoch": 0.9998738011105502, | |
| "kl": 3.125, | |
| "reward": -0.8051454126834869, | |
| "reward_std": 1.3035079042116802, | |
| "rewards/Qwen2-0.5B-Reward": -0.8051454126834869, | |
| "step": 2641, | |
| "total_flos": 0.0, | |
| "train_loss": 0.6071465962344739, | |
| "train_runtime": 159997.8149, | |
| "train_samples_per_second": 1.189, | |
| "train_steps_per_second": 0.017 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 2641, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 24, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |