trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9906542056074765,
  "eval_steps": 50,
  "global_step": 240,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.06230529595015576,
      "grad_norm": 56.34838588103866,
      "learning_rate": 5e-07,
      "logits/chosen": -2.7382490634918213,
      "logits/rejected": -2.7438971996307373,
      "logps/chosen": -245.8295135498047,
      "logps/rejected": -209.98046875,
      "loss": 0.691,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.015010075643658638,
      "rewards/margins": 0.004885209258645773,
      "rewards/rejected": 0.010124864056706429,
      "step": 5
    },
    {
      "epoch": 0.12461059190031153,
      "grad_norm": 45.9085359451971,
      "learning_rate": 1e-06,
      "logits/chosen": -2.7156083583831787,
      "logits/rejected": -2.690063238143921,
      "logps/chosen": -252.92239379882812,
      "logps/rejected": -216.73739624023438,
      "loss": 0.6632,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.4806228280067444,
      "rewards/margins": 0.08186856657266617,
      "rewards/rejected": 0.39875420928001404,
      "step": 10
    },
    {
      "epoch": 0.18691588785046728,
      "grad_norm": 58.41523189603188,
      "learning_rate": 9.988343845952696e-07,
      "logits/chosen": -2.598924160003662,
      "logits/rejected": -2.5831100940704346,
      "logps/chosen": -237.48486328125,
      "logps/rejected": -211.9320526123047,
      "loss": 0.6379,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 1.1204158067703247,
      "rewards/margins": 0.44865816831588745,
      "rewards/rejected": 0.671757698059082,
      "step": 15
    },
    {
      "epoch": 0.24922118380062305,
      "grad_norm": 41.1554668806477,
      "learning_rate": 9.953429730181652e-07,
      "logits/chosen": -2.470618963241577,
      "logits/rejected": -2.4658849239349365,
      "logps/chosen": -254.59793090820312,
      "logps/rejected": -209.291259765625,
      "loss": 0.6363,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 1.3058934211730957,
      "rewards/margins": 0.7762569189071655,
      "rewards/rejected": 0.5296366214752197,
      "step": 20
    },
    {
      "epoch": 0.3115264797507788,
      "grad_norm": 40.38894629339522,
      "learning_rate": 9.895420438411615e-07,
      "logits/chosen": -2.4649369716644287,
      "logits/rejected": -2.4389662742614746,
      "logps/chosen": -254.17141723632812,
      "logps/rejected": -217.0956573486328,
      "loss": 0.5787,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 1.326682686805725,
      "rewards/margins": 1.0829169750213623,
      "rewards/rejected": 0.2437656819820404,
      "step": 25
    },
    {
      "epoch": 0.37383177570093457,
      "grad_norm": 57.52273881572831,
      "learning_rate": 9.814586436738997e-07,
      "logits/chosen": -2.410034418106079,
      "logits/rejected": -2.4170825481414795,
      "logps/chosen": -275.00592041015625,
      "logps/rejected": -213.52603149414062,
      "loss": 0.7002,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 1.1761306524276733,
      "rewards/margins": 0.9896278381347656,
      "rewards/rejected": 0.18650276958942413,
      "step": 30
    },
    {
      "epoch": 0.43613707165109034,
      "grad_norm": 38.786439832034034,
      "learning_rate": 9.711304610594102e-07,
      "logits/chosen": -2.4654526710510254,
      "logits/rejected": -2.4857430458068848,
      "logps/chosen": -258.40081787109375,
      "logps/rejected": -234.67782592773438,
      "loss": 0.6262,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.8200791478157043,
      "rewards/margins": 1.1157252788543701,
      "rewards/rejected": -0.29564613103866577,
      "step": 35
    },
    {
      "epoch": 0.4984423676012461,
      "grad_norm": 42.06281667149408,
      "learning_rate": 9.586056507527264e-07,
      "logits/chosen": -2.578695297241211,
      "logits/rejected": -2.567582130432129,
      "logps/chosen": -247.830810546875,
      "logps/rejected": -212.32162475585938,
      "loss": 0.5893,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.46938619017601013,
      "rewards/margins": 0.9617160558700562,
      "rewards/rejected": -0.4923298954963684,
      "step": 40
    },
    {
      "epoch": 0.5607476635514018,
      "grad_norm": 40.51045424322713,
      "learning_rate": 9.439426092011875e-07,
      "logits/chosen": -2.530611753463745,
      "logits/rejected": -2.5346996784210205,
      "logps/chosen": -236.1021728515625,
      "logps/rejected": -191.10340881347656,
      "loss": 0.5809,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.8879886865615845,
      "rewards/margins": 1.1003386974334717,
      "rewards/rejected": -0.21235008537769318,
      "step": 45
    },
    {
      "epoch": 0.6230529595015576,
      "grad_norm": 41.070235006641056,
      "learning_rate": 9.272097022732443e-07,
      "logits/chosen": -2.4531688690185547,
      "logits/rejected": -2.44033145904541,
      "logps/chosen": -243.89498901367188,
      "logps/rejected": -221.97781372070312,
      "loss": 0.5728,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 1.1465950012207031,
      "rewards/margins": 1.246957540512085,
      "rewards/rejected": -0.10036251693964005,
      "step": 50
    },
    {
      "epoch": 0.6230529595015576,
      "eval_logits/chosen": -2.391545295715332,
      "eval_logits/rejected": -2.384368896484375,
      "eval_logps/chosen": -235.5865936279297,
      "eval_logps/rejected": -228.01226806640625,
      "eval_loss": 0.5968390107154846,
      "eval_rewards/accuracies": 0.7256944179534912,
      "eval_rewards/chosen": 0.7303421497344971,
      "eval_rewards/margins": 1.1997265815734863,
      "eval_rewards/rejected": -0.4693845212459564,
      "eval_runtime": 155.6733,
      "eval_samples_per_second": 14.646,
      "eval_steps_per_second": 0.231,
      "step": 50
    },
    {
      "epoch": 0.6853582554517134,
      "grad_norm": 39.027179175366236,
      "learning_rate": 9.084849465052209e-07,
      "logits/chosen": -2.3710460662841797,
      "logits/rejected": -2.3490705490112305,
      "logps/chosen": -255.9646759033203,
      "logps/rejected": -219.50857543945312,
      "loss": 0.5505,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.7654061317443848,
      "rewards/margins": 1.0589392185211182,
      "rewards/rejected": -0.2935330271720886,
      "step": 55
    },
    {
      "epoch": 0.7476635514018691,
      "grad_norm": 31.313233738677823,
      "learning_rate": 8.878556453522099e-07,
      "logits/chosen": -2.3525824546813965,
      "logits/rejected": -2.3543479442596436,
      "logps/chosen": -253.71505737304688,
      "logps/rejected": -201.6951141357422,
      "loss": 0.5816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9503009915351868,
      "rewards/margins": 1.210055947303772,
      "rewards/rejected": -0.2597549855709076,
      "step": 60
    },
    {
      "epoch": 0.8099688473520249,
      "grad_norm": 36.25718825843554,
      "learning_rate": 8.654179821390621e-07,
      "logits/chosen": -2.4189820289611816,
      "logits/rejected": -2.3751320838928223,
      "logps/chosen": -251.0222625732422,
      "logps/rejected": -240.55673217773438,
      "loss": 0.5284,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.047858476638794,
      "rewards/margins": 1.614628553390503,
      "rewards/rejected": -0.5667698979377747,
      "step": 65
    },
    {
      "epoch": 0.8722741433021807,
      "grad_norm": 42.10074218723004,
      "learning_rate": 8.41276571609327e-07,
      "logits/chosen": -2.2447798252105713,
      "logits/rejected": -2.228858232498169,
      "logps/chosen": -246.00088500976562,
      "logps/rejected": -217.8887176513672,
      "loss": 0.5096,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.5517180562019348,
      "rewards/margins": 1.4356729984283447,
      "rewards/rejected": -0.8839548230171204,
      "step": 70
    },
    {
      "epoch": 0.9345794392523364,
      "grad_norm": 33.6961723599163,
      "learning_rate": 8.155439721630264e-07,
      "logits/chosen": -2.2094359397888184,
      "logits/rejected": -2.175151824951172,
      "logps/chosen": -244.4361114501953,
      "logps/rejected": -232.34951782226562,
      "loss": 0.5524,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.25969991087913513,
      "rewards/margins": 1.9039891958236694,
      "rewards/rejected": -1.644289255142212,
      "step": 75
    },
    {
      "epoch": 0.9968847352024922,
      "grad_norm": 38.59605881477195,
      "learning_rate": 7.883401610574336e-07,
      "logits/chosen": -2.2218358516693115,
      "logits/rejected": -2.2164409160614014,
      "logps/chosen": -237.241455078125,
      "logps/rejected": -227.68930053710938,
      "loss": 0.5343,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.39141470193862915,
      "rewards/margins": 1.7186530828475952,
      "rewards/rejected": -1.3272384405136108,
      "step": 80
    },
    {
      "epoch": 1.0591900311526479,
      "grad_norm": 27.74166454190078,
      "learning_rate": 7.597919750177168e-07,
      "logits/chosen": -2.3127217292785645,
      "logits/rejected": -2.332404375076294,
      "logps/chosen": -225.7181854248047,
      "logps/rejected": -223.03555297851562,
      "loss": 0.2706,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.530363917350769,
      "rewards/margins": 3.146545171737671,
      "rewards/rejected": -1.6161813735961914,
      "step": 85
    },
    {
      "epoch": 1.1214953271028036,
      "grad_norm": 17.961582190346057,
      "learning_rate": 7.30032518865576e-07,
      "logits/chosen": -2.3584330081939697,
      "logits/rejected": -2.365981101989746,
      "logps/chosen": -220.9420166015625,
      "logps/rejected": -205.1517791748047,
      "loss": 0.2665,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.430635690689087,
      "rewards/margins": 2.625166416168213,
      "rewards/rejected": -1.1945306062698364,
      "step": 90
    },
    {
      "epoch": 1.1838006230529594,
      "grad_norm": 21.643840458648096,
      "learning_rate": 6.992005449231207e-07,
      "logits/chosen": -2.439763069152832,
      "logits/rejected": -2.425145149230957,
      "logps/chosen": -236.6886444091797,
      "logps/rejected": -216.1404571533203,
      "loss": 0.2656,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.9334638118743896,
      "rewards/margins": 2.866502523422241,
      "rewards/rejected": -0.9330384135246277,
      "step": 95
    },
    {
      "epoch": 1.2461059190031152,
      "grad_norm": 21.558166945326157,
      "learning_rate": 6.67439806085493e-07,
      "logits/chosen": -2.4390244483947754,
      "logits/rejected": -2.4261584281921387,
      "logps/chosen": -242.9800567626953,
      "logps/rejected": -260.5987548828125,
      "loss": 0.2338,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6462457180023193,
      "rewards/margins": 3.361034870147705,
      "rewards/rejected": -1.714788794517517,
      "step": 100
    },
    {
      "epoch": 1.2461059190031152,
      "eval_logits/chosen": -2.3908119201660156,
      "eval_logits/rejected": -2.375950813293457,
      "eval_logps/chosen": -234.32077026367188,
      "eval_logps/rejected": -234.03512573242188,
      "eval_loss": 0.6137881875038147,
      "eval_rewards/accuracies": 0.7708333134651184,
      "eval_rewards/chosen": 0.8569262623786926,
      "eval_rewards/margins": 1.92859947681427,
      "eval_rewards/rejected": -1.0716731548309326,
      "eval_runtime": 154.1188,
      "eval_samples_per_second": 14.794,
      "eval_steps_per_second": 0.234,
      "step": 100
    },
    {
      "epoch": 1.308411214953271,
      "grad_norm": 26.860281379520078,
      "learning_rate": 6.348983855785121e-07,
      "logits/chosen": -2.3314337730407715,
      "logits/rejected": -2.349992275238037,
      "logps/chosen": -249.3834991455078,
      "logps/rejected": -231.1989288330078,
      "loss": 0.2493,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.4759314060211182,
      "rewards/margins": 3.2000317573547363,
      "rewards/rejected": -1.7241003513336182,
      "step": 105
    },
    {
      "epoch": 1.3707165109034267,
      "grad_norm": 22.99769883052198,
      "learning_rate": 6.01728006526317e-07,
      "logits/chosen": -2.285414218902588,
      "logits/rejected": -2.2368404865264893,
      "logps/chosen": -239.63363647460938,
      "logps/rejected": -269.333984375,
      "loss": 0.304,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.348543405532837,
      "rewards/margins": 3.7768986225128174,
      "rewards/rejected": -2.428354501724243,
      "step": 110
    },
    {
      "epoch": 1.4330218068535825,
      "grad_norm": 24.081073307307026,
      "learning_rate": 5.680833245481234e-07,
      "logits/chosen": -2.1882622241973877,
      "logits/rejected": -2.156731367111206,
      "logps/chosen": -225.35452270507812,
      "logps/rejected": -217.11221313476562,
      "loss": 0.2717,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.0362424850463867,
      "rewards/margins": 3.186000347137451,
      "rewards/rejected": -2.1497581005096436,
      "step": 115
    },
    {
      "epoch": 1.4953271028037383,
      "grad_norm": 18.350121327926843,
      "learning_rate": 5.341212066823355e-07,
      "logits/chosen": -2.2497525215148926,
      "logits/rejected": -2.205735445022583,
      "logps/chosen": -250.9644775390625,
      "logps/rejected": -233.6098175048828,
      "loss": 0.3001,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.1595510244369507,
      "rewards/margins": 3.244880199432373,
      "rewards/rejected": -2.085329532623291,
      "step": 120
    },
    {
      "epoch": 1.557632398753894,
      "grad_norm": 26.07382841281333,
      "learning_rate": 5e-07,
      "logits/chosen": -2.2849984169006348,
      "logits/rejected": -2.28584623336792,
      "logps/chosen": -248.89193725585938,
      "logps/rejected": -232.82040405273438,
      "loss": 0.298,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.4610850811004639,
      "rewards/margins": 3.512969970703125,
      "rewards/rejected": -2.051884412765503,
      "step": 125
    },
    {
      "epoch": 1.6199376947040498,
      "grad_norm": 22.973286685710313,
      "learning_rate": 4.6587879331766457e-07,
      "logits/chosen": -2.2841477394104004,
      "logits/rejected": -2.2463271617889404,
      "logps/chosen": -243.2410125732422,
      "logps/rejected": -229.27896118164062,
      "loss": 0.2767,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.4858837127685547,
      "rewards/margins": 3.5905258655548096,
      "rewards/rejected": -2.104642152786255,
      "step": 130
    },
    {
      "epoch": 1.6822429906542056,
      "grad_norm": 22.789061692514213,
      "learning_rate": 4.3191667545187675e-07,
      "logits/chosen": -2.262861728668213,
      "logits/rejected": -2.2234511375427246,
      "logps/chosen": -241.4350128173828,
      "logps/rejected": -245.9232940673828,
      "loss": 0.2914,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": 1.3016725778579712,
      "rewards/margins": 3.095862865447998,
      "rewards/rejected": -1.794190764427185,
      "step": 135
    },
    {
      "epoch": 1.7445482866043613,
      "grad_norm": 23.65640774073897,
      "learning_rate": 3.9827199347368317e-07,
      "logits/chosen": -2.2054550647735596,
      "logits/rejected": -2.2026901245117188,
      "logps/chosen": -264.45501708984375,
      "logps/rejected": -237.9841766357422,
      "loss": 0.3003,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.9526735544204712,
      "rewards/margins": 3.4445793628692627,
      "rewards/rejected": -1.491905927658081,
      "step": 140
    },
    {
      "epoch": 1.8068535825545171,
      "grad_norm": 23.820154104399354,
      "learning_rate": 3.651016144214878e-07,
      "logits/chosen": -2.150700092315674,
      "logits/rejected": -2.1031599044799805,
      "logps/chosen": -229.19406127929688,
      "logps/rejected": -239.5289764404297,
      "loss": 0.3014,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 1.0087716579437256,
      "rewards/margins": 2.76188325881958,
      "rewards/rejected": -1.7531116008758545,
      "step": 145
    },
    {
      "epoch": 1.8691588785046729,
      "grad_norm": 20.680429909039592,
      "learning_rate": 3.325601939145069e-07,
      "logits/chosen": -2.0694642066955566,
      "logits/rejected": -2.020296573638916,
      "logps/chosen": -246.52316284179688,
      "logps/rejected": -234.9423828125,
      "loss": 0.2653,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.1492115259170532,
      "rewards/margins": 3.305049419403076,
      "rewards/rejected": -2.1558375358581543,
      "step": 150
    },
    {
      "epoch": 1.8691588785046729,
      "eval_logits/chosen": -2.076385259628296,
      "eval_logits/rejected": -2.037816047668457,
      "eval_logps/chosen": -238.1874237060547,
      "eval_logps/rejected": -240.6510467529297,
      "eval_loss": 0.582950234413147,
      "eval_rewards/accuracies": 0.7847222089767456,
      "eval_rewards/chosen": 0.4702628552913666,
      "eval_rewards/margins": 2.20352840423584,
      "eval_rewards/rejected": -1.7332653999328613,
      "eval_runtime": 154.0013,
      "eval_samples_per_second": 14.805,
      "eval_steps_per_second": 0.234,
      "step": 150
    },
    {
      "epoch": 1.9314641744548287,
      "grad_norm": 20.45293805027137,
      "learning_rate": 3.007994550768793e-07,
      "logits/chosen": -2.0885276794433594,
      "logits/rejected": -2.050126314163208,
      "logps/chosen": -242.3552703857422,
      "logps/rejected": -222.5155792236328,
      "loss": 0.2923,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.1564452648162842,
      "rewards/margins": 3.603581666946411,
      "rewards/rejected": -2.447136878967285,
      "step": 155
    },
    {
      "epoch": 1.9937694704049844,
      "grad_norm": 21.833836348878105,
      "learning_rate": 2.699674811344239e-07,
      "logits/chosen": -2.138195037841797,
      "logits/rejected": -2.0640130043029785,
      "logps/chosen": -249.90072631835938,
      "logps/rejected": -231.67251586914062,
      "loss": 0.28,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 1.1243386268615723,
      "rewards/margins": 2.991178274154663,
      "rewards/rejected": -1.8668396472930908,
      "step": 160
    },
    {
      "epoch": 2.05607476635514,
      "grad_norm": 16.21642740152857,
      "learning_rate": 2.4020802498228334e-07,
      "logits/chosen": -2.131587028503418,
      "logits/rejected": -2.0931520462036133,
      "logps/chosen": -232.63137817382812,
      "logps/rejected": -237.9394989013672,
      "loss": 0.1622,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 1.1016236543655396,
      "rewards/margins": 3.968158721923828,
      "rewards/rejected": -2.866534948348999,
      "step": 165
    },
    {
      "epoch": 2.1183800623052957,
      "grad_norm": 12.467550628862439,
      "learning_rate": 2.1165983894256646e-07,
      "logits/chosen": -2.179967164993286,
      "logits/rejected": -2.0978798866271973,
      "logps/chosen": -243.21743774414062,
      "logps/rejected": -230.3401641845703,
      "loss": 0.1948,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 1.2113419771194458,
      "rewards/margins": 3.5999789237976074,
      "rewards/rejected": -2.388637065887451,
      "step": 170
    },
    {
      "epoch": 2.1806853582554515,
      "grad_norm": 18.64446540169753,
      "learning_rate": 1.8445602783697373e-07,
      "logits/chosen": -2.1651523113250732,
      "logits/rejected": -2.1325273513793945,
      "logps/chosen": -252.69198608398438,
      "logps/rejected": -249.0771026611328,
      "loss": 0.1815,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.5569490194320679,
      "rewards/margins": 4.343931198120117,
      "rewards/rejected": -2.786982297897339,
      "step": 175
    },
    {
      "epoch": 2.2429906542056073,
      "grad_norm": 15.683999589479217,
      "learning_rate": 1.5872342839067304e-07,
      "logits/chosen": -2.18384051322937,
      "logits/rejected": -2.165940046310425,
      "logps/chosen": -263.38360595703125,
      "logps/rejected": -274.786865234375,
      "loss": 0.1487,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 1.9029051065444946,
      "rewards/margins": 4.339284896850586,
      "rewards/rejected": -2.436379909515381,
      "step": 180
    },
    {
      "epoch": 2.305295950155763,
      "grad_norm": 23.180618554844308,
      "learning_rate": 1.3458201786093794e-07,
      "logits/chosen": -2.1769142150878906,
      "logits/rejected": -2.158796787261963,
      "logps/chosen": -255.7793731689453,
      "logps/rejected": -226.1667022705078,
      "loss": 0.174,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7539451122283936,
      "rewards/margins": 3.86859130859375,
      "rewards/rejected": -2.1146464347839355,
      "step": 185
    },
    {
      "epoch": 2.367601246105919,
      "grad_norm": 17.006108918769268,
      "learning_rate": 1.1214435464779003e-07,
      "logits/chosen": -2.1792142391204834,
      "logits/rejected": -2.144963026046753,
      "logps/chosen": -245.10693359375,
      "logps/rejected": -237.35708618164062,
      "loss": 0.1675,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 1.6174907684326172,
      "rewards/margins": 3.498182773590088,
      "rewards/rejected": -1.8806921243667603,
      "step": 190
    },
    {
      "epoch": 2.4299065420560746,
      "grad_norm": 14.60006641247541,
      "learning_rate": 9.1515053494779e-08,
      "logits/chosen": -2.173074722290039,
      "logits/rejected": -2.155726194381714,
      "logps/chosen": -244.15890502929688,
      "logps/rejected": -243.330078125,
      "loss": 0.1436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8104774951934814,
      "rewards/margins": 4.1725993156433105,
      "rewards/rejected": -2.3621222972869873,
      "step": 195
    },
    {
      "epoch": 2.4922118380062304,
      "grad_norm": 13.747249297522233,
      "learning_rate": 7.279029772675571e-08,
      "logits/chosen": -2.153377056121826,
      "logits/rejected": -2.1178088188171387,
      "logps/chosen": -229.81005859375,
      "logps/rejected": -236.9427947998047,
      "loss": 0.1596,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 1.6026397943496704,
      "rewards/margins": 4.020419120788574,
      "rewards/rejected": -2.417778968811035,
      "step": 200
    },
    {
      "epoch": 2.4922118380062304,
      "eval_logits/chosen": -2.1339945793151855,
      "eval_logits/rejected": -2.098018169403076,
      "eval_logps/chosen": -236.99815368652344,
      "eval_logps/rejected": -241.3352813720703,
      "eval_loss": 0.5909473299980164,
      "eval_rewards/accuracies": 0.7951388955116272,
      "eval_rewards/chosen": 0.5891900062561035,
      "eval_rewards/margins": 2.390880584716797,
      "eval_rewards/rejected": -1.8016905784606934,
      "eval_runtime": 154.2153,
      "eval_samples_per_second": 14.785,
      "eval_steps_per_second": 0.233,
      "step": 200
    },
    {
      "epoch": 2.554517133956386,
      "grad_norm": 18.627527154150737,
      "learning_rate": 5.605739079881239e-08,
      "logits/chosen": -2.1388046741485596,
      "logits/rejected": -2.1039702892303467,
      "logps/chosen": -241.29098510742188,
      "logps/rejected": -246.86788940429688,
      "loss": 0.1755,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 1.7203861474990845,
      "rewards/margins": 3.916400194168091,
      "rewards/rejected": -2.196014404296875,
      "step": 205
    },
    {
      "epoch": 2.616822429906542,
      "grad_norm": 18.03684712892508,
      "learning_rate": 4.139434924727358e-08,
      "logits/chosen": -2.145224094390869,
      "logits/rejected": -2.0875296592712402,
      "logps/chosen": -259.2725524902344,
      "logps/rejected": -242.9119873046875,
      "loss": 0.1656,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.6005407571792603,
      "rewards/margins": 4.087174415588379,
      "rewards/rejected": -2.486633777618408,
      "step": 210
    },
    {
      "epoch": 2.6791277258566977,
      "grad_norm": 12.871046747595889,
      "learning_rate": 2.88695389405898e-08,
      "logits/chosen": -2.138601779937744,
      "logits/rejected": -2.0954153537750244,
      "logps/chosen": -243.4184112548828,
      "logps/rejected": -236.484130859375,
      "loss": 0.1621,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.6027095317840576,
      "rewards/margins": 4.136972904205322,
      "rewards/rejected": -2.5342631340026855,
      "step": 215
    },
    {
      "epoch": 2.7414330218068534,
      "grad_norm": 13.505191215361943,
      "learning_rate": 1.8541356326100433e-08,
      "logits/chosen": -2.142859935760498,
      "logits/rejected": -2.0753793716430664,
      "logps/chosen": -245.02413940429688,
      "logps/rejected": -245.5359649658203,
      "loss": 0.1323,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.5164835453033447,
      "rewards/margins": 4.027895927429199,
      "rewards/rejected": -2.5114119052886963,
      "step": 220
    },
    {
      "epoch": 2.803738317757009,
      "grad_norm": 14.033482831473906,
      "learning_rate": 1.0457956158838544e-08,
      "logits/chosen": -2.149101734161377,
      "logits/rejected": -2.109515428543091,
      "logps/chosen": -239.86318969726562,
      "logps/rejected": -244.78219604492188,
      "loss": 0.1469,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 1.368645429611206,
      "rewards/margins": 3.5874221324920654,
      "rewards/rejected": -2.2187767028808594,
      "step": 225
    },
    {
      "epoch": 2.866043613707165,
      "grad_norm": 22.877149395533472,
      "learning_rate": 4.657026981834622e-09,
      "logits/chosen": -2.1656360626220703,
      "logits/rejected": -2.0624213218688965,
      "logps/chosen": -248.471923828125,
      "logps/rejected": -235.92123413085938,
      "loss": 0.1531,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.731321930885315,
      "rewards/margins": 4.191580772399902,
      "rewards/rejected": -2.460258960723877,
      "step": 230
    },
    {
      "epoch": 2.9283489096573208,
      "grad_norm": 18.036264930612884,
      "learning_rate": 1.165615404730369e-09,
      "logits/chosen": -2.1174049377441406,
      "logits/rejected": -2.0800187587738037,
      "logps/chosen": -244.3081817626953,
      "logps/rejected": -245.9373321533203,
      "loss": 0.1678,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.6983273029327393,
      "rewards/margins": 4.150626182556152,
      "rewards/rejected": -2.452298641204834,
      "step": 235
    },
    {
      "epoch": 2.9906542056074765,
      "grad_norm": 13.000079824584395,
      "learning_rate": 0.0,
      "logits/chosen": -2.1518876552581787,
      "logits/rejected": -2.1043975353240967,
      "logps/chosen": -230.76272583007812,
      "logps/rejected": -275.3558044433594,
      "loss": 0.1542,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 1.3987993001937866,
      "rewards/margins": 4.600934982299805,
      "rewards/rejected": -3.2021358013153076,
      "step": 240
    },
    {
      "epoch": 2.9906542056074765,
      "step": 240,
      "total_flos": 2829829665718272.0,
      "train_loss": 0.3456081494688988,
      "train_runtime": 8828.9956,
      "train_samples_per_second": 6.97,
      "train_steps_per_second": 0.027
    }
  ],
  "logging_steps": 5,
  "max_steps": 240,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 2829829665718272.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}