{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.999129299085764,
  "eval_steps": 500,
  "global_step": 1148,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0017414018284719198,
      "grad_norm": 8.844049684334736,
      "learning_rate": 4.347826086956522e-09,
      "logits/chosen": -2.1702048778533936,
      "logits/rejected": -2.147840976715088,
      "logps/chosen": -357.1834716796875,
      "logps/rejected": -297.0339050292969,
      "loss": 0.7657,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.49489647150039673,
      "rewards/margins": -0.06785201281309128,
      "rewards/rejected": -0.42704448103904724,
      "step": 1
    },
    {
      "epoch": 0.0174140182847192,
      "grad_norm": 8.512120768481358,
      "learning_rate": 4.347826086956521e-08,
      "logits/chosen": -2.1466188430786133,
      "logits/rejected": -2.1188981533050537,
      "logps/chosen": -283.2376708984375,
      "logps/rejected": -265.96881103515625,
      "loss": 0.7959,
      "rewards/accuracies": 0.5347222089767456,
      "rewards/chosen": -0.24174706637859344,
      "rewards/margins": 0.014394841156899929,
      "rewards/rejected": -0.25614187121391296,
      "step": 10
    },
    {
      "epoch": 0.0348280365694384,
      "grad_norm": 8.239560203618277,
      "learning_rate": 8.695652173913042e-08,
      "logits/chosen": -2.1134533882141113,
      "logits/rejected": -2.117985963821411,
      "logps/chosen": -250.8323211669922,
      "logps/rejected": -261.24664306640625,
      "loss": 0.777,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.2070809304714203,
      "rewards/margins": -0.05868260934948921,
      "rewards/rejected": -0.14839830994606018,
      "step": 20
    },
    {
      "epoch": 0.052242054854157595,
      "grad_norm": 8.606900294875057,
      "learning_rate": 1.3043478260869563e-07,
      "logits/chosen": -2.110276699066162,
      "logits/rejected": -2.1439013481140137,
      "logps/chosen": -266.18218994140625,
      "logps/rejected": -271.2703552246094,
      "loss": 0.7799,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.2804611623287201,
      "rewards/margins": -0.13158798217773438,
      "rewards/rejected": -0.14887316524982452,
      "step": 30
    },
    {
      "epoch": 0.0696560731388768,
      "grad_norm": 9.692153660451718,
      "learning_rate": 1.7391304347826085e-07,
      "logits/chosen": -2.1535534858703613,
      "logits/rejected": -2.114678144454956,
      "logps/chosen": -277.32080078125,
      "logps/rejected": -286.0308837890625,
      "loss": 0.8205,
      "rewards/accuracies": 0.39375001192092896,
      "rewards/chosen": -0.35390037298202515,
      "rewards/margins": -0.1384916603565216,
      "rewards/rejected": -0.21540872752666473,
      "step": 40
    },
    {
      "epoch": 0.087070091423596,
      "grad_norm": 8.346879198961663,
      "learning_rate": 2.1739130434782607e-07,
      "logits/chosen": -2.1041924953460693,
      "logits/rejected": -2.127553939819336,
      "logps/chosen": -221.91073608398438,
      "logps/rejected": -258.43695068359375,
      "loss": 0.7782,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.20603910088539124,
      "rewards/margins": -0.035119764506816864,
      "rewards/rejected": -0.17091932892799377,
      "step": 50
    },
    {
      "epoch": 0.10448410970831519,
      "grad_norm": 8.67788838964822,
      "learning_rate": 2.6086956521739126e-07,
      "logits/chosen": -2.148763656616211,
      "logits/rejected": -2.1099791526794434,
      "logps/chosen": -280.0306701660156,
      "logps/rejected": -256.14227294921875,
      "loss": 0.7768,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.2249571979045868,
      "rewards/margins": 0.023237206041812897,
      "rewards/rejected": -0.2481943815946579,
      "step": 60
    },
    {
      "epoch": 0.12189812799303439,
      "grad_norm": 9.828309658683802,
      "learning_rate": 3.043478260869565e-07,
      "logits/chosen": -2.0887808799743652,
      "logits/rejected": -2.088336944580078,
      "logps/chosen": -332.12945556640625,
      "logps/rejected": -313.81414794921875,
      "loss": 0.7704,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.34262752532958984,
      "rewards/margins": -0.09613354504108429,
      "rewards/rejected": -0.24649401009082794,
      "step": 70
    },
    {
      "epoch": 0.1393121462777536,
      "grad_norm": 8.185880061317567,
      "learning_rate": 3.478260869565217e-07,
      "logits/chosen": -2.1172094345092773,
      "logits/rejected": -2.134474039077759,
      "logps/chosen": -296.28826904296875,
      "logps/rejected": -285.25518798828125,
      "loss": 0.7487,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.3245551586151123,
      "rewards/margins": 0.022397944703698158,
      "rewards/rejected": -0.3469530940055847,
      "step": 80
    },
    {
      "epoch": 0.1567261645624728,
      "grad_norm": 7.7420469340564635,
      "learning_rate": 3.9130434782608694e-07,
      "logits/chosen": -2.094935894012451,
      "logits/rejected": -2.1085808277130127,
      "logps/chosen": -303.3551940917969,
      "logps/rejected": -300.27947998046875,
      "loss": 0.7223,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.2835540771484375,
      "rewards/margins": 0.20690183341503143,
      "rewards/rejected": -0.4904559254646301,
      "step": 90
    },
    {
      "epoch": 0.174140182847192,
      "grad_norm": 8.651198054361824,
      "learning_rate": 4.3478260869565214e-07,
      "logits/chosen": -2.092306137084961,
      "logits/rejected": -2.0790088176727295,
      "logps/chosen": -271.1525573730469,
      "logps/rejected": -267.4534912109375,
      "loss": 0.7183,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.26196426153182983,
      "rewards/margins": 0.21165423095226288,
      "rewards/rejected": -0.4736185073852539,
      "step": 100
    },
    {
      "epoch": 0.19155420113191118,
      "grad_norm": 10.111518264609504,
      "learning_rate": 4.782608695652174e-07,
      "logits/chosen": -2.0904226303100586,
      "logits/rejected": -2.0757131576538086,
      "logps/chosen": -325.46441650390625,
      "logps/rejected": -300.5054931640625,
      "loss": 0.7089,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.38440728187561035,
      "rewards/margins": 0.20498594641685486,
      "rewards/rejected": -0.5893932580947876,
      "step": 110
    },
    {
      "epoch": 0.20896821941663038,
      "grad_norm": 8.357831874137293,
      "learning_rate": 4.99971097144235e-07,
      "logits/chosen": -2.081120491027832,
      "logits/rejected": -2.063978910446167,
      "logps/chosen": -278.9669189453125,
      "logps/rejected": -275.8960876464844,
      "loss": 0.6822,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.4673360288143158,
      "rewards/margins": 0.2232373058795929,
      "rewards/rejected": -0.6905733346939087,
      "step": 120
    },
    {
      "epoch": 0.22638223770134958,
      "grad_norm": 6.985340348696225,
      "learning_rate": 4.997399143945734e-07,
      "logits/chosen": -2.0402920246124268,
      "logits/rejected": -2.023264169692993,
      "logps/chosen": -304.0439453125,
      "logps/rejected": -308.83148193359375,
      "loss": 0.6861,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.5328105092048645,
      "rewards/margins": 0.33650466799736023,
      "rewards/rejected": -0.8693151473999023,
      "step": 130
    },
    {
      "epoch": 0.24379625598606877,
      "grad_norm": 7.907314001481999,
      "learning_rate": 4.992777627018238e-07,
      "logits/chosen": -2.0651631355285645,
      "logits/rejected": -2.0785372257232666,
      "logps/chosen": -282.19049072265625,
      "logps/rejected": -298.1554870605469,
      "loss": 0.6522,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.6244045495986938,
      "rewards/margins": 0.4292391836643219,
      "rewards/rejected": -1.053643822669983,
      "step": 140
    },
    {
      "epoch": 0.26121027427078797,
      "grad_norm": 8.703778108062284,
      "learning_rate": 4.985850694813964e-07,
      "logits/chosen": -2.0446629524230957,
      "logits/rejected": -1.9900391101837158,
      "logps/chosen": -335.6188659667969,
      "logps/rejected": -301.2302551269531,
      "loss": 0.6512,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.593495786190033,
      "rewards/margins": 0.33610254526138306,
      "rewards/rejected": -0.9295982122421265,
      "step": 150
    },
    {
      "epoch": 0.2786242925555072,
      "grad_norm": 7.604419369633016,
      "learning_rate": 4.976624753622489e-07,
      "logits/chosen": -2.008415937423706,
      "logits/rejected": -1.985809564590454,
      "logps/chosen": -274.2901306152344,
      "logps/rejected": -263.8446350097656,
      "loss": 0.6276,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.6323899030685425,
      "rewards/margins": 0.5014010667800903,
      "rewards/rejected": -1.1337908506393433,
      "step": 160
    },
    {
      "epoch": 0.29603831084022636,
      "grad_norm": 8.890727468027393,
      "learning_rate": 4.965108335944079e-07,
      "logits/chosen": -2.0417532920837402,
      "logits/rejected": -2.0029454231262207,
      "logps/chosen": -334.0169372558594,
      "logps/rejected": -292.4551086425781,
      "loss": 0.6525,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8084823489189148,
      "rewards/margins": 0.4327976703643799,
      "rewards/rejected": -1.24127995967865,
      "step": 170
    },
    {
      "epoch": 0.3134523291249456,
      "grad_norm": 9.042448192292673,
      "learning_rate": 4.951312092598509e-07,
      "logits/chosen": -2.0257630348205566,
      "logits/rejected": -2.0058140754699707,
      "logps/chosen": -308.8097229003906,
      "logps/rejected": -294.46380615234375,
      "loss": 0.633,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.7714217305183411,
      "rewards/margins": 0.4130379557609558,
      "rewards/rejected": -1.1844598054885864,
      "step": 180
    },
    {
      "epoch": 0.3308663474096648,
      "grad_norm": 9.382903560933434,
      "learning_rate": 4.935248782874788e-07,
      "logits/chosen": -2.0365254878997803,
      "logits/rejected": -2.0127861499786377,
      "logps/chosen": -313.80291748046875,
      "logps/rejected": -330.29217529296875,
      "loss": 0.6096,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.7942873239517212,
      "rewards/margins": 0.5464383363723755,
      "rewards/rejected": -1.3407256603240967,
      "step": 190
    },
    {
      "epoch": 0.348280365694384,
      "grad_norm": 7.5565820244970245,
      "learning_rate": 4.916933262730875e-07,
      "logits/chosen": -1.9759804010391235,
      "logits/rejected": -1.9610658884048462,
      "logps/chosen": -312.9435119628906,
      "logps/rejected": -316.52264404296875,
      "loss": 0.6078,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.6971460580825806,
      "rewards/margins": 0.8095090985298157,
      "rewards/rejected": -1.506655216217041,
      "step": 200
    },
    {
      "epoch": 0.3656943839791032,
      "grad_norm": 7.374096840574112,
      "learning_rate": 4.896382471054332e-07,
      "logits/chosen": -1.985717535018921,
      "logits/rejected": -1.9872894287109375,
      "logps/chosen": -254.1015167236328,
      "logps/rejected": -287.0960693359375,
      "loss": 0.5933,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.720018744468689,
      "rewards/margins": 0.7669107913970947,
      "rewards/rejected": -1.4869292974472046,
      "step": 210
    },
    {
      "epoch": 0.38310840226382237,
      "grad_norm": 7.624989591972422,
      "learning_rate": 4.873615413996599e-07,
      "logits/chosen": -1.9928096532821655,
      "logits/rejected": -1.9948679208755493,
      "logps/chosen": -325.39544677734375,
      "logps/rejected": -330.56683349609375,
      "loss": 0.582,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.781804084777832,
      "rewards/margins": 0.7554414868354797,
      "rewards/rejected": -1.5372456312179565,
      "step": 220
    },
    {
      "epoch": 0.4005224205485416,
      "grad_norm": 7.060510114814007,
      "learning_rate": 4.848653147395379e-07,
      "logits/chosen": -2.046459674835205,
      "logits/rejected": -2.046940565109253,
      "logps/chosen": -322.68267822265625,
      "logps/rejected": -305.71185302734375,
      "loss": 0.5925,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7969905734062195,
      "rewards/margins": 0.6938871145248413,
      "rewards/rejected": -1.4908777475357056,
      "step": 230
    },
    {
      "epoch": 0.41793643883326076,
      "grad_norm": 7.808787153158849,
      "learning_rate": 4.821518757301406e-07,
      "logits/chosen": -1.987998604774475,
      "logits/rejected": -1.9746402502059937,
      "logps/chosen": -329.2791442871094,
      "logps/rejected": -365.9977722167969,
      "loss": 0.5995,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.9474765062332153,
      "rewards/margins": 0.7399638295173645,
      "rewards/rejected": -1.687440276145935,
      "step": 240
    },
    {
      "epoch": 0.43535045711798,
      "grad_norm": 8.266081643322533,
      "learning_rate": 4.792237338627589e-07,
      "logits/chosen": -1.9381145238876343,
      "logits/rejected": -1.9238882064819336,
      "logps/chosen": -266.5848388671875,
      "logps/rejected": -302.54107666015625,
      "loss": 0.5462,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9154646992683411,
      "rewards/margins": 0.7565081715583801,
      "rewards/rejected": -1.6719729900360107,
      "step": 250
    },
    {
      "epoch": 0.45276447540269915,
      "grad_norm": 8.866767020249103,
      "learning_rate": 4.76083597194028e-07,
      "logits/chosen": -1.989005446434021,
      "logits/rejected": -1.9648574590682983,
      "logps/chosen": -327.95220947265625,
      "logps/rejected": -330.86798095703125,
      "loss": 0.573,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.8779212236404419,
      "rewards/margins": 0.710732638835907,
      "rewards/rejected": -1.588653802871704,
      "step": 260
    },
    {
      "epoch": 0.4701784936874184,
      "grad_norm": 8.129697459058349,
      "learning_rate": 4.72734369841415e-07,
      "logits/chosen": -1.9629056453704834,
      "logits/rejected": -1.9648876190185547,
      "logps/chosen": -300.1961669921875,
      "logps/rejected": -300.8248291015625,
      "loss": 0.544,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.8822592496871948,
      "rewards/margins": 0.7416678667068481,
      "rewards/rejected": -1.6239268779754639,
      "step": 270
    },
    {
      "epoch": 0.48759251197213754,
      "grad_norm": 9.294619871719249,
      "learning_rate": 4.691791492973796e-07,
      "logits/chosen": -1.9035098552703857,
      "logits/rejected": -1.894985556602478,
      "logps/chosen": -295.7370300292969,
      "logps/rejected": -308.4239501953125,
      "loss": 0.5556,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9453890919685364,
      "rewards/margins": 0.7959079742431641,
      "rewards/rejected": -1.7412970066070557,
      "step": 280
    },
    {
      "epoch": 0.5050065302568568,
      "grad_norm": 6.877618844659778,
      "learning_rate": 4.6542122356469673e-07,
      "logits/chosen": -1.9330962896347046,
      "logits/rejected": -1.9144872426986694,
      "logps/chosen": -305.0710144042969,
      "logps/rejected": -325.37078857421875,
      "loss": 0.5662,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0533506870269775,
      "rewards/margins": 0.7561749219894409,
      "rewards/rejected": -1.809525728225708,
      "step": 290
    },
    {
      "epoch": 0.5224205485415759,
      "grad_norm": 8.07276835193451,
      "learning_rate": 4.6146406811558627e-07,
      "logits/chosen": -1.9275829792022705,
      "logits/rejected": -1.886498212814331,
      "logps/chosen": -332.5879821777344,
      "logps/rejected": -334.9757080078125,
      "loss": 0.5644,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0968257188796997,
      "rewards/margins": 1.1701604127883911,
      "rewards/rejected": -2.266986131668091,
      "step": 300
    },
    {
      "epoch": 0.5398345668262952,
      "grad_norm": 7.657209782794845,
      "learning_rate": 4.5731134267746464e-07,
      "logits/chosen": -1.9209706783294678,
      "logits/rejected": -1.9280589818954468,
      "logps/chosen": -327.1905822753906,
      "logps/rejected": -352.9628601074219,
      "loss": 0.5311,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.9167665243148804,
      "rewards/margins": 1.3475202322006226,
      "rewards/rejected": -2.264286756515503,
      "step": 310
    },
    {
      "epoch": 0.5572485851110144,
      "grad_norm": 7.933868623349109,
      "learning_rate": 4.529668878482905e-07,
      "logits/chosen": -1.9039608240127563,
      "logits/rejected": -1.8573614358901978,
      "logps/chosen": -314.904541015625,
      "logps/rejected": -299.5747375488281,
      "loss": 0.5578,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.9148025512695312,
      "rewards/margins": 1.1070531606674194,
      "rewards/rejected": -2.0218558311462402,
      "step": 320
    },
    {
      "epoch": 0.5746626033957336,
      "grad_norm": 8.149673009731373,
      "learning_rate": 4.484347215446336e-07,
      "logits/chosen": -1.911172866821289,
      "logits/rejected": -1.911157250404358,
      "logps/chosen": -321.2845764160156,
      "logps/rejected": -319.6854553222656,
      "loss": 0.5135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1006008386611938,
      "rewards/margins": 1.0995060205459595,
      "rewards/rejected": -2.200106620788574,
      "step": 330
    },
    {
      "epoch": 0.5920766216804527,
      "grad_norm": 7.3434448446435345,
      "learning_rate": 4.4371903528575345e-07,
      "logits/chosen": -1.8941545486450195,
      "logits/rejected": -1.8800194263458252,
      "logps/chosen": -300.75286865234375,
      "logps/rejected": -299.771728515625,
      "loss": 0.5417,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0915629863739014,
      "rewards/margins": 0.905167281627655,
      "rewards/rejected": -1.9967304468154907,
      "step": 340
    },
    {
      "epoch": 0.6094906399651719,
      "grad_norm": 8.404700518910351,
      "learning_rate": 4.3882419031712327e-07,
      "logits/chosen": -1.8976866006851196,
      "logits/rejected": -1.8674396276474,
      "logps/chosen": -343.7884826660156,
      "logps/rejected": -319.5928039550781,
      "loss": 0.5609,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.1066405773162842,
      "rewards/margins": 1.020316481590271,
      "rewards/rejected": -2.1269569396972656,
      "step": 350
    },
    {
      "epoch": 0.6269046582498912,
      "grad_norm": 7.3185164911005955,
      "learning_rate": 4.33754713576985e-07,
      "logits/chosen": -1.8984521627426147,
      "logits/rejected": -1.9205257892608643,
      "logps/chosen": -294.2206115722656,
      "logps/rejected": -314.32537841796875,
      "loss": 0.5107,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.8535879850387573,
      "rewards/margins": 1.1626943349838257,
      "rewards/rejected": -2.016282558441162,
      "step": 360
    },
    {
      "epoch": 0.6443186765346104,
      "grad_norm": 9.474069498028365,
      "learning_rate": 4.2851529350966486e-07,
      "logits/chosen": -1.9127724170684814,
      "logits/rejected": -1.8862228393554688,
      "logps/chosen": -286.88067626953125,
      "logps/rejected": -310.11956787109375,
      "loss": 0.5159,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9502976536750793,
      "rewards/margins": 1.1574680805206299,
      "rewards/rejected": -2.1077656745910645,
      "step": 370
    },
    {
      "epoch": 0.6617326948193296,
      "grad_norm": 7.893380940466083,
      "learning_rate": 4.231107757295223e-07,
      "logits/chosen": -1.8950350284576416,
      "logits/rejected": -1.9111878871917725,
      "logps/chosen": -296.33111572265625,
      "logps/rejected": -329.1551208496094,
      "loss": 0.5129,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0437893867492676,
      "rewards/margins": 1.1004403829574585,
      "rewards/rejected": -2.1442298889160156,
      "step": 380
    },
    {
      "epoch": 0.6791467131040487,
      "grad_norm": 7.143443404774061,
      "learning_rate": 4.175461585395423e-07,
      "logits/chosen": -1.872230887413025,
      "logits/rejected": -1.8687286376953125,
      "logps/chosen": -284.56072998046875,
      "logps/rejected": -356.3804016113281,
      "loss": 0.5473,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.073300838470459,
      "rewards/margins": 0.9236575365066528,
      "rewards/rejected": -1.9969584941864014,
      "step": 390
    },
    {
      "epoch": 0.696560731388768,
      "grad_norm": 7.566534743434435,
      "learning_rate": 4.118265883087153e-07,
      "logits/chosen": -1.8513128757476807,
      "logits/rejected": -1.8668243885040283,
      "logps/chosen": -305.4124755859375,
      "logps/rejected": -326.4106140136719,
      "loss": 0.5283,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.2217845916748047,
      "rewards/margins": 1.4612276554107666,
      "rewards/rejected": -2.683012008666992,
      "step": 400
    },
    {
      "epoch": 0.7139747496734872,
      "grad_norm": 8.376082128534176,
      "learning_rate": 4.059573547124793e-07,
      "logits/chosen": -1.9212672710418701,
      "logits/rejected": -1.9072463512420654,
      "logps/chosen": -304.1024169921875,
      "logps/rejected": -336.0544128417969,
      "loss": 0.528,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.2535521984100342,
      "rewards/margins": 1.1496869325637817,
      "rewards/rejected": -2.4032387733459473,
      "step": 410
    },
    {
      "epoch": 0.7313887679582064,
      "grad_norm": 7.034176645851665,
      "learning_rate": 3.9994388584062835e-07,
      "logits/chosen": -1.8729559183120728,
      "logits/rejected": -1.9270069599151611,
      "logps/chosen": -293.9918212890625,
      "logps/rejected": -341.1524963378906,
      "loss": 0.5069,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1038367748260498,
      "rewards/margins": 1.3385248184204102,
      "rewards/rejected": -2.44236159324646,
      "step": 420
    },
    {
      "epoch": 0.7488027862429255,
      "grad_norm": 8.432815372444445,
      "learning_rate": 3.9379174317720867e-07,
      "logits/chosen": -1.9487426280975342,
      "logits/rejected": -1.9000110626220703,
      "logps/chosen": -355.57470703125,
      "logps/rejected": -340.25018310546875,
      "loss": 0.5201,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1578538417816162,
      "rewards/margins": 1.0737416744232178,
      "rewards/rejected": -2.231595516204834,
      "step": 430
    },
    {
      "epoch": 0.7662168045276447,
      "grad_norm": 7.446802151903319,
      "learning_rate": 3.875066164570476e-07,
      "logits/chosen": -1.9263708591461182,
      "logits/rejected": -1.8867603540420532,
      "logps/chosen": -285.8700866699219,
      "logps/rejected": -287.24871826171875,
      "loss": 0.5313,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.0816465616226196,
      "rewards/margins": 1.3171287775039673,
      "rewards/rejected": -2.398775577545166,
      "step": 440
    },
    {
      "epoch": 0.783630822812364,
      "grad_norm": 9.520826434911607,
      "learning_rate": 3.8109431840367094e-07,
      "logits/chosen": -1.8741915225982666,
      "logits/rejected": -1.8706003427505493,
      "logps/chosen": -299.64837646484375,
      "logps/rejected": -338.5923767089844,
      "loss": 0.491,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.0110626220703125,
      "rewards/margins": 1.6349632740020752,
      "rewards/rejected": -2.646026134490967,
      "step": 450
    },
    {
      "epoch": 0.8010448410970832,
      "grad_norm": 7.064936993256622,
      "learning_rate": 3.7456077935347586e-07,
      "logits/chosen": -1.8902429342269897,
      "logits/rejected": -1.8501617908477783,
      "logps/chosen": -315.59869384765625,
      "logps/rejected": -317.4733581542969,
      "loss": 0.5284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1239433288574219,
      "rewards/margins": 1.3349649906158447,
      "rewards/rejected": -2.4589083194732666,
      "step": 460
    },
    {
      "epoch": 0.8184588593818024,
      "grad_norm": 8.398209179162166,
      "learning_rate": 3.6791204177113076e-07,
      "logits/chosen": -1.8681262731552124,
      "logits/rejected": -1.8482496738433838,
      "logps/chosen": -344.9220275878906,
      "logps/rejected": -345.6761779785156,
      "loss": 0.5088,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1243764162063599,
      "rewards/margins": 1.2712541818618774,
      "rewards/rejected": -2.395630359649658,
      "step": 470
    },
    {
      "epoch": 0.8358728776665215,
      "grad_norm": 7.31046188617111,
      "learning_rate": 3.6115425466127523e-07,
      "logits/chosen": -1.918983817100525,
      "logits/rejected": -1.886206865310669,
      "logps/chosen": -309.50164794921875,
      "logps/rejected": -325.39874267578125,
      "loss": 0.5107,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0780818462371826,
      "rewards/margins": 1.3197687864303589,
      "rewards/rejected": -2.397850513458252,
      "step": 480
    },
    {
      "epoch": 0.8532868959512407,
      "grad_norm": 6.614312385974786,
      "learning_rate": 3.5429366788168686e-07,
      "logits/chosen": -1.8966795206069946,
      "logits/rejected": -1.8520530462265015,
      "logps/chosen": -352.29144287109375,
      "logps/rejected": -336.6670227050781,
      "loss": 0.4762,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.196468472480774,
      "rewards/margins": 1.4035875797271729,
      "rewards/rejected": -2.6000559329986572,
      "step": 490
    },
    {
      "epoch": 0.87070091423596,
      "grad_norm": 8.328829028480222,
      "learning_rate": 3.4733662636317615e-07,
      "logits/chosen": -1.8774421215057373,
      "logits/rejected": -1.8671128749847412,
      "logps/chosen": -296.5924377441406,
      "logps/rejected": -354.9656677246094,
      "loss": 0.492,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2636852264404297,
      "rewards/margins": 1.4453284740447998,
      "rewards/rejected": -2.7090134620666504,
      "step": 500
    },
    {
      "epoch": 0.8881149325206792,
      "grad_norm": 5.827647798506863,
      "learning_rate": 3.4028956424155383e-07,
      "logits/chosen": -1.8445724248886108,
      "logits/rejected": -1.849233865737915,
      "logps/chosen": -293.4377746582031,
      "logps/rejected": -325.58154296875,
      "loss": 0.4832,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.2322242259979248,
      "rewards/margins": 1.355747938156128,
      "rewards/rejected": -2.5879721641540527,
      "step": 510
    },
    {
      "epoch": 0.9055289508053983,
      "grad_norm": 9.237943434980734,
      "learning_rate": 3.3315899890709766e-07,
      "logits/chosen": -1.865921974182129,
      "logits/rejected": -1.859374761581421,
      "logps/chosen": -300.2720031738281,
      "logps/rejected": -308.1788024902344,
      "loss": 0.5281,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.1562912464141846,
      "rewards/margins": 1.6701053380966187,
      "rewards/rejected": -2.8263964653015137,
      "step": 520
    },
    {
      "epoch": 0.9229429690901175,
      "grad_norm": 7.154608450387294,
      "learning_rate": 3.259515249770236e-07,
      "logits/chosen": -1.8925888538360596,
      "logits/rejected": -1.8846027851104736,
      "logps/chosen": -291.4532775878906,
      "logps/rejected": -306.2005920410156,
      "loss": 0.5014,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.1419095993041992,
      "rewards/margins": 1.1768567562103271,
      "rewards/rejected": -2.3187661170959473,
      "step": 530
    },
    {
      "epoch": 0.9403569873748368,
      "grad_norm": 7.973301635002224,
      "learning_rate": 3.186738081965329e-07,
      "logits/chosen": -1.885546326637268,
      "logits/rejected": -1.9035097360610962,
      "logps/chosen": -275.60577392578125,
      "logps/rejected": -323.1102600097656,
      "loss": 0.5314,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.0190434455871582,
      "rewards/margins": 1.5625391006469727,
      "rewards/rejected": -2.581582546234131,
      "step": 540
    },
    {
      "epoch": 0.957771005659556,
      "grad_norm": 8.074029796063598,
      "learning_rate": 3.1133257927407875e-07,
      "logits/chosen": -1.9020392894744873,
      "logits/rejected": -1.9024975299835205,
      "logps/chosen": -316.66546630859375,
      "logps/rejected": -324.04022216796875,
      "loss": 0.4816,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.1249531507492065,
      "rewards/margins": 1.4930928945541382,
      "rewards/rejected": -2.618046283721924,
      "step": 550
    },
    {
      "epoch": 0.9751850239442751,
      "grad_norm": 7.882428991139421,
      "learning_rate": 3.0393462765655133e-07,
      "logits/chosen": -1.8837699890136719,
      "logits/rejected": -1.8635187149047852,
      "logps/chosen": -293.06964111328125,
      "logps/rejected": -306.56561279296875,
      "loss": 0.5136,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1180838346481323,
      "rewards/margins": 1.5258665084838867,
      "rewards/rejected": -2.6439504623413086,
      "step": 560
    },
    {
      "epoch": 0.9925990422289943,
      "grad_norm": 8.248403020134447,
      "learning_rate": 2.9648679525014047e-07,
      "logits/chosen": -1.9290965795516968,
      "logits/rejected": -1.9085859060287476,
      "logps/chosen": -329.0329284667969,
      "logps/rejected": -350.5735168457031,
      "loss": 0.4811,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0974798202514648,
      "rewards/margins": 1.4984683990478516,
      "rewards/rejected": -2.5959482192993164,
      "step": 570
    },
    {
      "epoch": 0.999564649542882,
      "eval_logits/chosen": -2.0208449363708496,
      "eval_logits/rejected": -1.998366355895996,
      "eval_logps/chosen": -314.73382568359375,
      "eval_logps/rejected": -316.5445556640625,
      "eval_loss": 0.4442681670188904,
      "eval_rewards/accuracies": 0.7658227682113647,
      "eval_rewards/chosen": -0.7693683505058289,
      "eval_rewards/margins": 1.6421033143997192,
      "eval_rewards/rejected": -2.4114716053009033,
      "eval_runtime": 406.9286,
      "eval_samples_per_second": 6.144,
      "eval_steps_per_second": 0.194,
      "step": 574
    },
    {
      "epoch": 1.0100130605137136,
      "grad_norm": 7.718298739428482,
      "learning_rate": 2.88995970092681e-07,
      "logits/chosen": -1.8420867919921875,
      "logits/rejected": -1.8146463632583618,
      "logps/chosen": -306.36444091796875,
      "logps/rejected": -326.7546081542969,
      "loss": 0.4825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0266987085342407,
      "rewards/margins": 1.6303106546401978,
      "rewards/rejected": -2.6570093631744385,
      "step": 580
    },
    {
      "epoch": 1.0274270787984328,
      "grad_norm": 7.901015022870867,
      "learning_rate": 2.8146907998333414e-07,
      "logits/chosen": -1.8415333032608032,
      "logits/rejected": -1.8439891338348389,
      "logps/chosen": -310.8014221191406,
      "logps/rejected": -343.7575378417969,
      "loss": 0.4447,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0716508626937866,
      "rewards/margins": 1.7897132635116577,
      "rewards/rejected": -2.8613638877868652,
      "step": 590
    },
    {
      "epoch": 1.0448410970831519,
      "grad_norm": 6.5950872734222346,
      "learning_rate": 2.7391308607549617e-07,
      "logits/chosen": -1.8718388080596924,
      "logits/rejected": -1.829660177230835,
      "logps/chosen": -291.44110107421875,
      "logps/rejected": -297.7255859375,
      "loss": 0.4381,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.0404047966003418,
      "rewards/margins": 1.6704820394515991,
      "rewards/rejected": -2.7108867168426514,
      "step": 600
    },
    {
      "epoch": 1.0622551153678712,
      "grad_norm": 8.275049862342016,
      "learning_rate": 2.663349764388602e-07,
      "logits/chosen": -1.8793365955352783,
      "logits/rejected": -1.865886926651001,
      "logps/chosen": -323.6871643066406,
      "logps/rejected": -344.07373046875,
      "loss": 0.4598,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.0635606050491333,
      "rewards/margins": 1.665509819984436,
      "rewards/rejected": -2.7290704250335693,
      "step": 610
    },
    {
      "epoch": 1.0796691336525903,
      "grad_norm": 6.468680208158552,
      "learning_rate": 2.587417595965833e-07,
      "logits/chosen": -1.879450798034668,
      "logits/rejected": -1.8610090017318726,
      "logps/chosen": -324.92266845703125,
      "logps/rejected": -350.06103515625,
      "loss": 0.4098,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9873930215835571,
      "rewards/margins": 2.050591468811035,
      "rewards/rejected": -3.0379843711853027,
      "step": 620
    },
    {
      "epoch": 1.0970831519373094,
      "grad_norm": 7.15585763140329,
      "learning_rate": 2.511404580435399e-07,
      "logits/chosen": -1.8371307849884033,
      "logits/rejected": -1.8505973815917969,
      "logps/chosen": -293.47369384765625,
      "logps/rejected": -327.27056884765625,
      "loss": 0.4404,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3015415668487549,
      "rewards/margins": 1.6929107904434204,
      "rewards/rejected": -2.9944522380828857,
      "step": 630
    },
    {
      "epoch": 1.1144971702220288,
      "grad_norm": 6.984823130077518,
      "learning_rate": 2.435381017516511e-07,
      "logits/chosen": -1.8350080251693726,
      "logits/rejected": -1.7832095623016357,
      "logps/chosen": -335.3113098144531,
      "logps/rejected": -323.35400390625,
      "loss": 0.4155,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.148928165435791,
      "rewards/margins": 1.5296584367752075,
      "rewards/rejected": -2.678586721420288,
      "step": 640
    },
    {
      "epoch": 1.1319111885067479,
      "grad_norm": 5.979906074722325,
      "learning_rate": 2.3594172166830066e-07,
      "logits/chosen": -1.8334687948226929,
      "logits/rejected": -1.8155654668807983,
      "logps/chosen": -307.70831298828125,
      "logps/rejected": -330.25201416015625,
      "loss": 0.425,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.1457269191741943,
      "rewards/margins": 1.5867958068847656,
      "rewards/rejected": -2.732522487640381,
      "step": 650
    },
    {
      "epoch": 1.1493252067914672,
      "grad_norm": 7.830387154655639,
      "learning_rate": 2.2835834321384799e-07,
      "logits/chosen": -1.8376436233520508,
      "logits/rejected": -1.8587745428085327,
      "logps/chosen": -279.7589416503906,
      "logps/rejected": -331.7318115234375,
      "loss": 0.4584,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.1179745197296143,
      "rewards/margins": 1.7496936321258545,
      "rewards/rejected": -2.8676681518554688,
      "step": 660
    },
    {
      "epoch": 1.1667392250761863,
      "grad_norm": 8.68408125755972,
      "learning_rate": 2.20794979784253e-07,
      "logits/chosen": -1.8474241495132446,
      "logits/rejected": -1.8166106939315796,
      "logps/chosen": -308.3825988769531,
      "logps/rejected": -340.49505615234375,
      "loss": 0.4256,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.1534303426742554,
      "rewards/margins": 1.7299216985702515,
      "rewards/rejected": -2.883352041244507,
      "step": 670
    },
    {
      "epoch": 1.1841532433609054,
      "grad_norm": 7.482176700645436,
      "learning_rate": 2.132586262648217e-07,
      "logits/chosen": -1.817705512046814,
      "logits/rejected": -1.834082007408142,
      "logps/chosen": -307.5914611816406,
      "logps/rejected": -363.5468444824219,
      "loss": 0.4337,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.1245834827423096,
      "rewards/margins": 1.9497578144073486,
      "rewards/rejected": -3.074341297149658,
      "step": 680
    },
    {
      "epoch": 1.2015672616456248,
      "grad_norm": 6.906451988777804,
      "learning_rate": 2.0575625256107107e-07,
      "logits/chosen": -1.8379526138305664,
      "logits/rejected": -1.884466528892517,
      "logps/chosen": -300.4937744140625,
      "logps/rejected": -333.07806396484375,
      "loss": 0.4575,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.0864216089248657,
      "rewards/margins": 1.6836363077163696,
      "rewards/rejected": -2.7700579166412354,
      "step": 690
    },
    {
      "epoch": 1.2189812799303439,
      "grad_norm": 8.713458214895912,
      "learning_rate": 1.9829479715269584e-07,
      "logits/chosen": -1.8097610473632812,
      "logits/rejected": -1.8087412118911743,
      "logps/chosen": -264.344970703125,
      "logps/rejected": -302.9281005859375,
      "loss": 0.4096,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.048769235610962,
      "rewards/margins": 1.8850406408309937,
      "rewards/rejected": -2.933809995651245,
      "step": 700
    },
    {
      "epoch": 1.2363952982150632,
      "grad_norm": 7.979056747900625,
      "learning_rate": 1.908811606765996e-07,
      "logits/chosen": -1.8214212656021118,
      "logits/rejected": -1.8329063653945923,
      "logps/chosen": -323.36663818359375,
      "logps/rejected": -353.60968017578125,
      "loss": 0.438,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4141101837158203,
      "rewards/margins": 1.3715263605117798,
      "rewards/rejected": -2.7856364250183105,
      "step": 710
    },
    {
      "epoch": 1.2538093164997823,
      "grad_norm": 6.508818515377178,
      "learning_rate": 1.8352219954492414e-07,
      "logits/chosen": -1.8119535446166992,
      "logits/rejected": -1.802263617515564,
      "logps/chosen": -307.2828369140625,
      "logps/rejected": -350.48431396484375,
      "loss": 0.4365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2268980741500854,
      "rewards/margins": 1.726485013961792,
      "rewards/rejected": -2.953382730484009,
      "step": 720
    },
    {
      "epoch": 1.2712233347845014,
      "grad_norm": 6.931452975928525,
      "learning_rate": 1.7622471960397922e-07,
      "logits/chosen": -1.7938334941864014,
      "logits/rejected": -1.7665965557098389,
      "logps/chosen": -302.1783447265625,
      "logps/rejected": -326.62249755859375,
      "loss": 0.4606,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.3224048614501953,
      "rewards/margins": 1.6808092594146729,
      "rewards/rejected": -3.0032143592834473,
      "step": 730
    },
    {
      "epoch": 1.2886373530692208,
      "grad_norm": 8.138939346919802,
      "learning_rate": 1.6899546983993814e-07,
      "logits/chosen": -1.814523458480835,
      "logits/rejected": -1.7968757152557373,
      "logps/chosen": -317.0029296875,
      "logps/rejected": -324.82989501953125,
      "loss": 0.4281,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.3806750774383545,
      "rewards/margins": 1.450002908706665,
      "rewards/rejected": -2.8306779861450195,
      "step": 740
    },
    {
      "epoch": 1.30605137135394,
      "grad_norm": 8.322373346016452,
      "learning_rate": 1.618411361371193e-07,
      "logits/chosen": -1.8447377681732178,
      "logits/rejected": -1.8304115533828735,
      "logps/chosen": -305.2979431152344,
      "logps/rejected": -337.8390197753906,
      "loss": 0.4398,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.33003830909729,
      "rewards/margins": 1.685516119003296,
      "rewards/rejected": -3.015554189682007,
      "step": 750
    },
    {
      "epoch": 1.3234653896386592,
      "grad_norm": 6.4726812434646215,
      "learning_rate": 1.547683350946268e-07,
      "logits/chosen": -1.8292150497436523,
      "logits/rejected": -1.7922427654266357,
      "logps/chosen": -303.19183349609375,
      "logps/rejected": -330.76763916015625,
      "loss": 0.4305,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -1.2067954540252686,
      "rewards/margins": 1.7238584756851196,
      "rewards/rejected": -2.9306540489196777,
      "step": 760
    },
    {
      "epoch": 1.3408794079233783,
      "grad_norm": 8.88569955786512,
      "learning_rate": 1.477836079070687e-07,
      "logits/chosen": -1.8078101873397827,
      "logits/rejected": -1.8098089694976807,
      "logps/chosen": -318.552490234375,
      "logps/rejected": -389.4412536621094,
      "loss": 0.4382,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.3204103708267212,
      "rewards/margins": 1.9033386707305908,
      "rewards/rejected": -3.2237486839294434,
      "step": 770
    },
    {
      "epoch": 1.3582934262080975,
      "grad_norm": 7.216510660729595,
      "learning_rate": 1.4089341431501228e-07,
      "logits/chosen": -1.8286221027374268,
      "logits/rejected": -1.7622663974761963,
      "logps/chosen": -319.9723815917969,
      "logps/rejected": -306.8332824707031,
      "loss": 0.445,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.1543892621994019,
      "rewards/margins": 1.3471624851226807,
      "rewards/rejected": -2.501551628112793,
      "step": 780
    },
    {
      "epoch": 1.3757074444928168,
      "grad_norm": 7.94753934067897,
      "learning_rate": 1.3410412663077075e-07,
      "logits/chosen": -1.8084096908569336,
      "logits/rejected": -1.761850357055664,
      "logps/chosen": -339.21478271484375,
      "logps/rejected": -370.8948669433594,
      "loss": 0.4386,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.287804365158081,
      "rewards/margins": 1.8586862087249756,
      "rewards/rejected": -3.1464905738830566,
      "step": 790
    },
    {
      "epoch": 1.393121462777536,
      "grad_norm": 7.862200568331616,
      "learning_rate": 1.2742202384504757e-07,
      "logits/chosen": -1.8279838562011719,
      "logits/rejected": -1.7604541778564453,
      "logps/chosen": -342.8545837402344,
      "logps/rejected": -362.16741943359375,
      "loss": 0.4268,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2307243347167969,
      "rewards/margins": 2.1258704662323,
      "rewards/rejected": -3.3565948009490967,
      "step": 800
    },
    {
      "epoch": 1.4105354810622552,
      "grad_norm": 7.614225716428272,
      "learning_rate": 1.208532858198875e-07,
      "logits/chosen": -1.878159761428833,
      "logits/rejected": -1.8207435607910156,
      "logps/chosen": -311.9944152832031,
      "logps/rejected": -338.685791015625,
      "loss": 0.4569,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.287249207496643,
      "rewards/margins": 1.7099910974502563,
      "rewards/rejected": -2.9972400665283203,
      "step": 810
    },
    {
      "epoch": 1.4279494993469743,
      "grad_norm": 7.802404003221548,
      "learning_rate": 1.1440398757330589e-07,
      "logits/chosen": -1.7993576526641846,
      "logits/rejected": -1.8076026439666748,
      "logps/chosen": -305.1946105957031,
      "logps/rejected": -340.22064208984375,
      "loss": 0.426,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -1.240817666053772,
      "rewards/margins": 1.9903885126113892,
      "rewards/rejected": -3.231205701828003,
      "step": 820
    },
    {
      "epoch": 1.4453635176316935,
      "grad_norm": 8.548804802089611,
      "learning_rate": 1.0808009366088155e-07,
      "logits/chosen": -1.8294792175292969,
      "logits/rejected": -1.8587710857391357,
      "logps/chosen": -331.3726501464844,
      "logps/rejected": -372.3400573730469,
      "loss": 0.4528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4628008604049683,
      "rewards/margins": 2.1078591346740723,
      "rewards/rejected": -3.570659637451172,
      "step": 830
    },
    {
      "epoch": 1.4627775359164126,
      "grad_norm": 7.8729677363627655,
      "learning_rate": 1.0188745265950985e-07,
      "logits/chosen": -1.8870255947113037,
      "logits/rejected": -1.829559326171875,
      "logps/chosen": -346.153076171875,
      "logps/rejected": -341.63800048828125,
      "loss": 0.4365,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -1.2471317052841187,
      "rewards/margins": 1.8014585971832275,
      "rewards/rejected": -3.0485901832580566,
      "step": 840
    },
    {
      "epoch": 1.480191554201132,
      "grad_norm": 5.641683304745779,
      "learning_rate": 9.583179175841666e-08,
      "logits/chosen": -1.829275131225586,
      "logits/rejected": -1.8477256298065186,
      "logps/chosen": -267.6588134765625,
      "logps/rejected": -306.33013916015625,
      "loss": 0.4114,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2097364664077759,
      "rewards/margins": 2.002096652984619,
      "rewards/rejected": -3.2118327617645264,
      "step": 850
    },
    {
      "epoch": 1.4976055724858512,
      "grad_norm": 7.824850709437681,
      "learning_rate": 8.991871146243696e-08,
      "logits/chosen": -1.829201102256775,
      "logits/rejected": -1.8199669122695923,
      "logps/chosen": -324.50634765625,
      "logps/rejected": -340.0578918457031,
      "loss": 0.4155,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.1470457315444946,
      "rewards/margins": 1.8675695657730103,
      "rewards/rejected": -3.014615297317505,
      "step": 860
    },
    {
      "epoch": 1.5150195907705704,
      "grad_norm": 7.420933527321597,
      "learning_rate": 8.415368041245513e-08,
      "logits/chosen": -1.8213714361190796,
      "logits/rejected": -1.805848479270935,
      "logps/chosen": -313.3262634277344,
      "logps/rejected": -359.66839599609375,
      "loss": 0.4142,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.1957765817642212,
      "rewards/margins": 2.093334197998047,
      "rewards/rejected": -3.2891106605529785,
      "step": 870
    },
    {
      "epoch": 1.5324336090552895,
      "grad_norm": 8.000337441351382,
      "learning_rate": 7.854203032779894e-08,
      "logits/chosen": -1.8367372751235962,
      "logits/rejected": -1.765334129333496,
      "logps/chosen": -327.23223876953125,
      "logps/rejected": -326.16058349609375,
      "loss": 0.4372,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2566728591918945,
      "rewards/margins": 1.9884045124053955,
      "rewards/rejected": -3.245077610015869,
      "step": 880
    },
    {
      "epoch": 1.5498476273400086,
      "grad_norm": 7.961786383598606,
      "learning_rate": 7.308895107526317e-08,
      "logits/chosen": -1.764020562171936,
      "logits/rejected": -1.7526534795761108,
      "logps/chosen": -306.38909912109375,
      "logps/rejected": -344.85662841796875,
      "loss": 0.4549,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.4552664756774902,
      "rewards/margins": 1.583604097366333,
      "rewards/rejected": -3.038870334625244,
      "step": 890
    },
    {
      "epoch": 1.567261645624728,
      "grad_norm": 7.884902071112754,
      "learning_rate": 6.779948586932389e-08,
      "logits/chosen": -1.8405787944793701,
      "logits/rejected": -1.802128553390503,
      "logps/chosen": -316.4015808105469,
      "logps/rejected": -352.408203125,
      "loss": 0.4411,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.3523671627044678,
      "rewards/margins": 1.8380180597305298,
      "rewards/rejected": -3.190384864807129,
      "step": 900
    },
    {
      "epoch": 1.5846756639094473,
      "grad_norm": 7.462122668724652,
      "learning_rate": 6.267852660798256e-08,
      "logits/chosen": -1.8612207174301147,
      "logits/rejected": -1.8054310083389282,
      "logps/chosen": -335.10284423828125,
      "logps/rejected": -338.3561096191406,
      "loss": 0.4141,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3175737857818604,
      "rewards/margins": 2.1890790462493896,
      "rewards/rejected": -3.50665283203125,
      "step": 910
    },
    {
      "epoch": 1.6020896821941664,
      "grad_norm": 6.711798383913066,
      "learning_rate": 5.7730809348553315e-08,
      "logits/chosen": -1.8333396911621094,
      "logits/rejected": -1.8674615621566772,
      "logps/chosen": -284.63458251953125,
      "logps/rejected": -342.7533874511719,
      "loss": 0.4277,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.3060389757156372,
      "rewards/margins": 2.4165334701538086,
      "rewards/rejected": -3.7225723266601562,
      "step": 920
    },
    {
      "epoch": 1.6195037004788855,
      "grad_norm": 7.142658972521167,
      "learning_rate": 5.296090992757746e-08,
      "logits/chosen": -1.830082654953003,
      "logits/rejected": -1.8328027725219727,
      "logps/chosen": -298.26983642578125,
      "logps/rejected": -348.23681640625,
      "loss": 0.4234,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3221434354782104,
      "rewards/margins": 1.7589191198349,
      "rewards/rejected": -3.0810627937316895,
      "step": 930
    },
    {
      "epoch": 1.6369177187636046,
      "grad_norm": 7.115286161498245,
      "learning_rate": 4.8373239728916326e-08,
      "logits/chosen": -1.832006812095642,
      "logits/rejected": -1.8517078161239624,
      "logps/chosen": -274.48822021484375,
      "logps/rejected": -338.4820251464844,
      "loss": 0.4519,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -1.3210662603378296,
      "rewards/margins": 1.7622454166412354,
      "rewards/rejected": -3.0833117961883545,
      "step": 940
    },
    {
      "epoch": 1.654331737048324,
      "grad_norm": 6.8163876140930535,
      "learning_rate": 4.397204160393628e-08,
      "logits/chosen": -1.8819580078125,
      "logits/rejected": -1.8156343698501587,
      "logps/chosen": -333.33148193359375,
      "logps/rejected": -354.68731689453125,
      "loss": 0.4308,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.2990891933441162,
      "rewards/margins": 1.7505340576171875,
      "rewards/rejected": -3.0496232509613037,
      "step": 950
    },
    {
      "epoch": 1.6717457553330433,
      "grad_norm": 8.28379226070995,
      "learning_rate": 3.9761385947558845e-08,
      "logits/chosen": -1.8182241916656494,
      "logits/rejected": -1.8369022607803345,
      "logps/chosen": -351.6387634277344,
      "logps/rejected": -368.983642578125,
      "loss": 0.4071,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.2291834354400635,
      "rewards/margins": 2.216266632080078,
      "rewards/rejected": -3.4454503059387207,
      "step": 960
    },
    {
      "epoch": 1.6891597736177624,
      "grad_norm": 8.337035098998763,
      "learning_rate": 3.574516693380511e-08,
      "logits/chosen": -1.7874618768692017,
      "logits/rejected": -1.7554718255996704,
      "logps/chosen": -297.45477294921875,
      "logps/rejected": -323.4504699707031,
      "loss": 0.448,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2606045007705688,
      "rewards/margins": 1.9532302618026733,
      "rewards/rejected": -3.213834762573242,
      "step": 970
    },
    {
      "epoch": 1.7065737919024815,
      "grad_norm": 6.881561292806064,
      "learning_rate": 3.192709891431586e-08,
      "logits/chosen": -1.8716919422149658,
      "logits/rejected": -1.8375675678253174,
      "logps/chosen": -349.1175842285156,
      "logps/rejected": -365.41925048828125,
      "loss": 0.4342,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2199608087539673,
      "rewards/margins": 2.3282952308654785,
      "rewards/rejected": -3.5482559204101562,
      "step": 980
    },
    {
      "epoch": 1.7239878101872006,
      "grad_norm": 7.044868858442619,
      "learning_rate": 2.8310712983178524e-08,
      "logits/chosen": -1.816122055053711,
      "logits/rejected": -1.8189055919647217,
      "logps/chosen": -335.2227783203125,
      "logps/rejected": -352.8528137207031,
      "loss": 0.4186,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.3702499866485596,
      "rewards/margins": 1.6819785833358765,
      "rewards/rejected": -3.0522289276123047,
      "step": 990
    },
    {
      "epoch": 1.74140182847192,
      "grad_norm": 7.750615477099287,
      "learning_rate": 2.4899353711237247e-08,
      "logits/chosen": -1.8078467845916748,
      "logits/rejected": -1.742173433303833,
      "logps/chosen": -347.10052490234375,
      "logps/rejected": -330.6437072753906,
      "loss": 0.4174,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.270995855331421,
      "rewards/margins": 1.9451954364776611,
      "rewards/rejected": -3.216191053390503,
      "step": 1000
    },
    {
      "epoch": 1.758815846756639,
      "grad_norm": 6.969253287896509,
      "learning_rate": 2.1696176052907105e-08,
      "logits/chosen": -1.8637828826904297,
      "logits/rejected": -1.82550847530365,
      "logps/chosen": -325.39471435546875,
      "logps/rejected": -346.37591552734375,
      "loss": 0.4263,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3712494373321533,
      "rewards/margins": 2.0143678188323975,
      "rewards/rejected": -3.3856170177459717,
      "step": 1010
    },
    {
      "epoch": 1.7762298650413584,
      "grad_norm": 8.199142967454366,
      "learning_rate": 1.8704142428352528e-08,
      "logits/chosen": -1.818613052368164,
      "logits/rejected": -1.8036258220672607,
      "logps/chosen": -284.3831481933594,
      "logps/rejected": -321.70147705078125,
      "loss": 0.4301,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.2892320156097412,
      "rewards/margins": 1.6578948497772217,
      "rewards/rejected": -2.947126865386963,
      "step": 1020
    },
    {
      "epoch": 1.7936438833260775,
      "grad_norm": 7.283653996594183,
      "learning_rate": 1.592601998372886e-08,
      "logits/chosen": -1.848200798034668,
      "logits/rejected": -1.818587303161621,
      "logps/chosen": -323.9294128417969,
      "logps/rejected": -317.20880126953125,
      "loss": 0.4462,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.3375409841537476,
      "rewards/margins": 1.8687034845352173,
      "rewards/rejected": -3.206244707107544,
      "step": 1030
    },
    {
      "epoch": 1.8110579016107966,
      "grad_norm": 6.789416469264994,
      "learning_rate": 1.336437803202059e-08,
      "logits/chosen": -1.8344234228134155,
      "logits/rejected": -1.8148353099822998,
      "logps/chosen": -312.7352600097656,
      "logps/rejected": -310.73590087890625,
      "loss": 0.4184,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.3291056156158447,
      "rewards/margins": 1.833807349205017,
      "rewards/rejected": -3.1629128456115723,
      "step": 1040
    },
    {
      "epoch": 1.828471919895516,
      "grad_norm": 9.045666403812552,
      "learning_rate": 1.102158567684336e-08,
      "logits/chosen": -1.8186872005462646,
      "logits/rejected": -1.7984111309051514,
      "logps/chosen": -285.7058410644531,
      "logps/rejected": -336.37152099609375,
      "loss": 0.4223,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2095156908035278,
      "rewards/margins": 1.7752981185913086,
      "rewards/rejected": -2.984813928604126,
      "step": 1050
    },
    {
      "epoch": 1.845885938180235,
      "grad_norm": 7.977465181476077,
      "learning_rate": 8.899809621407045e-09,
      "logits/chosen": -1.7870635986328125,
      "logits/rejected": -1.8181617259979248,
      "logps/chosen": -307.60504150390625,
      "logps/rejected": -341.0693054199219,
      "loss": 0.4372,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.4250774383544922,
      "rewards/margins": 1.5568941831588745,
      "rewards/rejected": -2.981971502304077,
      "step": 1060
    },
    {
      "epoch": 1.8632999564649544,
      "grad_norm": 9.145690660979717,
      "learning_rate": 7.001012164666392e-09,
      "logits/chosen": -1.7559747695922852,
      "logits/rejected": -1.793068528175354,
      "logps/chosen": -290.89910888671875,
      "logps/rejected": -365.016845703125,
      "loss": 0.4183,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.3601760864257812,
      "rewards/margins": 1.4941210746765137,
      "rewards/rejected": -2.854297161102295,
      "step": 1070
    },
    {
      "epoch": 1.8807139747496735,
      "grad_norm": 8.329808172715783,
      "learning_rate": 5.326949386512764e-09,
      "logits/chosen": -1.7931833267211914,
      "logits/rejected": -1.8623930215835571,
      "logps/chosen": -278.6099548339844,
      "logps/rejected": -348.0333557128906,
      "loss": 0.4307,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.2068655490875244,
      "rewards/margins": 1.9154495000839233,
      "rewards/rejected": -3.122314929962158,
      "step": 1080
    },
    {
      "epoch": 1.8981279930343926,
      "grad_norm": 6.910823092593948,
      "learning_rate": 3.879169523684639e-09,
      "logits/chosen": -1.8190221786499023,
      "logits/rejected": -1.837376594543457,
      "logps/chosen": -293.0564880371094,
      "logps/rejected": -336.56268310546875,
      "loss": 0.4229,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2015409469604492,
      "rewards/margins": 1.7382638454437256,
      "rewards/rejected": -2.939804792404175,
      "step": 1090
    },
    {
      "epoch": 1.9155420113191117,
      "grad_norm": 7.909316386386267,
      "learning_rate": 2.65901153789963e-09,
      "logits/chosen": -1.8148295879364014,
      "logits/rejected": -1.8017858266830444,
      "logps/chosen": -362.39788818359375,
      "logps/rejected": -370.3252258300781,
      "loss": 0.4304,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.4832170009613037,
      "rewards/margins": 1.842717170715332,
      "rewards/rejected": -3.3259341716766357,
      "step": 1100
    },
    {
      "epoch": 1.932956029603831,
      "grad_norm": 6.466067457229619,
      "learning_rate": 1.6676038775320089e-09,
      "logits/chosen": -1.860700249671936,
      "logits/rejected": -1.8107763528823853,
      "logps/chosen": -396.46697998046875,
      "logps/rejected": -366.28985595703125,
      "loss": 0.4388,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.36767578125,
      "rewards/margins": 1.7400925159454346,
      "rewards/rejected": -3.1077682971954346,
      "step": 1110
    },
    {
      "epoch": 1.9503700478885504,
      "grad_norm": 7.554151818649213,
      "learning_rate": 9.058634339806914e-10,
      "logits/chosen": -1.8141323328018188,
      "logits/rejected": -1.7517668008804321,
      "logps/chosen": -326.8406066894531,
      "logps/rejected": -324.5015563964844,
      "loss": 0.4345,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.2346785068511963,
      "rewards/margins": 2.030879497528076,
      "rewards/rejected": -3.2655582427978516,
      "step": 1120
    },
    {
      "epoch": 1.9677840661732695,
      "grad_norm": 8.387671658761494,
      "learning_rate": 3.74494693693439e-10,
      "logits/chosen": -1.8116728067398071,
      "logits/rejected": -1.8073583841323853,
      "logps/chosen": -298.8089599609375,
      "logps/rejected": -326.7283935546875,
      "loss": 0.4399,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.414529800415039,
      "rewards/margins": 1.7662906646728516,
      "rewards/rejected": -3.1808202266693115,
      "step": 1130
    },
    {
      "epoch": 1.9851980844579886,
      "grad_norm": 8.218002216024905,
      "learning_rate": 7.39890866310322e-11,
      "logits/chosen": -1.854061484336853,
      "logits/rejected": -1.821889877319336,
      "logps/chosen": -303.42376708984375,
      "logps/rejected": -340.0009460449219,
      "loss": 0.4589,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.2597966194152832,
      "rewards/margins": 1.6994686126708984,
      "rewards/rejected": -2.9592652320861816,
      "step": 1140
    },
    {
      "epoch": 1.999129299085764,
      "eval_logits/chosen": -2.0084238052368164,
      "eval_logits/rejected": -1.985912799835205,
      "eval_logps/chosen": -321.51898193359375,
      "eval_logps/rejected": -330.6773986816406,
      "eval_loss": 0.4342744052410126,
      "eval_rewards/accuracies": 0.7784810066223145,
      "eval_rewards/chosen": -1.1086275577545166,
      "eval_rewards/margins": 2.009486198425293,
      "eval_rewards/rejected": -3.1181137561798096,
      "eval_runtime": 577.6821,
      "eval_samples_per_second": 4.328,
      "eval_steps_per_second": 0.137,
      "step": 1148
    },
    {
      "epoch": 1.999129299085764,
      "step": 1148,
      "total_flos": 0.0,
      "train_loss": 0.5132030140128285,
      "train_runtime": 83852.5321,
      "train_samples_per_second": 1.753,
      "train_steps_per_second": 0.014
    }
  ],
  "logging_steps": 10,
  "max_steps": 1148,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}