{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.97196261682243,
  "eval_steps": 50,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.09345794392523364,
      "grad_norm": 60.911718623004575,
      "learning_rate": 5e-07,
      "logits/chosen": -2.7505905628204346,
      "logits/rejected": -2.7347910404205322,
      "logps/chosen": -268.56610107421875,
      "logps/rejected": -197.9918670654297,
      "loss": 0.6901,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 0.023711280897259712,
      "rewards/margins": 0.013997456058859825,
      "rewards/rejected": 0.009713825769722462,
      "step": 5
    },
    {
      "epoch": 0.18691588785046728,
      "grad_norm": 52.11990471190019,
      "learning_rate": 1e-06,
      "logits/chosen": -2.7212181091308594,
      "logits/rejected": -2.7145252227783203,
      "logps/chosen": -263.0387268066406,
      "logps/rejected": -212.9672088623047,
      "loss": 0.6421,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.5446255803108215,
      "rewards/margins": 0.25802525877952576,
      "rewards/rejected": 0.2866002917289734,
      "step": 10
    },
    {
      "epoch": 0.2803738317757009,
      "grad_norm": 47.29095735423265,
      "learning_rate": 9.972240926774166e-07,
      "logits/chosen": -2.633105516433716,
      "logits/rejected": -2.60020112991333,
      "logps/chosen": -255.480712890625,
      "logps/rejected": -195.80056762695312,
      "loss": 0.6413,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 1.5034700632095337,
      "rewards/margins": 0.6529636383056641,
      "rewards/rejected": 0.850506603717804,
      "step": 15
    },
    {
      "epoch": 0.37383177570093457,
      "grad_norm": 46.52606913589643,
      "learning_rate": 9.889271933555212e-07,
      "logits/chosen": -2.576512336730957,
      "logits/rejected": -2.5848257541656494,
      "logps/chosen": -249.4801025390625,
      "logps/rejected": -228.74624633789062,
      "loss": 0.6341,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 1.605277419090271,
      "rewards/margins": 0.9651613235473633,
      "rewards/rejected": 0.6401159167289734,
      "step": 20
    },
    {
      "epoch": 0.4672897196261682,
      "grad_norm": 46.161897408432196,
      "learning_rate": 9.752014277286431e-07,
      "logits/chosen": -2.643796682357788,
      "logits/rejected": -2.643813133239746,
      "logps/chosen": -244.34548950195312,
      "logps/rejected": -226.01144409179688,
      "loss": 0.5939,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 1.4437496662139893,
      "rewards/margins": 1.0351409912109375,
      "rewards/rejected": 0.4086088240146637,
      "step": 25
    },
    {
      "epoch": 0.5607476635514018,
      "grad_norm": 47.338837202921155,
      "learning_rate": 9.561992016100291e-07,
      "logits/chosen": -2.666031837463379,
      "logits/rejected": -2.6790859699249268,
      "logps/chosen": -245.907958984375,
      "logps/rejected": -238.5924835205078,
      "loss": 0.653,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.311069369316101,
      "rewards/margins": 1.3995612859725952,
      "rewards/rejected": -0.08849197626113892,
      "step": 30
    },
    {
      "epoch": 0.6542056074766355,
      "grad_norm": 37.4693773707841,
      "learning_rate": 9.321315086741915e-07,
      "logits/chosen": -2.7113430500030518,
      "logits/rejected": -2.6925148963928223,
      "logps/chosen": -252.6649932861328,
      "logps/rejected": -224.28744506835938,
      "loss": 0.5817,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.6021201610565186,
      "rewards/margins": 1.8016893863677979,
      "rewards/rejected": -0.19956918060779572,
      "step": 35
    },
    {
      "epoch": 0.7476635514018691,
      "grad_norm": 42.59386751350814,
      "learning_rate": 9.032655876613635e-07,
      "logits/chosen": -2.6804230213165283,
      "logits/rejected": -2.6603970527648926,
      "logps/chosen": -254.66592407226562,
      "logps/rejected": -232.6195831298828,
      "loss": 0.6423,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.9250097274780273,
      "rewards/margins": 1.0544471740722656,
      "rewards/rejected": -0.12943752110004425,
      "step": 40
    },
    {
      "epoch": 0.8411214953271028,
      "grad_norm": 38.94368445871969,
      "learning_rate": 8.699219550575952e-07,
      "logits/chosen": -2.6410422325134277,
      "logits/rejected": -2.620154619216919,
      "logps/chosen": -243.9797821044922,
      "logps/rejected": -212.0509033203125,
      "loss": 0.5545,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 1.005589246749878,
      "rewards/margins": 1.2179254293441772,
      "rewards/rejected": -0.21233627200126648,
      "step": 45
    },
    {
      "epoch": 0.9345794392523364,
      "grad_norm": 32.489482448510415,
      "learning_rate": 8.324708461985124e-07,
      "logits/chosen": -2.5788509845733643,
      "logits/rejected": -2.5809426307678223,
      "logps/chosen": -240.7474365234375,
      "logps/rejected": -201.14662170410156,
      "loss": 0.5653,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.9063326716423035,
      "rewards/margins": 1.26765775680542,
      "rewards/rejected": -0.36132508516311646,
      "step": 50
    },
    {
      "epoch": 0.9345794392523364,
      "eval_logits/chosen": -2.5697171688079834,
      "eval_logits/rejected": -2.5530402660369873,
      "eval_logps/chosen": -250.48052978515625,
      "eval_logps/rejected": -229.00634765625,
      "eval_loss": 0.5525274276733398,
      "eval_rewards/accuracies": 0.765625,
      "eval_rewards/chosen": 0.9592523574829102,
      "eval_rewards/margins": 1.6393791437149048,
      "eval_rewards/rejected": -0.6801266670227051,
      "eval_runtime": 100.9828,
      "eval_samples_per_second": 15.052,
      "eval_steps_per_second": 0.238,
      "step": 50
    },
    {
      "epoch": 1.02803738317757,
      "grad_norm": 24.387699558422295,
      "learning_rate": 7.913281043133977e-07,
      "logits/chosen": -2.5588839054107666,
      "logits/rejected": -2.551605701446533,
      "logps/chosen": -254.15304565429688,
      "logps/rejected": -232.09591674804688,
      "loss": 0.484,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 1.21238112449646,
      "rewards/margins": 1.854658842086792,
      "rewards/rejected": -0.6422780156135559,
      "step": 55
    },
    {
      "epoch": 1.1214953271028036,
      "grad_norm": 16.284840931900277,
      "learning_rate": 7.469505631561317e-07,
      "logits/chosen": -2.478685140609741,
      "logits/rejected": -2.477550983428955,
      "logps/chosen": -256.5254211425781,
      "logps/rejected": -222.71279907226562,
      "loss": 0.2287,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9828132390975952,
      "rewards/margins": 2.876364231109619,
      "rewards/rejected": -0.8935510516166687,
      "step": 60
    },
    {
      "epoch": 1.2149532710280373,
      "grad_norm": 22.220068053555675,
      "learning_rate": 6.998309744925411e-07,
      "logits/chosen": -2.4371962547302246,
      "logits/rejected": -2.4383292198181152,
      "logps/chosen": -245.124267578125,
      "logps/rejected": -238.2715606689453,
      "loss": 0.2741,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.2250008583068848,
      "rewards/margins": 3.028366804122925,
      "rewards/rejected": -0.8033663630485535,
      "step": 65
    },
    {
      "epoch": 1.308411214953271,
      "grad_norm": 25.13810110324404,
      "learning_rate": 6.504925367674594e-07,
      "logits/chosen": -2.443160057067871,
      "logits/rejected": -2.4514541625976562,
      "logps/chosen": -231.23880004882812,
      "logps/rejected": -251.7025909423828,
      "loss": 0.2665,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.0451855659484863,
      "rewards/margins": 3.1943559646606445,
      "rewards/rejected": -1.1491702795028687,
      "step": 70
    },
    {
      "epoch": 1.4018691588785046,
      "grad_norm": 21.55583766195851,
      "learning_rate": 5.994830857031499e-07,
      "logits/chosen": -2.472496747970581,
      "logits/rejected": -2.4373836517333984,
      "logps/chosen": -256.70831298828125,
      "logps/rejected": -240.29275512695312,
      "loss": 0.2784,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": 1.972904920578003,
      "rewards/margins": 3.4955241680145264,
      "rewards/rejected": -1.5226194858551025,
      "step": 75
    },
    {
      "epoch": 1.4953271028037383,
      "grad_norm": 19.53365898582386,
      "learning_rate": 5.473690113345342e-07,
      "logits/chosen": -2.469449043273926,
      "logits/rejected": -2.4470181465148926,
      "logps/chosen": -248.6388397216797,
      "logps/rejected": -195.94500732421875,
      "loss": 0.2449,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.9075500965118408,
      "rewards/margins": 3.1177237033843994,
      "rewards/rejected": -1.210173487663269,
      "step": 80
    },
    {
      "epoch": 1.588785046728972,
      "grad_norm": 23.69859964059544,
      "learning_rate": 4.947289690242102e-07,
      "logits/chosen": -2.502014636993408,
      "logits/rejected": -2.455413341522217,
      "logps/chosen": -249.62973022460938,
      "logps/rejected": -229.01437377929688,
      "loss": 0.2729,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.150496006011963,
      "rewards/margins": 3.2722465991973877,
      "rewards/rejected": -1.121749997138977,
      "step": 85
    },
    {
      "epoch": 1.6822429906542056,
      "grad_norm": 29.464420925123257,
      "learning_rate": 4.421474542878194e-07,
      "logits/chosen": -2.4790539741516113,
      "logits/rejected": -2.463782787322998,
      "logps/chosen": -243.14852905273438,
      "logps/rejected": -244.40896606445312,
      "loss": 0.3061,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 1.6596744060516357,
      "rewards/margins": 3.1171317100524902,
      "rewards/rejected": -1.457457423210144,
      "step": 90
    },
    {
      "epoch": 1.7757009345794392,
      "grad_norm": 23.243574134240056,
      "learning_rate": 3.902083127725186e-07,
      "logits/chosen": -2.506107807159424,
      "logits/rejected": -2.4774811267852783,
      "logps/chosen": -253.5945587158203,
      "logps/rejected": -210.60678100585938,
      "loss": 0.2932,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": 1.9583078622817993,
      "rewards/margins": 3.305584669113159,
      "rewards/rejected": -1.3472764492034912,
      "step": 95
    },
    {
      "epoch": 1.8691588785046729,
      "grad_norm": 23.53393549322468,
      "learning_rate": 3.394882574513519e-07,
      "logits/chosen": -2.5429701805114746,
      "logits/rejected": -2.535693407058716,
      "logps/chosen": -248.3795928955078,
      "logps/rejected": -225.8032684326172,
      "loss": 0.2953,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 1.7190853357315063,
      "rewards/margins": 3.4785995483398438,
      "rewards/rejected": -1.7595138549804688,
      "step": 100
    },
    {
      "epoch": 1.8691588785046729,
      "eval_logits/chosen": -2.573176860809326,
      "eval_logits/rejected": -2.5532119274139404,
      "eval_logps/chosen": -248.56651306152344,
      "eval_logps/rejected": -236.0966796875,
      "eval_loss": 0.5386662483215332,
      "eval_rewards/accuracies": 0.8229166865348816,
      "eval_rewards/chosen": 1.1506575345993042,
      "eval_rewards/margins": 2.5398218631744385,
      "eval_rewards/rejected": -1.3891644477844238,
      "eval_runtime": 100.7243,
      "eval_samples_per_second": 15.091,
      "eval_steps_per_second": 0.238,
      "step": 100
    },
    {
      "epoch": 1.9626168224299065,
      "grad_norm": 22.997540870154687,
      "learning_rate": 2.9055046501619083e-07,
      "logits/chosen": -2.5645899772644043,
      "logits/rejected": -2.5487587451934814,
      "logps/chosen": -222.7235565185547,
      "logps/rejected": -242.82247924804688,
      "loss": 0.2947,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.1347827911376953,
      "rewards/margins": 3.9442334175109863,
      "rewards/rejected": -1.8094507455825806,
      "step": 105
    },
    {
      "epoch": 2.05607476635514,
      "grad_norm": 16.343385074167667,
      "learning_rate": 2.439383225725225e-07,
      "logits/chosen": -2.562004327774048,
      "logits/rejected": -2.5361239910125732,
      "logps/chosen": -238.68008422851562,
      "logps/rejected": -228.2875213623047,
      "loss": 0.2247,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 1.7707144021987915,
      "rewards/margins": 3.44102144241333,
      "rewards/rejected": -1.6703068017959595,
      "step": 110
    },
    {
      "epoch": 2.149532710280374,
      "grad_norm": 15.41052576518734,
      "learning_rate": 2.0016939407046986e-07,
      "logits/chosen": -2.542348623275757,
      "logits/rejected": -2.540949583053589,
      "logps/chosen": -256.00067138671875,
      "logps/rejected": -238.72549438476562,
      "loss": 0.1623,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 1.9835551977157593,
      "rewards/margins": 3.942012071609497,
      "rewards/rejected": -1.9584569931030273,
      "step": 115
    },
    {
      "epoch": 2.2429906542056073,
      "grad_norm": 16.071668642185152,
      "learning_rate": 1.5972967346655448e-07,
      "logits/chosen": -2.5395591259002686,
      "logits/rejected": -2.524195671081543,
      "logps/chosen": -251.2150421142578,
      "logps/rejected": -219.9102020263672,
      "loss": 0.1611,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.183037519454956,
      "rewards/margins": 3.8635833263397217,
      "rewards/rejected": -1.6805460453033447,
      "step": 120
    },
    {
      "epoch": 2.336448598130841,
      "grad_norm": 14.26653941598662,
      "learning_rate": 1.2306818842696715e-07,
      "logits/chosen": -2.5451834201812744,
      "logits/rejected": -2.5221030712127686,
      "logps/chosen": -217.0515594482422,
      "logps/rejected": -225.6378173828125,
      "loss": 0.1695,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 1.7758986949920654,
      "rewards/margins": 3.466110944747925,
      "rewards/rejected": -1.6902124881744385,
      "step": 125
    },
    {
      "epoch": 2.4299065420560746,
      "grad_norm": 17.883690816294713,
      "learning_rate": 9.059201449082043e-08,
      "logits/chosen": -2.5324203968048096,
      "logits/rejected": -2.497401475906372,
      "logps/chosen": -237.7636260986328,
      "logps/rejected": -231.6305389404297,
      "loss": 0.1788,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.1109304428100586,
      "rewards/margins": 4.035120964050293,
      "rewards/rejected": -1.9241914749145508,
      "step": 130
    },
    {
      "epoch": 2.5233644859813085,
      "grad_norm": 11.508052610248667,
      "learning_rate": 6.266175505426957e-08,
      "logits/chosen": -2.5180368423461914,
      "logits/rejected": -2.4927144050598145,
      "logps/chosen": -240.50064086914062,
      "logps/rejected": -262.6056823730469,
      "loss": 0.1525,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.23576283454895,
      "rewards/margins": 4.589751243591309,
      "rewards/rejected": -2.3539886474609375,
      "step": 135
    },
    {
      "epoch": 2.616822429906542,
      "grad_norm": 16.438365430086915,
      "learning_rate": 3.958753736408105e-08,
      "logits/chosen": -2.4942755699157715,
      "logits/rejected": -2.4786365032196045,
      "logps/chosen": -243.6421356201172,
      "logps/rejected": -258.7626037597656,
      "loss": 0.1404,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.1195948123931885,
      "rewards/margins": 4.04628324508667,
      "rewards/rejected": -1.9266889095306396,
      "step": 140
    },
    {
      "epoch": 2.710280373831776,
      "grad_norm": 19.86719872132447,
      "learning_rate": 2.162556897965101e-08,
      "logits/chosen": -2.4958689212799072,
      "logits/rejected": -2.494823455810547,
      "logps/chosen": -238.25851440429688,
      "logps/rejected": -243.26904296875,
      "loss": 0.1805,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 2.2601914405822754,
      "rewards/margins": 3.9592623710632324,
      "rewards/rejected": -1.6990711688995361,
      "step": 145
    },
    {
      "epoch": 2.803738317757009,
      "grad_norm": 13.411487071212091,
      "learning_rate": 8.975292939244927e-09,
      "logits/chosen": -2.489874839782715,
      "logits/rejected": -2.46675705909729,
      "logps/chosen": -218.33712768554688,
      "logps/rejected": -225.78857421875,
      "loss": 0.1521,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 2.140179395675659,
      "rewards/margins": 4.0314483642578125,
      "rewards/rejected": -1.8912690877914429,
      "step": 150
    },
    {
      "epoch": 2.803738317757009,
      "eval_logits/chosen": -2.4992964267730713,
      "eval_logits/rejected": -2.4707489013671875,
      "eval_logps/chosen": -247.51153564453125,
      "eval_logps/rejected": -236.0965118408203,
      "eval_loss": 0.5317534804344177,
      "eval_rewards/accuracies": 0.8072916865348816,
      "eval_rewards/chosen": 1.2561508417129517,
      "eval_rewards/margins": 2.6452977657318115,
      "eval_rewards/rejected": -1.3891468048095703,
      "eval_runtime": 100.9942,
      "eval_samples_per_second": 15.05,
      "eval_steps_per_second": 0.238,
      "step": 150
    },
    {
      "epoch": 2.897196261682243,
      "grad_norm": 17.267692644524978,
      "learning_rate": 1.7771732184357901e-09,
      "logits/chosen": -2.5087661743164062,
      "logits/rejected": -2.4657070636749268,
      "logps/chosen": -240.25390625,
      "logps/rejected": -254.7338104248047,
      "loss": 0.163,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.3995561599731445,
      "rewards/margins": 4.272937774658203,
      "rewards/rejected": -1.8733813762664795,
      "step": 155
    },
    {
      "epoch": 2.97196261682243,
      "step": 159,
      "total_flos": 1874604243025920.0,
      "train_loss": 0.35437198479970294,
      "train_runtime": 5742.7927,
      "train_samples_per_second": 7.144,
      "train_steps_per_second": 0.028
    }
  ],
  "logging_steps": 5,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 1874604243025920.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}