{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.97196261682243,
  "eval_steps": 50,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.09345794392523364,
      "grad_norm": 64.25907086156798,
      "learning_rate": 5e-07,
      "logits/chosen": -2.7352654933929443,
      "logits/rejected": -2.715487003326416,
      "logps/chosen": -269.43072509765625,
      "logps/rejected": -218.1147918701172,
      "loss": 0.6899,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 0.018381251022219658,
      "rewards/margins": 0.008254295215010643,
      "rewards/rejected": 0.010126957669854164,
      "step": 5
    },
    {
      "epoch": 0.18691588785046728,
      "grad_norm": 46.235718976229606,
      "learning_rate": 1e-06,
      "logits/chosen": -2.6723597049713135,
      "logits/rejected": -2.683154821395874,
      "logps/chosen": -270.26348876953125,
      "logps/rejected": -199.8683624267578,
      "loss": 0.6442,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.5999149084091187,
      "rewards/margins": 0.22199003398418427,
      "rewards/rejected": 0.3779248595237732,
      "step": 10
    },
    {
      "epoch": 0.2803738317757009,
      "grad_norm": 40.05093234219346,
      "learning_rate": 9.972240926774166e-07,
      "logits/chosen": -2.583867073059082,
      "logits/rejected": -2.5568575859069824,
      "logps/chosen": -238.4031982421875,
      "logps/rejected": -207.7652130126953,
      "loss": 0.614,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 1.6396440267562866,
      "rewards/margins": 0.7280690670013428,
      "rewards/rejected": 0.9115749597549438,
      "step": 15
    },
    {
      "epoch": 0.37383177570093457,
      "grad_norm": 46.527169597733305,
      "learning_rate": 9.889271933555212e-07,
      "logits/chosen": -2.450702667236328,
      "logits/rejected": -2.4366519451141357,
      "logps/chosen": -239.6464080810547,
      "logps/rejected": -188.81138610839844,
      "loss": 0.5975,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 1.8856239318847656,
      "rewards/margins": 0.852216362953186,
      "rewards/rejected": 1.0334076881408691,
      "step": 20
    },
    {
      "epoch": 0.4672897196261682,
      "grad_norm": 40.30005689440578,
      "learning_rate": 9.752014277286431e-07,
      "logits/chosen": -2.350933790206909,
      "logits/rejected": -2.3373398780822754,
      "logps/chosen": -251.7142791748047,
      "logps/rejected": -187.19984436035156,
      "loss": 0.566,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 2.030355215072632,
      "rewards/margins": 1.2459053993225098,
      "rewards/rejected": 0.7844498157501221,
      "step": 25
    },
    {
      "epoch": 0.5607476635514018,
      "grad_norm": 38.93034965929662,
      "learning_rate": 9.561992016100291e-07,
      "logits/chosen": -2.2758495807647705,
      "logits/rejected": -2.2311055660247803,
      "logps/chosen": -269.446044921875,
      "logps/rejected": -203.4172821044922,
      "loss": 0.558,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 1.7135908603668213,
      "rewards/margins": 1.3296939134597778,
      "rewards/rejected": 0.3838968873023987,
      "step": 30
    },
    {
      "epoch": 0.6542056074766355,
      "grad_norm": 46.97283156919659,
      "learning_rate": 9.321315086741915e-07,
      "logits/chosen": -2.253507137298584,
      "logits/rejected": -2.221372127532959,
      "logps/chosen": -247.94827270507812,
      "logps/rejected": -200.51922607421875,
      "loss": 0.579,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 1.2021849155426025,
      "rewards/margins": 1.4166221618652344,
      "rewards/rejected": -0.21443717181682587,
      "step": 35
    },
    {
      "epoch": 0.7476635514018691,
      "grad_norm": 44.407618951863526,
      "learning_rate": 9.032655876613635e-07,
      "logits/chosen": -2.3225274085998535,
      "logits/rejected": -2.321155071258545,
      "logps/chosen": -245.7095947265625,
      "logps/rejected": -196.00083923339844,
      "loss": 0.5802,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.9343897104263306,
      "rewards/margins": 1.0495737791061401,
      "rewards/rejected": -0.11518410593271255,
      "step": 40
    },
    {
      "epoch": 0.8411214953271028,
      "grad_norm": 35.484899569132175,
      "learning_rate": 8.699219550575952e-07,
      "logits/chosen": -2.3453164100646973,
      "logits/rejected": -2.3334288597106934,
      "logps/chosen": -239.4101104736328,
      "logps/rejected": -217.1162109375,
      "loss": 0.538,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 1.2655799388885498,
      "rewards/margins": 1.8225781917572021,
      "rewards/rejected": -0.5569985508918762,
      "step": 45
    },
    {
      "epoch": 0.9345794392523364,
      "grad_norm": 38.47661119202761,
      "learning_rate": 8.324708461985124e-07,
      "logits/chosen": -2.336545944213867,
      "logits/rejected": -2.3381569385528564,
      "logps/chosen": -242.3145751953125,
      "logps/rejected": -205.20687866210938,
      "loss": 0.5069,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 1.2903130054473877,
      "rewards/margins": 1.6887279748916626,
      "rewards/rejected": -0.39841485023498535,
      "step": 50
    },
    {
      "epoch": 0.9345794392523364,
      "eval_logits/chosen": -2.3625147342681885,
      "eval_logits/rejected": -2.3323090076446533,
      "eval_logps/chosen": -268.0925598144531,
      "eval_logps/rejected": -191.181396484375,
      "eval_loss": 0.544481635093689,
      "eval_rewards/accuracies": 0.7708333134651184,
      "eval_rewards/chosen": 1.1549440622329712,
      "eval_rewards/margins": 1.484925627708435,
      "eval_rewards/rejected": -0.3299814760684967,
      "eval_runtime": 101.1546,
      "eval_samples_per_second": 15.027,
      "eval_steps_per_second": 0.237,
      "step": 50
    },
    {
      "epoch": 1.02803738317757,
      "grad_norm": 21.885999790849485,
      "learning_rate": 7.913281043133977e-07,
      "logits/chosen": -2.3184032440185547,
      "logits/rejected": -2.2938473224639893,
      "logps/chosen": -250.3560028076172,
      "logps/rejected": -233.30630493164062,
      "loss": 0.4787,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 1.2203357219696045,
      "rewards/margins": 2.1394553184509277,
      "rewards/rejected": -0.9191198348999023,
      "step": 55
    },
    {
      "epoch": 1.1214953271028036,
      "grad_norm": 14.465910382172174,
      "learning_rate": 7.469505631561317e-07,
      "logits/chosen": -2.2429802417755127,
      "logits/rejected": -2.2311806678771973,
      "logps/chosen": -239.5115966796875,
      "logps/rejected": -209.9403076171875,
      "loss": 0.2297,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 1.8955681324005127,
      "rewards/margins": 3.0370676517486572,
      "rewards/rejected": -1.1414995193481445,
      "step": 60
    },
    {
      "epoch": 1.2149532710280373,
      "grad_norm": 22.768586089667867,
      "learning_rate": 6.998309744925411e-07,
      "logits/chosen": -2.2206153869628906,
      "logits/rejected": -2.1977906227111816,
      "logps/chosen": -241.6571044921875,
      "logps/rejected": -230.4139404296875,
      "loss": 0.2463,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 2.490936756134033,
      "rewards/margins": 3.5978896617889404,
      "rewards/rejected": -1.1069526672363281,
      "step": 65
    },
    {
      "epoch": 1.308411214953271,
      "grad_norm": 21.234164671655243,
      "learning_rate": 6.504925367674594e-07,
      "logits/chosen": -2.2947399616241455,
      "logits/rejected": -2.2711234092712402,
      "logps/chosen": -247.6901397705078,
      "logps/rejected": -232.25143432617188,
      "loss": 0.2289,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 2.7376725673675537,
      "rewards/margins": 4.084096431732178,
      "rewards/rejected": -1.3464237451553345,
      "step": 70
    },
    {
      "epoch": 1.4018691588785046,
      "grad_norm": 23.214123112548595,
      "learning_rate": 5.994830857031499e-07,
      "logits/chosen": -2.3061466217041016,
      "logits/rejected": -2.2935497760772705,
      "logps/chosen": -235.4070587158203,
      "logps/rejected": -238.2843475341797,
      "loss": 0.2618,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": 2.649362802505493,
      "rewards/margins": 3.7771477699279785,
      "rewards/rejected": -1.1277848482131958,
      "step": 75
    },
    {
      "epoch": 1.4953271028037383,
      "grad_norm": 23.002370346021507,
      "learning_rate": 5.473690113345342e-07,
      "logits/chosen": -2.30381441116333,
      "logits/rejected": -2.2565557956695557,
      "logps/chosen": -241.03811645507812,
      "logps/rejected": -209.29025268554688,
      "loss": 0.2933,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 2.297919750213623,
      "rewards/margins": 3.4454472064971924,
      "rewards/rejected": -1.1475274562835693,
      "step": 80
    },
    {
      "epoch": 1.588785046728972,
      "grad_norm": 17.986443749085126,
      "learning_rate": 4.947289690242102e-07,
      "logits/chosen": -2.259481906890869,
      "logits/rejected": -2.2490382194519043,
      "logps/chosen": -242.26708984375,
      "logps/rejected": -219.21975708007812,
      "loss": 0.2607,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.5205655097961426,
      "rewards/margins": 3.4021854400634766,
      "rewards/rejected": -0.8816197514533997,
      "step": 85
    },
    {
      "epoch": 1.6822429906542056,
      "grad_norm": 23.527129100232212,
      "learning_rate": 4.421474542878194e-07,
      "logits/chosen": -2.250342845916748,
      "logits/rejected": -2.206367015838623,
      "logps/chosen": -218.79141235351562,
      "logps/rejected": -249.013916015625,
      "loss": 0.2821,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 2.248781442642212,
      "rewards/margins": 3.799814224243164,
      "rewards/rejected": -1.5510326623916626,
      "step": 90
    },
    {
      "epoch": 1.7757009345794392,
      "grad_norm": 20.08726736786646,
      "learning_rate": 3.902083127725186e-07,
      "logits/chosen": -2.210190534591675,
      "logits/rejected": -2.192523241043091,
      "logps/chosen": -221.87985229492188,
      "logps/rejected": -220.18179321289062,
      "loss": 0.2499,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 2.266261577606201,
      "rewards/margins": 3.49079966545105,
      "rewards/rejected": -1.224537968635559,
      "step": 95
    },
    {
      "epoch": 1.8691588785046729,
      "grad_norm": 23.627843810647025,
      "learning_rate": 3.394882574513519e-07,
      "logits/chosen": -2.202239990234375,
      "logits/rejected": -2.1847739219665527,
      "logps/chosen": -247.0044708251953,
      "logps/rejected": -197.84573364257812,
      "loss": 0.33,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 2.2673051357269287,
      "rewards/margins": 3.611363649368286,
      "rewards/rejected": -1.3440587520599365,
      "step": 100
    },
    {
      "epoch": 1.8691588785046729,
      "eval_logits/chosen": -2.227185010910034,
      "eval_logits/rejected": -2.1957762241363525,
      "eval_logps/chosen": -261.9975891113281,
      "eval_logps/rejected": -194.4145965576172,
      "eval_loss": 0.5182002186775208,
      "eval_rewards/accuracies": 0.8229166865348816,
      "eval_rewards/chosen": 1.7644418478012085,
      "eval_rewards/margins": 2.417741060256958,
      "eval_rewards/rejected": -0.6532991528511047,
      "eval_runtime": 100.7777,
      "eval_samples_per_second": 15.083,
      "eval_steps_per_second": 0.238,
      "step": 100
    },
    {
      "epoch": 1.9626168224299065,
      "grad_norm": 21.751449762111058,
      "learning_rate": 2.9055046501619083e-07,
      "logits/chosen": -2.224174976348877,
      "logits/rejected": -2.170074701309204,
      "logps/chosen": -228.6382293701172,
      "logps/rejected": -245.8506317138672,
      "loss": 0.3113,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 2.3702425956726074,
      "rewards/margins": 3.569148302078247,
      "rewards/rejected": -1.1989054679870605,
      "step": 105
    },
    {
      "epoch": 2.05607476635514,
      "grad_norm": 14.770503354093574,
      "learning_rate": 2.439383225725225e-07,
      "logits/chosen": -2.189666986465454,
      "logits/rejected": -2.1793408393859863,
      "logps/chosen": -235.7542724609375,
      "logps/rejected": -218.64761352539062,
      "loss": 0.1889,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": 2.4319849014282227,
      "rewards/margins": 3.8941452503204346,
      "rewards/rejected": -1.4621602296829224,
      "step": 110
    },
    {
      "epoch": 2.149532710280374,
      "grad_norm": 12.833265134737909,
      "learning_rate": 2.0016939407046986e-07,
      "logits/chosen": -2.2413907051086426,
      "logits/rejected": -2.1873061656951904,
      "logps/chosen": -228.26724243164062,
      "logps/rejected": -209.82168579101562,
      "loss": 0.1437,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 2.307067394256592,
      "rewards/margins": 3.9373679161071777,
      "rewards/rejected": -1.6303008794784546,
      "step": 115
    },
    {
      "epoch": 2.2429906542056073,
      "grad_norm": 11.822968447156347,
      "learning_rate": 1.5972967346655448e-07,
      "logits/chosen": -2.276632308959961,
      "logits/rejected": -2.248224973678589,
      "logps/chosen": -223.19857788085938,
      "logps/rejected": -219.5464630126953,
      "loss": 0.1335,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": 2.4452242851257324,
      "rewards/margins": 3.830425977706909,
      "rewards/rejected": -1.3852016925811768,
      "step": 120
    },
    {
      "epoch": 2.336448598130841,
      "grad_norm": 18.39765754158861,
      "learning_rate": 1.2306818842696715e-07,
      "logits/chosen": -2.324186325073242,
      "logits/rejected": -2.2840449810028076,
      "logps/chosen": -234.97903442382812,
      "logps/rejected": -232.93582153320312,
      "loss": 0.1592,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": 2.6819796562194824,
      "rewards/margins": 4.276698112487793,
      "rewards/rejected": -1.5947178602218628,
      "step": 125
    },
    {
      "epoch": 2.4299065420560746,
      "grad_norm": 15.47036404967669,
      "learning_rate": 9.059201449082043e-08,
      "logits/chosen": -2.314243793487549,
      "logits/rejected": -2.2900912761688232,
      "logps/chosen": -240.45388793945312,
      "logps/rejected": -212.1385498046875,
      "loss": 0.1483,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 2.4335389137268066,
      "rewards/margins": 3.873295307159424,
      "rewards/rejected": -1.4397567510604858,
      "step": 130
    },
    {
      "epoch": 2.5233644859813085,
      "grad_norm": 19.170309374619883,
      "learning_rate": 6.266175505426957e-08,
      "logits/chosen": -2.283237934112549,
      "logits/rejected": -2.266099691390991,
      "logps/chosen": -241.7128448486328,
      "logps/rejected": -223.68429565429688,
      "loss": 0.155,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 2.2079215049743652,
      "rewards/margins": 4.085274696350098,
      "rewards/rejected": -1.8773536682128906,
      "step": 135
    },
    {
      "epoch": 2.616822429906542,
      "grad_norm": 14.296438566762903,
      "learning_rate": 3.958753736408105e-08,
      "logits/chosen": -2.3092122077941895,
      "logits/rejected": -2.2786989212036133,
      "logps/chosen": -230.52639770507812,
      "logps/rejected": -231.3815460205078,
      "loss": 0.1404,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.2946927547454834,
      "rewards/margins": 4.114151954650879,
      "rewards/rejected": -1.8194587230682373,
      "step": 140
    },
    {
      "epoch": 2.710280373831776,
      "grad_norm": 16.63743908750786,
      "learning_rate": 2.162556897965101e-08,
      "logits/chosen": -2.3131165504455566,
      "logits/rejected": -2.294689178466797,
      "logps/chosen": -237.7254180908203,
      "logps/rejected": -216.6897430419922,
      "loss": 0.158,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": 2.0540032386779785,
      "rewards/margins": 3.7803573608398438,
      "rewards/rejected": -1.7263540029525757,
      "step": 145
    },
    {
      "epoch": 2.803738317757009,
      "grad_norm": 16.098164019474485,
      "learning_rate": 8.975292939244927e-09,
      "logits/chosen": -2.31139874458313,
      "logits/rejected": -2.2886271476745605,
      "logps/chosen": -231.0320281982422,
      "logps/rejected": -239.72476196289062,
      "loss": 0.1617,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 2.205547332763672,
      "rewards/margins": 4.294602870941162,
      "rewards/rejected": -2.0890560150146484,
      "step": 150
    },
    {
      "epoch": 2.803738317757009,
      "eval_logits/chosen": -2.32828426361084,
      "eval_logits/rejected": -2.301532030105591,
      "eval_logps/chosen": -264.09381103515625,
      "eval_logps/rejected": -198.0356903076172,
      "eval_loss": 0.5213030576705933,
      "eval_rewards/accuracies": 0.8177083134651184,
      "eval_rewards/chosen": 1.5548186302185059,
      "eval_rewards/margins": 2.570230484008789,
      "eval_rewards/rejected": -1.0154114961624146,
      "eval_runtime": 100.5048,
      "eval_samples_per_second": 15.124,
      "eval_steps_per_second": 0.239,
      "step": 150
    },
    {
      "epoch": 2.897196261682243,
      "grad_norm": 15.518735395097615,
      "learning_rate": 1.7771732184357901e-09,
      "logits/chosen": -2.3752760887145996,
      "logits/rejected": -2.2865824699401855,
      "logps/chosen": -246.28573608398438,
      "logps/rejected": -245.54965209960938,
      "loss": 0.1333,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 2.6093807220458984,
      "rewards/margins": 4.653998374938965,
      "rewards/rejected": -2.044617176055908,
      "step": 155
    },
    {
      "epoch": 2.97196261682243,
      "step": 159,
      "total_flos": 1874604243025920.0,
      "train_loss": 0.33642754952112836,
      "train_runtime": 6089.7617,
      "train_samples_per_second": 6.737,
      "train_steps_per_second": 0.026
    }
  ],
  "logging_steps": 5,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 1874604243025920.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}