{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.8695652173913043,
  "eval_steps": 40,
  "global_step": 120,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.036231884057971016,
      "grad_norm": 61.66789827681496,
      "learning_rate": 5e-07,
      "logits/chosen": -2.7321553230285645,
      "logits/rejected": -2.7100937366485596,
      "logps/chosen": -182.5845489501953,
      "logps/rejected": -189.55001831054688,
      "loss": 0.689,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.0021577859297394753,
      "rewards/margins": 0.005646524019539356,
      "rewards/rejected": -0.007804309483617544,
      "step": 5
    },
    {
      "epoch": 0.07246376811594203,
      "grad_norm": 44.96703657621111,
      "learning_rate": 1e-06,
      "logits/chosen": -2.753889799118042,
      "logits/rejected": -2.7519516944885254,
      "logps/chosen": -197.34320068359375,
      "logps/rejected": -184.00961303710938,
      "loss": 0.6275,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.030874451622366905,
      "rewards/margins": 0.18904080986976624,
      "rewards/rejected": -0.15816636383533478,
      "step": 10
    },
    {
      "epoch": 0.10869565217391304,
      "grad_norm": 51.54402636298773,
      "learning_rate": 9.996221126793764e-07,
      "logits/chosen": -2.694437265396118,
      "logits/rejected": -2.691904067993164,
      "logps/chosen": -203.14883422851562,
      "logps/rejected": -204.52386474609375,
      "loss": 0.5851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.6205412745475769,
      "rewards/margins": 0.9350436925888062,
      "rewards/rejected": -0.314502477645874,
      "step": 15
    },
    {
      "epoch": 0.14492753623188406,
      "grad_norm": 35.11282014809116,
      "learning_rate": 9.984890219128145e-07,
      "logits/chosen": -2.609405517578125,
      "logits/rejected": -2.5795176029205322,
      "logps/chosen": -188.33395385742188,
      "logps/rejected": -192.52633666992188,
      "loss": 0.5144,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.8838651776313782,
      "rewards/margins": 1.275496244430542,
      "rewards/rejected": -0.39163118600845337,
      "step": 20
    },
    {
      "epoch": 0.18115942028985507,
      "grad_norm": 37.05591291167376,
      "learning_rate": 9.966024404228493e-07,
      "logits/chosen": -2.4429798126220703,
      "logits/rejected": -2.4225034713745117,
      "logps/chosen": -179.79977416992188,
      "logps/rejected": -179.25279235839844,
      "loss": 0.5036,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.40532931685447693,
      "rewards/margins": 0.8947975039482117,
      "rewards/rejected": -0.4894680976867676,
      "step": 25
    },
    {
      "epoch": 0.21739130434782608,
      "grad_norm": 33.27618771325657,
      "learning_rate": 9.939652198703783e-07,
      "logits/chosen": -2.319044589996338,
      "logits/rejected": -2.320253372192383,
      "logps/chosen": -188.62039184570312,
      "logps/rejected": -193.8806915283203,
      "loss": 0.4987,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.6481167078018188,
      "rewards/margins": 1.2183058261871338,
      "rewards/rejected": -0.5701891183853149,
      "step": 30
    },
    {
      "epoch": 0.2536231884057971,
      "grad_norm": 38.645073192304736,
      "learning_rate": 9.905813465442354e-07,
      "logits/chosen": -2.242053747177124,
      "logits/rejected": -2.2155580520629883,
      "logps/chosen": -203.94546508789062,
      "logps/rejected": -194.8705291748047,
      "loss": 0.5091,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.8871867060661316,
      "rewards/margins": 1.2737131118774414,
      "rewards/rejected": -0.3865264356136322,
      "step": 35
    },
    {
      "epoch": 0.2898550724637681,
      "grad_norm": 30.937642063399068,
      "learning_rate": 9.864559353357187e-07,
      "logits/chosen": -2.0998620986938477,
      "logits/rejected": -2.094836473464966,
      "logps/chosen": -182.6898956298828,
      "logps/rejected": -185.73983764648438,
      "loss": 0.4889,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.0410809516906738,
      "rewards/margins": 1.0222995281219482,
      "rewards/rejected": 0.018781563267111778,
      "step": 40
    },
    {
      "epoch": 0.2898550724637681,
      "eval_logits/chosen": -2.035081624984741,
      "eval_logits/rejected": -1.9875677824020386,
      "eval_logps/chosen": -188.77523803710938,
      "eval_logps/rejected": -200.09498596191406,
      "eval_loss": 0.4642000198364258,
      "eval_rewards/accuracies": 0.7943548560142517,
      "eval_rewards/chosen": 1.1544201374053955,
      "eval_rewards/margins": 1.3431099653244019,
      "eval_rewards/rejected": -0.18868987262248993,
      "eval_runtime": 248.6586,
      "eval_samples_per_second": 15.773,
      "eval_steps_per_second": 0.249,
      "step": 40
    },
    {
      "epoch": 0.32608695652173914,
      "grad_norm": 29.205255063362404,
      "learning_rate": 9.815952220071804e-07,
      "logits/chosen": -1.9733244180679321,
      "logits/rejected": -1.9242098331451416,
      "logps/chosen": -195.65652465820312,
      "logps/rejected": -221.744384765625,
      "loss": 0.4587,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 1.3803393840789795,
      "rewards/margins": 1.8610279560089111,
      "rewards/rejected": -0.4806883931159973,
      "step": 45
    },
    {
      "epoch": 0.36231884057971014,
      "grad_norm": 29.678687938529745,
      "learning_rate": 9.76006553766365e-07,
      "logits/chosen": -1.8085625171661377,
      "logits/rejected": -1.772071123123169,
      "logps/chosen": -198.9405517578125,
      "logps/rejected": -203.29978942871094,
      "loss": 0.4531,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.8211376070976257,
      "rewards/margins": 1.646414041519165,
      "rewards/rejected": -0.8252763748168945,
      "step": 50
    },
    {
      "epoch": 0.39855072463768115,
      "grad_norm": 30.73732601293307,
      "learning_rate": 9.696983781607415e-07,
      "logits/chosen": -1.7908868789672852,
      "logits/rejected": -1.7599233388900757,
      "logps/chosen": -183.01809692382812,
      "logps/rejected": -172.02801513671875,
      "loss": 0.4595,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": 0.607209324836731,
      "rewards/margins": 1.649515151977539,
      "rewards/rejected": -1.042305827140808,
      "step": 55
    },
    {
      "epoch": 0.43478260869565216,
      "grad_norm": 41.49881496846909,
      "learning_rate": 9.626802303086209e-07,
      "logits/chosen": -1.9050066471099854,
      "logits/rejected": -1.8625679016113281,
      "logps/chosen": -186.8179931640625,
      "logps/rejected": -194.26707458496094,
      "loss": 0.4276,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.4140642285346985,
      "rewards/margins": 1.712418794631958,
      "rewards/rejected": -1.2983543872833252,
      "step": 60
    },
    {
      "epoch": 0.47101449275362317,
      "grad_norm": 36.480881476794956,
      "learning_rate": 9.549627184863528e-07,
      "logits/chosen": -2.1638290882110596,
      "logits/rejected": -2.0793471336364746,
      "logps/chosen": -191.9010467529297,
      "logps/rejected": -193.20651245117188,
      "loss": 0.4293,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": 0.045776158571243286,
      "rewards/margins": 1.6033704280853271,
      "rewards/rejected": -1.5575940608978271,
      "step": 65
    },
    {
      "epoch": 0.5072463768115942,
      "grad_norm": 26.318712537847357,
      "learning_rate": 9.465575080933957e-07,
      "logits/chosen": -2.1214632987976074,
      "logits/rejected": -2.081162214279175,
      "logps/chosen": -172.66419982910156,
      "logps/rejected": -208.19149780273438,
      "loss": 0.3963,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.17696644365787506,
      "rewards/margins": 1.8183233737945557,
      "rewards/rejected": -1.6413570642471313,
      "step": 70
    },
    {
      "epoch": 0.5434782608695652,
      "grad_norm": 32.35315227999789,
      "learning_rate": 9.374773040194878e-07,
      "logits/chosen": -2.1271562576293945,
      "logits/rejected": -2.0680038928985596,
      "logps/chosen": -205.5594482421875,
      "logps/rejected": -210.7834014892578,
      "loss": 0.4353,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.16769471764564514,
      "rewards/margins": 1.786273717880249,
      "rewards/rejected": -1.6185792684555054,
      "step": 75
    },
    {
      "epoch": 0.5797101449275363,
      "grad_norm": 29.96593011887677,
      "learning_rate": 9.277358314405818e-07,
      "logits/chosen": -2.046504497528076,
      "logits/rejected": -2.0122628211975098,
      "logps/chosen": -189.37771606445312,
      "logps/rejected": -206.10659790039062,
      "loss": 0.3941,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2374907284975052,
      "rewards/margins": 1.910300612449646,
      "rewards/rejected": -2.1477913856506348,
      "step": 80
    },
    {
      "epoch": 0.5797101449275363,
      "eval_logits/chosen": -1.9900643825531006,
      "eval_logits/rejected": -1.9449083805084229,
      "eval_logps/chosen": -202.59437561035156,
      "eval_logps/rejected": -221.12725830078125,
      "eval_loss": 0.4217630624771118,
      "eval_rewards/accuracies": 0.8044354915618896,
      "eval_rewards/chosen": -0.22749020159244537,
      "eval_rewards/margins": 2.0644266605377197,
      "eval_rewards/rejected": -2.291916847229004,
      "eval_runtime": 247.3121,
      "eval_samples_per_second": 15.859,
      "eval_steps_per_second": 0.251,
      "step": 80
    },
    {
      "epoch": 0.6159420289855072,
      "grad_norm": 34.207898226903815,
      "learning_rate": 9.173478150725651e-07,
      "logits/chosen": -2.0169568061828613,
      "logits/rejected": -1.9391515254974365,
      "logps/chosen": -209.73440551757812,
      "logps/rejected": -215.14205932617188,
      "loss": 0.422,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": 0.1130056157708168,
      "rewards/margins": 2.3998470306396484,
      "rewards/rejected": -2.2868411540985107,
      "step": 85
    },
    {
      "epoch": 0.6521739130434783,
      "grad_norm": 28.850805260840193,
      "learning_rate": 9.063289569141251e-07,
      "logits/chosen": -2.1180360317230225,
      "logits/rejected": -2.080235242843628,
      "logps/chosen": -214.2292022705078,
      "logps/rejected": -223.37564086914062,
      "loss": 0.4135,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.5250645279884338,
      "rewards/margins": 2.2779579162597656,
      "rewards/rejected": -1.7528936862945557,
      "step": 90
    },
    {
      "epoch": 0.6884057971014492,
      "grad_norm": 34.73111091101309,
      "learning_rate": 8.946959125124051e-07,
      "logits/chosen": -2.2047152519226074,
      "logits/rejected": -2.1581788063049316,
      "logps/chosen": -209.3234405517578,
      "logps/rejected": -195.6946563720703,
      "loss": 0.4077,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": 0.25443512201309204,
      "rewards/margins": 2.2029635906219482,
      "rewards/rejected": -1.948528528213501,
      "step": 95
    },
    {
      "epoch": 0.7246376811594203,
      "grad_norm": 32.202226649445414,
      "learning_rate": 8.824662657873238e-07,
      "logits/chosen": -2.256134510040283,
      "logits/rejected": -2.2395310401916504,
      "logps/chosen": -176.19464111328125,
      "logps/rejected": -209.0999298095703,
      "loss": 0.3767,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.3457742929458618,
      "rewards/margins": 1.9383150339126587,
      "rewards/rejected": -2.2840893268585205,
      "step": 100
    },
    {
      "epoch": 0.7608695652173914,
      "grad_norm": 31.35693490170537,
      "learning_rate": 8.696585024526135e-07,
      "logits/chosen": -2.3358142375946045,
      "logits/rejected": -2.307068347930908,
      "logps/chosen": -191.8298797607422,
      "logps/rejected": -227.55612182617188,
      "loss": 0.4017,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2825666069984436,
      "rewards/margins": 2.5402400493621826,
      "rewards/rejected": -2.8228065967559814,
      "step": 105
    },
    {
      "epoch": 0.7971014492753623,
      "grad_norm": 36.29919961993296,
      "learning_rate": 8.562919820737535e-07,
      "logits/chosen": -2.2931602001190186,
      "logits/rejected": -2.2568557262420654,
      "logps/chosen": -207.34158325195312,
      "logps/rejected": -208.9143524169922,
      "loss": 0.3767,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.3017815351486206,
      "rewards/margins": 2.3830084800720215,
      "rewards/rejected": -2.6847901344299316,
      "step": 110
    },
    {
      "epoch": 0.8333333333333334,
      "grad_norm": 28.819478551304137,
      "learning_rate": 8.423869088050315e-07,
      "logits/chosen": -2.2734763622283936,
      "logits/rejected": -2.2522785663604736,
      "logps/chosen": -195.6924591064453,
      "logps/rejected": -221.85104370117188,
      "loss": 0.3892,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.12773282825946808,
      "rewards/margins": 2.3888843059539795,
      "rewards/rejected": -2.5166170597076416,
      "step": 115
    },
    {
      "epoch": 0.8695652173913043,
      "grad_norm": 37.085951928654964,
      "learning_rate": 8.2796430084997e-07,
      "logits/chosen": -2.2070839405059814,
      "logits/rejected": -2.1736464500427246,
      "logps/chosen": -196.70889282226562,
      "logps/rejected": -207.93417358398438,
      "loss": 0.3717,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.12862932682037354,
      "rewards/margins": 2.698848009109497,
      "rewards/rejected": -2.570218563079834,
      "step": 120
    },
    {
      "epoch": 0.8695652173913043,
      "eval_logits/chosen": -2.122933864593506,
      "eval_logits/rejected": -2.0901710987091064,
      "eval_logps/chosen": -202.4208221435547,
      "eval_logps/rejected": -223.09356689453125,
      "eval_loss": 0.438678115606308,
      "eval_rewards/accuracies": 0.8286290168762207,
      "eval_rewards/chosen": -0.2101391851902008,
      "eval_rewards/margins": 2.278407573699951,
      "eval_rewards/rejected": -2.488546848297119,
      "eval_runtime": 247.3754,
      "eval_samples_per_second": 15.854,
      "eval_steps_per_second": 0.251,
      "step": 120
    }
  ],
  "logging_steps": 5,
  "max_steps": 414,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 40,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1414680891359232.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}