File size: 12,475 Bytes
c9061bf |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.4925373134328357,
"eval_steps": 50,
"global_step": 100,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.07462686567164178,
"grad_norm": 53.89263578664345,
"learning_rate": 5e-07,
"logits/chosen": -2.747152805328369,
"logits/rejected": -2.7397713661193848,
"logps/chosen": -254.6358642578125,
"logps/rejected": -205.4340057373047,
"loss": 0.6908,
"rewards/accuracies": 0.3375000059604645,
"rewards/chosen": 0.01600971445441246,
"rewards/margins": 0.003508391324430704,
"rewards/rejected": 0.012501319870352745,
"step": 5
},
{
"epoch": 0.14925373134328357,
"grad_norm": 51.3561964257765,
"learning_rate": 1e-06,
"logits/chosen": -2.701726198196411,
"logits/rejected": -2.6838648319244385,
"logps/chosen": -239.114501953125,
"logps/rejected": -208.04202270507812,
"loss": 0.6434,
"rewards/accuracies": 0.637499988079071,
"rewards/chosen": 0.47272032499313354,
"rewards/margins": 0.1491888463497162,
"rewards/rejected": 0.3235314190387726,
"step": 10
},
{
"epoch": 0.22388059701492538,
"grad_norm": 47.770878647824766,
"learning_rate": 9.983100718730718e-07,
"logits/chosen": -2.455693006515503,
"logits/rejected": -2.4368350505828857,
"logps/chosen": -238.6224365234375,
"logps/rejected": -190.65304565429688,
"loss": 0.6652,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": 1.1667336225509644,
"rewards/margins": 0.4549005925655365,
"rewards/rejected": 0.711833119392395,
"step": 15
},
{
"epoch": 0.29850746268656714,
"grad_norm": 41.77400166740322,
"learning_rate": 9.932517109205849e-07,
"logits/chosen": -2.3191988468170166,
"logits/rejected": -2.298821210861206,
"logps/chosen": -234.3042755126953,
"logps/rejected": -199.0926971435547,
"loss": 0.6162,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": 1.481550931930542,
"rewards/margins": 0.7337996363639832,
"rewards/rejected": 0.7477512955665588,
"step": 20
},
{
"epoch": 0.373134328358209,
"grad_norm": 37.097180665689756,
"learning_rate": 9.848591102083375e-07,
"logits/chosen": -2.243551731109619,
"logits/rejected": -2.2224888801574707,
"logps/chosen": -270.208251953125,
"logps/rejected": -196.4565887451172,
"loss": 0.596,
"rewards/accuracies": 0.762499988079071,
"rewards/chosen": 1.8057639598846436,
"rewards/margins": 1.2407209873199463,
"rewards/rejected": 0.565043032169342,
"step": 25
},
{
"epoch": 0.44776119402985076,
"grad_norm": 40.16458707164062,
"learning_rate": 9.731890013043367e-07,
"logits/chosen": -2.2443346977233887,
"logits/rejected": -2.251404285430908,
"logps/chosen": -231.3487548828125,
"logps/rejected": -221.5731964111328,
"loss": 0.5597,
"rewards/accuracies": 0.75,
"rewards/chosen": 1.3293423652648926,
"rewards/margins": 1.2652809619903564,
"rewards/rejected": 0.0640614926815033,
"step": 30
},
{
"epoch": 0.5223880597014925,
"grad_norm": 46.613500144353665,
"learning_rate": 9.583202707897073e-07,
"logits/chosen": -2.3597989082336426,
"logits/rejected": -2.3549132347106934,
"logps/chosen": -241.8134765625,
"logps/rejected": -214.6074676513672,
"loss": 0.6498,
"rewards/accuracies": 0.668749988079071,
"rewards/chosen": 0.8961586952209473,
"rewards/margins": 0.7394484877586365,
"rewards/rejected": 0.15671022236347198,
"step": 35
},
{
"epoch": 0.5970149253731343,
"grad_norm": 42.571655553623124,
"learning_rate": 9.403534270080829e-07,
"logits/chosen": -2.3809351921081543,
"logits/rejected": -2.3687262535095215,
"logps/chosen": -239.2801513671875,
"logps/rejected": -209.7950439453125,
"loss": 0.6099,
"rewards/accuracies": 0.7437499761581421,
"rewards/chosen": 1.0974634885787964,
"rewards/margins": 1.2669336795806885,
"rewards/rejected": -0.16947026550769806,
"step": 40
},
{
"epoch": 0.6716417910447762,
"grad_norm": 40.02985298301046,
"learning_rate": 9.19409920658098e-07,
"logits/chosen": -2.2248692512512207,
"logits/rejected": -2.2323927879333496,
"logps/chosen": -262.9397888183594,
"logps/rejected": -202.8191680908203,
"loss": 0.612,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": 1.1908448934555054,
"rewards/margins": 1.1182587146759033,
"rewards/rejected": 0.07258612662553787,
"step": 45
},
{
"epoch": 0.746268656716418,
"grad_norm": 44.22077560247569,
"learning_rate": 8.956313238215823e-07,
"logits/chosen": -2.1219964027404785,
"logits/rejected": -2.0875980854034424,
"logps/chosen": -242.64578247070312,
"logps/rejected": -204.98623657226562,
"loss": 0.6007,
"rewards/accuracies": 0.768750011920929,
"rewards/chosen": 1.1351102590560913,
"rewards/margins": 1.0577830076217651,
"rewards/rejected": 0.07732732594013214,
"step": 50
},
{
"epoch": 0.746268656716418,
"eval_logits/chosen": -2.118994951248169,
"eval_logits/rejected": -2.085069417953491,
"eval_logps/chosen": -264.02423095703125,
"eval_logps/rejected": -229.3638458251953,
"eval_loss": 0.6151112914085388,
"eval_rewards/accuracies": 0.762499988079071,
"eval_rewards/chosen": 1.2504631280899048,
"eval_rewards/margins": 1.4133154153823853,
"eval_rewards/rejected": -0.16285225749015808,
"eval_runtime": 128.217,
"eval_samples_per_second": 14.819,
"eval_steps_per_second": 0.234,
"step": 50
},
{
"epoch": 0.8208955223880597,
"grad_norm": 40.18630443466436,
"learning_rate": 8.691783729769873e-07,
"logits/chosen": -2.11806321144104,
"logits/rejected": -2.0859131813049316,
"logps/chosen": -245.06289672851562,
"logps/rejected": -208.0782928466797,
"loss": 0.5871,
"rewards/accuracies": 0.831250011920929,
"rewards/chosen": 0.9679476618766785,
"rewards/margins": 1.4604319334030151,
"rewards/rejected": -0.49248433113098145,
"step": 55
},
{
"epoch": 0.8955223880597015,
"grad_norm": 35.47562973498541,
"learning_rate": 8.402298824670029e-07,
"logits/chosen": -2.089561939239502,
"logits/rejected": -2.082063674926758,
"logps/chosen": -249.4997100830078,
"logps/rejected": -218.75180053710938,
"loss": 0.5522,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.7627511024475098,
"rewards/margins": 1.2879369258880615,
"rewards/rejected": -0.5251858830451965,
"step": 60
},
{
"epoch": 0.9701492537313433,
"grad_norm": 33.55304707235284,
"learning_rate": 8.089815357650089e-07,
"logits/chosen": -2.074007749557495,
"logits/rejected": -2.046736717224121,
"logps/chosen": -254.79116821289062,
"logps/rejected": -216.1507110595703,
"loss": 0.5687,
"rewards/accuracies": 0.78125,
"rewards/chosen": 0.8360864520072937,
"rewards/margins": 1.374267816543579,
"rewards/rejected": -0.5381813049316406,
"step": 65
},
{
"epoch": 1.044776119402985,
"grad_norm": 21.072890393731228,
"learning_rate": 7.756445627110522e-07,
"logits/chosen": -2.046535015106201,
"logits/rejected": -2.0300545692443848,
"logps/chosen": -242.8757781982422,
"logps/rejected": -222.74545288085938,
"loss": 0.4086,
"rewards/accuracies": 0.831250011920929,
"rewards/chosen": 1.4517204761505127,
"rewards/margins": 2.1768176555633545,
"rewards/rejected": -0.7250972986221313,
"step": 70
},
{
"epoch": 1.1194029850746268,
"grad_norm": 22.42027466653792,
"learning_rate": 7.404443116588547e-07,
"logits/chosen": -2.0193662643432617,
"logits/rejected": -1.9956552982330322,
"logps/chosen": -241.31900024414062,
"logps/rejected": -227.45822143554688,
"loss": 0.2538,
"rewards/accuracies": 0.9437500238418579,
"rewards/chosen": 1.9737942218780518,
"rewards/margins": 2.759068489074707,
"rewards/rejected": -0.7852741479873657,
"step": 75
},
{
"epoch": 1.1940298507462686,
"grad_norm": 16.719506543900728,
"learning_rate": 7.036187261857288e-07,
"logits/chosen": -2.040085554122925,
"logits/rejected": -1.9938533306121826,
"logps/chosen": -238.9818572998047,
"logps/rejected": -231.0389862060547,
"loss": 0.2171,
"rewards/accuracies": 0.918749988079071,
"rewards/chosen": 2.3720481395721436,
"rewards/margins": 3.3235080242156982,
"rewards/rejected": -0.9514597654342651,
"step": 80
},
{
"epoch": 1.2686567164179103,
"grad_norm": 26.325361102777258,
"learning_rate": 6.654167366624008e-07,
"logits/chosen": -2.0701422691345215,
"logits/rejected": -2.0505211353302,
"logps/chosen": -225.2556915283203,
"logps/rejected": -230.9950408935547,
"loss": 0.2527,
"rewards/accuracies": 0.9375,
"rewards/chosen": 2.3914668560028076,
"rewards/margins": 3.2805893421173096,
"rewards/rejected": -0.8891223669052124,
"step": 85
},
{
"epoch": 1.3432835820895521,
"grad_norm": 22.27953814968023,
"learning_rate": 6.260965775552713e-07,
"logits/chosen": -2.1463778018951416,
"logits/rejected": -2.1030120849609375,
"logps/chosen": -232.5117950439453,
"logps/rejected": -203.0765838623047,
"loss": 0.2471,
"rewards/accuracies": 0.90625,
"rewards/chosen": 2.4263741970062256,
"rewards/margins": 3.221815586090088,
"rewards/rejected": -0.7954414486885071,
"step": 90
},
{
"epoch": 1.417910447761194,
"grad_norm": 25.58307192903391,
"learning_rate": 5.859240418356614e-07,
"logits/chosen": -2.1819968223571777,
"logits/rejected": -2.145015239715576,
"logps/chosen": -227.00015258789062,
"logps/rejected": -219.8661346435547,
"loss": 0.3243,
"rewards/accuracies": 0.9375,
"rewards/chosen": 2.5345447063446045,
"rewards/margins": 3.464165449142456,
"rewards/rejected": -0.9296209216117859,
"step": 95
},
{
"epoch": 1.4925373134328357,
"grad_norm": 22.088806026665704,
"learning_rate": 5.451706842957421e-07,
"logits/chosen": -2.2039341926574707,
"logits/rejected": -2.186410427093506,
"logps/chosen": -221.75912475585938,
"logps/rejected": -216.10397338867188,
"loss": 0.2744,
"rewards/accuracies": 0.8812500238418579,
"rewards/chosen": 2.241814613342285,
"rewards/margins": 3.3029322624206543,
"rewards/rejected": -1.0611175298690796,
"step": 100
},
{
"epoch": 1.4925373134328357,
"eval_logits/chosen": -2.2504680156707764,
"eval_logits/rejected": -2.214540958404541,
"eval_logps/chosen": -258.7962341308594,
"eval_logps/rejected": -230.7489471435547,
"eval_loss": 0.582641065120697,
"eval_rewards/accuracies": 0.7541666626930237,
"eval_rewards/chosen": 1.7732579708099365,
"eval_rewards/margins": 2.074618339538574,
"eval_rewards/rejected": -0.30136018991470337,
"eval_runtime": 128.1598,
"eval_samples_per_second": 14.825,
"eval_steps_per_second": 0.234,
"step": 100
}
],
"logging_steps": 5,
"max_steps": 201,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 100,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 1178822762299392.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|