| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.0026666666666666666, | |
| "eval_steps": 500, | |
| "global_step": 30, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 2718.050079345703, | |
| "epoch": 8.888888888888889e-05, | |
| "grad_norm": 0.6192231375922369, | |
| "kl": 0.0, | |
| "learning_rate": 3.571428571428571e-08, | |
| "loss": -0.0, | |
| "reward": 2.116292830556631, | |
| "reward_std": 4.779026746749878, | |
| "rewards/total_reward_func": 2.116292830556631, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 3229.1500854492188, | |
| "epoch": 0.00017777777777777779, | |
| "grad_norm": 0.5851864267502562, | |
| "kl": 0.0, | |
| "learning_rate": 7.142857142857142e-08, | |
| "loss": -0.0, | |
| "reward": 0.5039098858833313, | |
| "reward_std": 4.289918065071106, | |
| "rewards/total_reward_func": 0.5039098858833313, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 2268.050018310547, | |
| "epoch": 0.0002666666666666667, | |
| "grad_norm": 0.6345571752209076, | |
| "kl": 0.0002942085266113281, | |
| "learning_rate": 1.0714285714285713e-07, | |
| "loss": 0.0, | |
| "reward": 1.3949939515441656, | |
| "reward_std": 3.23672516644001, | |
| "rewards/total_reward_func": 1.3949939515441656, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 1839.0000305175781, | |
| "epoch": 0.00035555555555555557, | |
| "grad_norm": 0.6374935763333437, | |
| "kl": 0.000255584716796875, | |
| "learning_rate": 1.4285714285714285e-07, | |
| "loss": 0.0, | |
| "reward": 2.791418433189392, | |
| "reward_std": 2.9814966320991516, | |
| "rewards/total_reward_func": 2.791418433189392, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 2416.3999938964844, | |
| "epoch": 0.00044444444444444447, | |
| "grad_norm": 0.6383136164663117, | |
| "kl": 0.0002701282501220703, | |
| "learning_rate": 1.7857142857142858e-07, | |
| "loss": 0.0, | |
| "reward": 1.640659123659134, | |
| "reward_std": 4.59553736448288, | |
| "rewards/total_reward_func": 1.640659123659134, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 2115.35009765625, | |
| "epoch": 0.0005333333333333334, | |
| "grad_norm": 0.6559290578416846, | |
| "kl": 0.00028443336486816406, | |
| "learning_rate": 2.1428571428571426e-07, | |
| "loss": 0.0, | |
| "reward": 2.4659183472394943, | |
| "reward_std": 3.740811139345169, | |
| "rewards/total_reward_func": 2.4659183472394943, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 2952.8499755859375, | |
| "epoch": 0.0006222222222222223, | |
| "grad_norm": 0.5811564588363323, | |
| "kl": 0.0002536773681640625, | |
| "learning_rate": 2.5e-07, | |
| "loss": 0.0, | |
| "reward": 2.0189540088176727, | |
| "reward_std": 3.650411069393158, | |
| "rewards/total_reward_func": 2.0189540088176727, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 3190.999969482422, | |
| "epoch": 0.0007111111111111111, | |
| "grad_norm": 0.6189187735785002, | |
| "kl": 0.0003533363342285156, | |
| "learning_rate": 2.857142857142857e-07, | |
| "loss": -0.0, | |
| "reward": -1.4477039803750813, | |
| "reward_std": 3.6360936164855957, | |
| "rewards/total_reward_func": -1.4477039803750813, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 2972.10009765625, | |
| "epoch": 0.0008, | |
| "grad_norm": 0.6842322911932173, | |
| "kl": 0.00037479400634765625, | |
| "learning_rate": 3.2142857142857145e-07, | |
| "loss": 0.0, | |
| "reward": 2.728776067495346, | |
| "reward_std": 3.11775803565979, | |
| "rewards/total_reward_func": 2.728776067495346, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 1998.1999816894531, | |
| "epoch": 0.0008888888888888889, | |
| "grad_norm": 0.6444993913830819, | |
| "kl": 0.00024318695068359375, | |
| "learning_rate": 3.5714285714285716e-07, | |
| "loss": 0.0, | |
| "reward": 2.077220171689987, | |
| "reward_std": 4.106507062911987, | |
| "rewards/total_reward_func": 2.077220171689987, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 2594.0499877929688, | |
| "epoch": 0.0009777777777777777, | |
| "grad_norm": 0.5975596004351243, | |
| "kl": 0.0003218650817871094, | |
| "learning_rate": 3.928571428571428e-07, | |
| "loss": 0.0, | |
| "reward": 1.5957689583301544, | |
| "reward_std": 4.772671937942505, | |
| "rewards/total_reward_func": 1.5957689583301544, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 3097.4500122070312, | |
| "epoch": 0.0010666666666666667, | |
| "grad_norm": 0.6306417782745113, | |
| "kl": 0.0003795623779296875, | |
| "learning_rate": 4.285714285714285e-07, | |
| "loss": 0.0, | |
| "reward": 0.1589522361755371, | |
| "reward_std": 4.308449387550354, | |
| "rewards/total_reward_func": 0.1589522361755371, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 2461.800048828125, | |
| "epoch": 0.0011555555555555555, | |
| "grad_norm": 0.5255731669888752, | |
| "kl": 0.0003116130828857422, | |
| "learning_rate": 4.6428571428571427e-07, | |
| "loss": 0.0, | |
| "reward": 1.6479228138923645, | |
| "reward_std": 2.908693313598633, | |
| "rewards/total_reward_func": 1.6479228138923645, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 2828.4000091552734, | |
| "epoch": 0.0012444444444444445, | |
| "grad_norm": 0.6202530623554537, | |
| "kl": 0.0003025531768798828, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0, | |
| "reward": 1.1665852814912796, | |
| "reward_std": 4.7892152070999146, | |
| "rewards/total_reward_func": 1.1665852814912796, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 3782.85009765625, | |
| "epoch": 0.0013333333333333333, | |
| "grad_norm": 0.6301844467737111, | |
| "kl": 0.0003418922424316406, | |
| "learning_rate": 4.999935101463869e-07, | |
| "loss": 0.0, | |
| "reward": -1.3677009493112564, | |
| "reward_std": 4.384381651878357, | |
| "rewards/total_reward_func": -1.3677009493112564, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 2890.100128173828, | |
| "epoch": 0.0014222222222222223, | |
| "grad_norm": 0.6098646440372368, | |
| "kl": 0.00028228759765625, | |
| "learning_rate": 4.999740409224932e-07, | |
| "loss": 0.0, | |
| "reward": 2.2114409431815147, | |
| "reward_std": 4.764661192893982, | |
| "rewards/total_reward_func": 2.2114409431815147, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 3049.750045776367, | |
| "epoch": 0.001511111111111111, | |
| "grad_norm": 0.6819736594876971, | |
| "kl": 0.0003757476806640625, | |
| "learning_rate": 4.999415933391384e-07, | |
| "loss": 0.0, | |
| "reward": 1.3528989106416702, | |
| "reward_std": 3.6589310467243195, | |
| "rewards/total_reward_func": 1.3528989106416702, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 2049.9500122070312, | |
| "epoch": 0.0016, | |
| "grad_norm": 0.8155246119069368, | |
| "kl": 0.00027060508728027344, | |
| "learning_rate": 4.998961690809627e-07, | |
| "loss": 0.0, | |
| "reward": 0.5832462757825851, | |
| "reward_std": 4.575035452842712, | |
| "rewards/total_reward_func": 0.5832462757825851, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 2366.85009765625, | |
| "epoch": 0.0016888888888888889, | |
| "grad_norm": 0.6218339427438726, | |
| "kl": 0.0002536773681640625, | |
| "learning_rate": 4.998377705063407e-07, | |
| "loss": 0.0, | |
| "reward": 0.6414558179676533, | |
| "reward_std": 4.986617684364319, | |
| "rewards/total_reward_func": 0.6414558179676533, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 3348.7500915527344, | |
| "epoch": 0.0017777777777777779, | |
| "grad_norm": 0.5512786524811666, | |
| "kl": 0.00033664703369140625, | |
| "learning_rate": 4.997664006472578e-07, | |
| "loss": 0.0, | |
| "reward": 0.6009512692689896, | |
| "reward_std": 4.6273534297943115, | |
| "rewards/total_reward_func": 0.6009512692689896, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 3526.4000854492188, | |
| "epoch": 0.0018666666666666666, | |
| "grad_norm": 0.5031601581414681, | |
| "kl": 0.0003418922424316406, | |
| "learning_rate": 4.996820632091536e-07, | |
| "loss": 0.0, | |
| "reward": 0.1938244104385376, | |
| "reward_std": 3.324654757976532, | |
| "rewards/total_reward_func": 0.1938244104385376, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 3173.7000732421875, | |
| "epoch": 0.0019555555555555554, | |
| "grad_norm": 0.5376155862324518, | |
| "kl": 0.00034999847412109375, | |
| "learning_rate": 4.995847625707292e-07, | |
| "loss": 0.0, | |
| "reward": 1.1223694756627083, | |
| "reward_std": 4.764593482017517, | |
| "rewards/total_reward_func": 1.1223694756627083, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 2309.7500610351562, | |
| "epoch": 0.0020444444444444447, | |
| "grad_norm": 0.697384852452982, | |
| "kl": 0.0003190040588378906, | |
| "learning_rate": 4.994745037837194e-07, | |
| "loss": 0.0, | |
| "reward": 1.649430025368929, | |
| "reward_std": 4.3076507449150085, | |
| "rewards/total_reward_func": 1.649430025368929, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 3129.800018310547, | |
| "epoch": 0.0021333333333333334, | |
| "grad_norm": 0.6535932860292653, | |
| "kl": 0.00032711029052734375, | |
| "learning_rate": 4.993512925726318e-07, | |
| "loss": 0.0, | |
| "reward": -0.3294833302497864, | |
| "reward_std": 4.482808172702789, | |
| "rewards/total_reward_func": -0.3294833302497864, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 3112.9500579833984, | |
| "epoch": 0.0022222222222222222, | |
| "grad_norm": 0.6586438664678015, | |
| "kl": 0.0003428459167480469, | |
| "learning_rate": 4.992151353344481e-07, | |
| "loss": 0.0, | |
| "reward": 1.9663346596062183, | |
| "reward_std": 3.5024783611297607, | |
| "rewards/total_reward_func": 1.9663346596062183, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 3459.6000366210938, | |
| "epoch": 0.002311111111111111, | |
| "grad_norm": 0.5558711418419443, | |
| "kl": 0.00035071372985839844, | |
| "learning_rate": 4.990660391382923e-07, | |
| "loss": 0.0, | |
| "reward": 1.041447937488556, | |
| "reward_std": 4.875528454780579, | |
| "rewards/total_reward_func": 1.041447937488556, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 3298.100128173828, | |
| "epoch": 0.0024, | |
| "grad_norm": 0.564389276700835, | |
| "kl": 0.000347137451171875, | |
| "learning_rate": 4.989040117250646e-07, | |
| "loss": 0.0, | |
| "reward": -0.36215686798095703, | |
| "reward_std": 3.4190066680312157, | |
| "rewards/total_reward_func": -0.36215686798095703, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 2197.850067138672, | |
| "epoch": 0.002488888888888889, | |
| "grad_norm": 0.6845246040542481, | |
| "kl": 0.00031113624572753906, | |
| "learning_rate": 4.987290615070384e-07, | |
| "loss": 0.0, | |
| "reward": 1.5534338653087616, | |
| "reward_std": 4.177336096763611, | |
| "rewards/total_reward_func": 1.5534338653087616, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 3759.9500732421875, | |
| "epoch": 0.002577777777777778, | |
| "grad_norm": 0.6159462476594502, | |
| "kl": 0.00035572052001953125, | |
| "learning_rate": 4.985411975674243e-07, | |
| "loss": 0.0, | |
| "reward": 1.3788942694664001, | |
| "reward_std": 3.83202987909317, | |
| "rewards/total_reward_func": 1.3788942694664001, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 2328.9000244140625, | |
| "epoch": 0.0026666666666666666, | |
| "grad_norm": 0.5959887864273203, | |
| "kl": 0.00030517578125, | |
| "learning_rate": 4.983404296598978e-07, | |
| "loss": 0.0, | |
| "reward": 2.809926450252533, | |
| "reward_std": 4.070581257343292, | |
| "rewards/total_reward_func": 2.809926450252533, | |
| "step": 30 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 450, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 10, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |