| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 3.0, | |
| "eval_steps": 500, | |
| "global_step": 228, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "completion_length": 99.10416793823242, | |
| "epoch": 0.013157894736842105, | |
| "grad_norm": 1.1334748507973709, | |
| "kl": 0.0, | |
| "learning_rate": 7.142857142857142e-08, | |
| "loss": -0.0, | |
| "reward": 0.6663198471069336, | |
| "reward_std": 0.4498612582683563, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.4163198471069336, | |
| "step": 1 | |
| }, | |
| { | |
| "completion_length": 91.58333587646484, | |
| "epoch": 0.02631578947368421, | |
| "grad_norm": 16.060906203241963, | |
| "kl": 0.0, | |
| "learning_rate": 1.4285714285714285e-07, | |
| "loss": -0.0, | |
| "reward": 0.9228171110153198, | |
| "reward_std": 0.4594407230615616, | |
| "rewards/correct_code_reward_func": 0.5416666865348816, | |
| "rewards/len_reward_func": 0.38115040957927704, | |
| "step": 2 | |
| }, | |
| { | |
| "completion_length": 72.02083587646484, | |
| "epoch": 0.039473684210526314, | |
| "grad_norm": 3.1753635339709736, | |
| "kl": 0.00012445449829101562, | |
| "learning_rate": 2.1428571428571426e-07, | |
| "loss": 0.0, | |
| "reward": 0.6041666865348816, | |
| "reward_std": 0.25199543684720993, | |
| "rewards/correct_code_reward_func": 0.1666666679084301, | |
| "rewards/len_reward_func": 0.4375000149011612, | |
| "step": 3 | |
| }, | |
| { | |
| "completion_length": 87.33333396911621, | |
| "epoch": 0.05263157894736842, | |
| "grad_norm": 5.760299497142578, | |
| "kl": -1.5407800674438477e-05, | |
| "learning_rate": 2.857142857142857e-07, | |
| "loss": -0.0, | |
| "reward": 0.7341760993003845, | |
| "reward_std": 0.37608518451452255, | |
| "rewards/correct_code_reward_func": 0.31250002048909664, | |
| "rewards/len_reward_func": 0.4216760843992233, | |
| "step": 4 | |
| }, | |
| { | |
| "completion_length": 71.85416793823242, | |
| "epoch": 0.06578947368421052, | |
| "grad_norm": 12.020598074533623, | |
| "kl": 0.00014781951904296875, | |
| "learning_rate": 3.5714285714285716e-07, | |
| "loss": 0.0, | |
| "reward": 0.5998998582363129, | |
| "reward_std": 0.20088782534003258, | |
| "rewards/correct_code_reward_func": 0.14583333395421505, | |
| "rewards/len_reward_func": 0.45406651496887207, | |
| "step": 5 | |
| }, | |
| { | |
| "completion_length": 89.77083587646484, | |
| "epoch": 0.07894736842105263, | |
| "grad_norm": 0.8550201889336397, | |
| "kl": 4.374980926513672e-05, | |
| "learning_rate": 4.285714285714285e-07, | |
| "loss": 0.0, | |
| "reward": 0.5605520606040955, | |
| "reward_std": 0.2179059162735939, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.45638537406921387, | |
| "step": 6 | |
| }, | |
| { | |
| "completion_length": 72.79167175292969, | |
| "epoch": 0.09210526315789473, | |
| "grad_norm": 3.0170118497826293, | |
| "kl": 0.00016558170318603516, | |
| "learning_rate": 5e-07, | |
| "loss": 0.0, | |
| "reward": 0.9927114844322205, | |
| "reward_std": 0.40002472698688507, | |
| "rewards/correct_code_reward_func": 0.5625, | |
| "rewards/len_reward_func": 0.4302114397287369, | |
| "step": 7 | |
| }, | |
| { | |
| "completion_length": 81.66667175292969, | |
| "epoch": 0.10526315789473684, | |
| "grad_norm": 4.916856440024865, | |
| "kl": 7.331371307373047e-05, | |
| "learning_rate": 4.999747408985249e-07, | |
| "loss": 0.0, | |
| "reward": 0.9147771000862122, | |
| "reward_std": 0.43089577555656433, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.39394378662109375, | |
| "step": 8 | |
| }, | |
| { | |
| "completion_length": 106.1875057220459, | |
| "epoch": 0.11842105263157894, | |
| "grad_norm": 7.051099519560641, | |
| "kl": 0.00026702880859375, | |
| "learning_rate": 4.998989686982771e-07, | |
| "loss": 0.0, | |
| "reward": 0.9007994532585144, | |
| "reward_std": 0.30253712832927704, | |
| "rewards/correct_code_reward_func": 0.4791666716337204, | |
| "rewards/len_reward_func": 0.4216327518224716, | |
| "step": 9 | |
| }, | |
| { | |
| "completion_length": 89.25, | |
| "epoch": 0.13157894736842105, | |
| "grad_norm": 3.0953263442901586, | |
| "kl": 0.00014472007751464844, | |
| "learning_rate": 4.997726987107581e-07, | |
| "loss": 0.0, | |
| "reward": 0.8582921326160431, | |
| "reward_std": 0.4781641513109207, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.4416254311800003, | |
| "step": 10 | |
| }, | |
| { | |
| "completion_length": 61.0625, | |
| "epoch": 0.14473684210526316, | |
| "grad_norm": 26.321796409287952, | |
| "kl": 0.00046253204345703125, | |
| "learning_rate": 4.995959564516996e-07, | |
| "loss": 0.0, | |
| "reward": 1.1543937921524048, | |
| "reward_std": 0.46843117475509644, | |
| "rewards/correct_code_reward_func": 0.7708333432674408, | |
| "rewards/len_reward_func": 0.3835604339838028, | |
| "step": 11 | |
| }, | |
| { | |
| "completion_length": 57.687503814697266, | |
| "epoch": 0.15789473684210525, | |
| "grad_norm": 2.6899417118195452, | |
| "kl": 0.000301361083984375, | |
| "learning_rate": 4.993687776359066e-07, | |
| "loss": 0.0, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.37565621733665466, | |
| "rewards/correct_code_reward_func": 0.395833358168602, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 12 | |
| }, | |
| { | |
| "completion_length": 160.35416793823242, | |
| "epoch": 0.17105263157894737, | |
| "grad_norm": 2.247154751130096, | |
| "kl": 0.0006351470947265625, | |
| "learning_rate": 4.990912081700413e-07, | |
| "loss": 0.0, | |
| "reward": 0.5879067778587341, | |
| "reward_std": 0.3584153801202774, | |
| "rewards/correct_code_reward_func": 0.2291666679084301, | |
| "rewards/len_reward_func": 0.3587401211261749, | |
| "step": 13 | |
| }, | |
| { | |
| "completion_length": 54.916669845581055, | |
| "epoch": 0.18421052631578946, | |
| "grad_norm": 7.0002606987315685, | |
| "kl": 0.000507354736328125, | |
| "learning_rate": 4.987633041433461e-07, | |
| "loss": 0.0, | |
| "reward": 0.9894838631153107, | |
| "reward_std": 0.5111564844846725, | |
| "rewards/correct_code_reward_func": 0.5416666716337204, | |
| "rewards/len_reward_func": 0.4478171765804291, | |
| "step": 14 | |
| }, | |
| { | |
| "completion_length": 63.06250190734863, | |
| "epoch": 0.19736842105263158, | |
| "grad_norm": 2.6443252242541284, | |
| "kl": 0.001739501953125, | |
| "learning_rate": 4.983851318163097e-07, | |
| "loss": 0.0, | |
| "reward": 0.8326185643672943, | |
| "reward_std": 0.29923243820667267, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.4367852360010147, | |
| "step": 15 | |
| }, | |
| { | |
| "completion_length": 99.39583587646484, | |
| "epoch": 0.21052631578947367, | |
| "grad_norm": 1.0079451181922614, | |
| "kl": 0.00141143798828125, | |
| "learning_rate": 4.979567676072775e-07, | |
| "loss": 0.0, | |
| "reward": 0.852995753288269, | |
| "reward_std": 0.34659716486930847, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.45716238021850586, | |
| "step": 16 | |
| }, | |
| { | |
| "completion_length": 53.08333396911621, | |
| "epoch": 0.2236842105263158, | |
| "grad_norm": 9.967406022456908, | |
| "kl": 0.0040435791015625, | |
| "learning_rate": 4.9747829807701e-07, | |
| "loss": 0.0, | |
| "reward": 0.8658536076545715, | |
| "reward_std": 0.426578089594841, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.44918693602085114, | |
| "step": 17 | |
| }, | |
| { | |
| "completion_length": 143.7916717529297, | |
| "epoch": 0.23684210526315788, | |
| "grad_norm": 1.9448270150109064, | |
| "kl": 0.001434326171875, | |
| "learning_rate": 4.9694981991119e-07, | |
| "loss": 0.0, | |
| "reward": 0.7298745512962341, | |
| "reward_std": 0.28270113095641136, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.39654120802879333, | |
| "step": 18 | |
| }, | |
| { | |
| "completion_length": 61.041669845581055, | |
| "epoch": 0.25, | |
| "grad_norm": 3.1720724842373182, | |
| "kl": 0.00389862060546875, | |
| "learning_rate": 4.963714399008868e-07, | |
| "loss": 0.0, | |
| "reward": 0.718452662229538, | |
| "reward_std": 0.37869784235954285, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.46845264732837677, | |
| "step": 19 | |
| }, | |
| { | |
| "completion_length": 79.89583587646484, | |
| "epoch": 0.2631578947368421, | |
| "grad_norm": 1.7632061082251036, | |
| "kl": 0.00446319580078125, | |
| "learning_rate": 4.957432749209755e-07, | |
| "loss": 0.0, | |
| "reward": 0.7348142862319946, | |
| "reward_std": 0.1320388102903962, | |
| "rewards/correct_code_reward_func": 0.25, | |
| "rewards/len_reward_func": 0.48481428623199463, | |
| "step": 20 | |
| }, | |
| { | |
| "completion_length": 81.18750381469727, | |
| "epoch": 0.27631578947368424, | |
| "grad_norm": 2.5858288436684593, | |
| "kl": 0.00498199462890625, | |
| "learning_rate": 4.950654519065195e-07, | |
| "loss": 0.0, | |
| "reward": 0.6811683177947998, | |
| "reward_std": 0.1932540312409401, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.472834974527359, | |
| "step": 21 | |
| }, | |
| { | |
| "completion_length": 87.56250381469727, | |
| "epoch": 0.2894736842105263, | |
| "grad_norm": 18.2585507937141, | |
| "kl": 0.03173828125, | |
| "learning_rate": 4.943381078271214e-07, | |
| "loss": 0.0, | |
| "reward": 0.6875, | |
| "reward_std": 0.28126099705696106, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 22 | |
| }, | |
| { | |
| "completion_length": 134.64583587646484, | |
| "epoch": 0.3026315789473684, | |
| "grad_norm": 17.07507069634021, | |
| "kl": 0.12744140625, | |
| "learning_rate": 4.935613896592445e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9976747632026672, | |
| "reward_std": 0.22606945782899857, | |
| "rewards/correct_code_reward_func": 0.5416666865348816, | |
| "rewards/len_reward_func": 0.45600807666778564, | |
| "step": 23 | |
| }, | |
| { | |
| "completion_length": 52.25000190734863, | |
| "epoch": 0.3157894736842105, | |
| "grad_norm": 5.933528514518437, | |
| "kl": 0.05426025390625, | |
| "learning_rate": 4.92735454356513e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.2994871214032173, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 24 | |
| }, | |
| { | |
| "completion_length": 96.29167175292969, | |
| "epoch": 0.32894736842105265, | |
| "grad_norm": 0.7449673513689808, | |
| "kl": 0.0208740234375, | |
| "learning_rate": 4.918604688179958e-07, | |
| "loss": 0.0, | |
| "reward": 0.5824915617704391, | |
| "reward_std": 0.1475645862519741, | |
| "rewards/correct_code_reward_func": 0.1041666716337204, | |
| "rewards/len_reward_func": 0.47832491993904114, | |
| "step": 25 | |
| }, | |
| { | |
| "completion_length": 57.04166793823242, | |
| "epoch": 0.34210526315789475, | |
| "grad_norm": 7.282800644316476, | |
| "kl": 0.05474853515625, | |
| "learning_rate": 4.90936609854481e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.4474997818470001, | |
| "rewards/correct_code_reward_func": 0.5416666865348816, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 26 | |
| }, | |
| { | |
| "completion_length": 69.29166984558105, | |
| "epoch": 0.35526315789473684, | |
| "grad_norm": 11.501413945029437, | |
| "kl": 0.037353515625, | |
| "learning_rate": 4.89964064152747e-07, | |
| "loss": 0.0, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.2730504274368286, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 27 | |
| }, | |
| { | |
| "completion_length": 52.708335876464844, | |
| "epoch": 0.3684210526315789, | |
| "grad_norm": 1.4375830290890173, | |
| "kl": 0.05767822265625, | |
| "learning_rate": 4.889430282378384e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7291666865348816, | |
| "reward_std": 0.21322893351316452, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 28 | |
| }, | |
| { | |
| "completion_length": 84.875, | |
| "epoch": 0.3815789473684211, | |
| "grad_norm": 1.8947890037281183, | |
| "kl": 0.079345703125, | |
| "learning_rate": 4.878737084333535e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7291666865348816, | |
| "reward_std": 0.30859363824129105, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 29 | |
| }, | |
| { | |
| "completion_length": 91.70833587646484, | |
| "epoch": 0.39473684210526316, | |
| "grad_norm": 1.2731739041206802, | |
| "kl": 0.03851318359375, | |
| "learning_rate": 4.867563208197519e-07, | |
| "loss": 0.0, | |
| "reward": 0.799913227558136, | |
| "reward_std": 0.3069130629301071, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.4457465559244156, | |
| "step": 30 | |
| }, | |
| { | |
| "completion_length": 72.29166793823242, | |
| "epoch": 0.40789473684210525, | |
| "grad_norm": 1.4676508584854064, | |
| "kl": 0.04327392578125, | |
| "learning_rate": 4.855910911906906e-07, | |
| "loss": 0.0, | |
| "reward": 0.9658634960651398, | |
| "reward_std": 0.3308482989668846, | |
| "rewards/correct_code_reward_func": 0.520833358168602, | |
| "rewards/len_reward_func": 0.4450301378965378, | |
| "step": 31 | |
| }, | |
| { | |
| "completion_length": 76.93750381469727, | |
| "epoch": 0.42105263157894735, | |
| "grad_norm": 4.605001948885057, | |
| "kl": 0.087890625, | |
| "learning_rate": 4.843782550073969e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.22516431659460068, | |
| "rewards/correct_code_reward_func": 0.375, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 32 | |
| }, | |
| { | |
| "completion_length": 55.77083396911621, | |
| "epoch": 0.4342105263157895, | |
| "grad_norm": 1.0747031450058906, | |
| "kl": 0.234375, | |
| "learning_rate": 4.83118057351089e-07, | |
| "loss": 0.0002, | |
| "reward": 0.6458333432674408, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 33 | |
| }, | |
| { | |
| "completion_length": 58.02083396911621, | |
| "epoch": 0.4473684210526316, | |
| "grad_norm": 3.450736535043091, | |
| "kl": 0.06573486328125, | |
| "learning_rate": 4.818107528734503e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9100490808486938, | |
| "reward_std": 0.49446116387844086, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.47254903614521027, | |
| "step": 34 | |
| }, | |
| { | |
| "completion_length": 64.47916793823242, | |
| "epoch": 0.4605263157894737, | |
| "grad_norm": 3.8860905534674637, | |
| "kl": 0.0714111328125, | |
| "learning_rate": 4.804566057451729e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8531287312507629, | |
| "reward_std": 0.3388620540499687, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.49896204471588135, | |
| "step": 35 | |
| }, | |
| { | |
| "completion_length": 49.06250286102295, | |
| "epoch": 0.47368421052631576, | |
| "grad_norm": 118.2111884355624, | |
| "kl": 0.9837646484375, | |
| "learning_rate": 4.790558896025742e-07, | |
| "loss": 0.001, | |
| "reward": 0.8689024448394775, | |
| "reward_std": 0.31673362106084824, | |
| "rewards/correct_code_reward_func": 0.375, | |
| "rewards/len_reward_func": 0.49390244483947754, | |
| "step": 36 | |
| }, | |
| { | |
| "completion_length": 82.68750381469727, | |
| "epoch": 0.4868421052631579, | |
| "grad_norm": 1.2725798743612686, | |
| "kl": 0.0997314453125, | |
| "learning_rate": 4.776088874923041e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7270315587520599, | |
| "reward_std": 0.36151498556137085, | |
| "rewards/correct_code_reward_func": 0.229166679084301, | |
| "rewards/len_reward_func": 0.49786490201950073, | |
| "step": 37 | |
| }, | |
| { | |
| "completion_length": 67.29166984558105, | |
| "epoch": 0.5, | |
| "grad_norm": 0.6682530578144538, | |
| "kl": 0.24072265625, | |
| "learning_rate": 4.7611589181414734e-07, | |
| "loss": 0.0002, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.19500282034277916, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 38 | |
| }, | |
| { | |
| "completion_length": 82.89583587646484, | |
| "epoch": 0.5131578947368421, | |
| "grad_norm": 1.896117750609313, | |
| "kl": 0.109375, | |
| "learning_rate": 4.745772042619388e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7291666865348816, | |
| "reward_std": 0.28126102685928345, | |
| "rewards/correct_code_reward_func": 0.229166679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 39 | |
| }, | |
| { | |
| "completion_length": 58.06250190734863, | |
| "epoch": 0.5263157894736842, | |
| "grad_norm": 118.0980466511993, | |
| "kl": 44.5732421875, | |
| "learning_rate": 4.729931357625986e-07, | |
| "loss": 0.0447, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.3177001550793648, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 40 | |
| }, | |
| { | |
| "completion_length": 74.12500381469727, | |
| "epoch": 0.5394736842105263, | |
| "grad_norm": 0.7236725923896844, | |
| "kl": 0.41094970703125, | |
| "learning_rate": 4.7136400641330245e-07, | |
| "loss": 0.0004, | |
| "reward": 0.6250000149011612, | |
| "reward_std": 0.1178511306643486, | |
| "rewards/correct_code_reward_func": 0.1458333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 41 | |
| }, | |
| { | |
| "completion_length": 46.895835876464844, | |
| "epoch": 0.5526315789473685, | |
| "grad_norm": 4.631491683010619, | |
| "kl": 1.0185546875, | |
| "learning_rate": 4.696901454167988e-07, | |
| "loss": 0.001, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.2903675064444542, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 42 | |
| }, | |
| { | |
| "completion_length": 60.79166793823242, | |
| "epoch": 0.5657894736842105, | |
| "grad_norm": 3.3754705022635965, | |
| "kl": 0.15283203125, | |
| "learning_rate": 4.6797189101488576e-07, | |
| "loss": 0.0002, | |
| "reward": 0.979166716337204, | |
| "reward_std": 0.1767766959965229, | |
| "rewards/correct_code_reward_func": 0.5000000149011612, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 43 | |
| }, | |
| { | |
| "completion_length": 43.583335876464844, | |
| "epoch": 0.5789473684210527, | |
| "grad_norm": 95.77831861506337, | |
| "kl": 2.21484375, | |
| "learning_rate": 4.662095904200617e-07, | |
| "loss": 0.0022, | |
| "reward": 0.9375000596046448, | |
| "reward_std": 0.3310800567269325, | |
| "rewards/correct_code_reward_func": 0.4375000111758709, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 44 | |
| }, | |
| { | |
| "completion_length": 64.0625, | |
| "epoch": 0.5921052631578947, | |
| "grad_norm": 0.9899941998859535, | |
| "kl": 0.0555419921875, | |
| "learning_rate": 4.6440359974536304e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7708333730697632, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 45 | |
| }, | |
| { | |
| "completion_length": 53.04166793823242, | |
| "epoch": 0.6052631578947368, | |
| "grad_norm": 3.552556057212125, | |
| "kl": 0.229736328125, | |
| "learning_rate": 4.6255428393240354e-07, | |
| "loss": 0.0002, | |
| "reward": 1.0833333730697632, | |
| "reward_std": 0.19500282034277916, | |
| "rewards/correct_code_reward_func": 0.5833333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 46 | |
| }, | |
| { | |
| "completion_length": 40.52083396911621, | |
| "epoch": 0.618421052631579, | |
| "grad_norm": 27.248705920564117, | |
| "kl": 7.90234375, | |
| "learning_rate": 4.606620166776294e-07, | |
| "loss": 0.0079, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.16623876243829727, | |
| "rewards/correct_code_reward_func": 0.3750000223517418, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 47 | |
| }, | |
| { | |
| "completion_length": 54.68750190734863, | |
| "epoch": 0.631578947368421, | |
| "grad_norm": 6.398561296574553, | |
| "kl": 0.744384765625, | |
| "learning_rate": 4.587271803568055e-07, | |
| "loss": 0.0007, | |
| "reward": 0.5416666865348816, | |
| "reward_std": 0.1451837718486786, | |
| "rewards/correct_code_reward_func": 0.0625, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 48 | |
| }, | |
| { | |
| "completion_length": 48.29166793823242, | |
| "epoch": 0.6447368421052632, | |
| "grad_norm": 4.1122533170941535, | |
| "kl": 0.148681640625, | |
| "learning_rate": 4.567501659477476e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8958333730697632, | |
| "reward_std": 0.3205290287733078, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 49 | |
| }, | |
| { | |
| "completion_length": 39.3125, | |
| "epoch": 0.6578947368421053, | |
| "grad_norm": 53.38113189457624, | |
| "kl": 13.2625732421875, | |
| "learning_rate": 4.5473137295131625e-07, | |
| "loss": 0.0133, | |
| "reward": 0.9375, | |
| "reward_std": 0.22516433894634247, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 50 | |
| }, | |
| { | |
| "completion_length": 42.83333396911621, | |
| "epoch": 0.6710526315789473, | |
| "grad_norm": 18178.959749921032, | |
| "kl": 4768.09375, | |
| "learning_rate": 4.526712093106887e-07, | |
| "loss": 4.7525, | |
| "reward": 1.0208333432674408, | |
| "reward_std": 0.2041093371808529, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 51 | |
| }, | |
| { | |
| "completion_length": 71.66667175292969, | |
| "epoch": 0.6842105263157895, | |
| "grad_norm": 1.9579852812558598, | |
| "kl": 0.474609375, | |
| "learning_rate": 4.5057009132892455e-07, | |
| "loss": 0.0005, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.1178511306643486, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.4583333432674408, | |
| "step": 52 | |
| }, | |
| { | |
| "completion_length": 57.83333396911621, | |
| "epoch": 0.6973684210526315, | |
| "grad_norm": 3.4888745953712426, | |
| "kl": 0.9111328125, | |
| "learning_rate": 4.4842844358484233e-07, | |
| "loss": 0.0009, | |
| "reward": 0.9375000298023224, | |
| "reward_std": 0.22516431659460068, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 53 | |
| }, | |
| { | |
| "completion_length": 86.72916793823242, | |
| "epoch": 0.7105263157894737, | |
| "grad_norm": 3.0697908089632087, | |
| "kl": 0.12158203125, | |
| "learning_rate": 4.4624669884722364e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.31142252683639526, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 54 | |
| }, | |
| { | |
| "completion_length": 53.895835876464844, | |
| "epoch": 0.7236842105263158, | |
| "grad_norm": 1.4628769742470222, | |
| "kl": 0.137939453125, | |
| "learning_rate": 4.4402529798736214e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0, | |
| "reward_std": 0.2994871214032173, | |
| "rewards/correct_code_reward_func": 0.5, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 55 | |
| }, | |
| { | |
| "completion_length": 51.770835876464844, | |
| "epoch": 0.7368421052631579, | |
| "grad_norm": 19.218612806133308, | |
| "kl": 6.68017578125, | |
| "learning_rate": 4.417646898899758e-07, | |
| "loss": 0.0067, | |
| "reward": 0.7708333432674408, | |
| "reward_std": 0.2041093371808529, | |
| "rewards/correct_code_reward_func": 0.2708333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 56 | |
| }, | |
| { | |
| "completion_length": 48.875, | |
| "epoch": 0.75, | |
| "grad_norm": 1.1426530146279454, | |
| "kl": 0.115234375, | |
| "learning_rate": 4.394653313624992e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.3478616699576378, | |
| "rewards/correct_code_reward_func": 0.5000000149011612, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 57 | |
| }, | |
| { | |
| "completion_length": 39.91666793823242, | |
| "epoch": 0.7631578947368421, | |
| "grad_norm": 1.424776813399897, | |
| "kl": 2.8671875, | |
| "learning_rate": 4.3712768704277524e-07, | |
| "loss": 0.0029, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.22233545035123825, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 58 | |
| }, | |
| { | |
| "completion_length": 70.31250190734863, | |
| "epoch": 0.7763157894736842, | |
| "grad_norm": 3.1751573512407827, | |
| "kl": 0.115234375, | |
| "learning_rate": 4.3475222930516473e-07, | |
| "loss": 0.0001, | |
| "reward": 0.873985230922699, | |
| "reward_std": 0.25253836810588837, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.4781518578529358, | |
| "step": 59 | |
| }, | |
| { | |
| "completion_length": 54.54166793823242, | |
| "epoch": 0.7894736842105263, | |
| "grad_norm": 58.36302253834897, | |
| "kl": 1.03515625, | |
| "learning_rate": 4.32339438165092e-07, | |
| "loss": 0.001, | |
| "reward": 0.916666716337204, | |
| "reward_std": 0.42927367985248566, | |
| "rewards/correct_code_reward_func": 0.416666679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 60 | |
| }, | |
| { | |
| "completion_length": 63.583335876464844, | |
| "epoch": 0.8026315789473685, | |
| "grad_norm": 2.827911135736344, | |
| "kl": 0.29736328125, | |
| "learning_rate": 4.29889801182047e-07, | |
| "loss": 0.0003, | |
| "reward": 0.791666716337204, | |
| "reward_std": 0.1451837606728077, | |
| "rewards/correct_code_reward_func": 0.29166667722165585, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 61 | |
| }, | |
| { | |
| "completion_length": 65.41666793823242, | |
| "epoch": 0.8157894736842105, | |
| "grad_norm": 1.5958266464282538, | |
| "kl": 0.065673828125, | |
| "learning_rate": 4.274038133610628e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8333333730697632, | |
| "reward_std": 0.15430335700511932, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 62 | |
| }, | |
| { | |
| "completion_length": 31.270834922790527, | |
| "epoch": 0.8289473684210527, | |
| "grad_norm": 6.489845534414021, | |
| "kl": 1.37109375, | |
| "learning_rate": 4.248819770526884e-07, | |
| "loss": 0.0014, | |
| "reward": 1.1250000298023224, | |
| "reward_std": 0.376638799905777, | |
| "rewards/correct_code_reward_func": 0.6250000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 63 | |
| }, | |
| { | |
| "completion_length": 27.750000953674316, | |
| "epoch": 0.8421052631578947, | |
| "grad_norm": 1.7297180083254418, | |
| "kl": 4.314453125, | |
| "learning_rate": 4.223248018514777e-07, | |
| "loss": 0.0043, | |
| "reward": 1.1458333730697632, | |
| "reward_std": 0.30859364569187164, | |
| "rewards/correct_code_reward_func": 0.6458333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 64 | |
| }, | |
| { | |
| "completion_length": 47.35416793823242, | |
| "epoch": 0.8552631578947368, | |
| "grad_norm": 15.360105674599206, | |
| "kl": 0.5107421875, | |
| "learning_rate": 4.1973280449301364e-07, | |
| "loss": 0.0005, | |
| "reward": 1.0833333730697632, | |
| "reward_std": 0.3177001550793648, | |
| "rewards/correct_code_reward_func": 0.5833333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 65 | |
| }, | |
| { | |
| "completion_length": 34.500000953674316, | |
| "epoch": 0.868421052631579, | |
| "grad_norm": 0.8012424945081584, | |
| "kl": 0.1363525390625, | |
| "learning_rate": 4.171065087494909e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7083333730697632, | |
| "reward_std": 0.07715167850255966, | |
| "rewards/correct_code_reward_func": 0.2083333395421505, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 66 | |
| }, | |
| { | |
| "completion_length": 68.00000190734863, | |
| "epoch": 0.881578947368421, | |
| "grad_norm": 1.8876444286085294, | |
| "kl": 0.2359619140625, | |
| "learning_rate": 4.144464453238748e-07, | |
| "loss": 0.0002, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.36751919239759445, | |
| "rewards/correct_code_reward_func": 0.4791666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 67 | |
| }, | |
| { | |
| "completion_length": 111.93750762939453, | |
| "epoch": 0.8947368421052632, | |
| "grad_norm": 0.7306458054174331, | |
| "kl": 0.102294921875, | |
| "learning_rate": 4.1175315174266135e-07, | |
| "loss": 0.0001, | |
| "reward": 0.5625000298023224, | |
| "reward_std": 0.1767766959965229, | |
| "rewards/correct_code_reward_func": 0.06250000186264515, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 68 | |
| }, | |
| { | |
| "completion_length": 92.27083969116211, | |
| "epoch": 0.9078947368421053, | |
| "grad_norm": 1.8168097569883273, | |
| "kl": 0.185546875, | |
| "learning_rate": 4.090271722472576e-07, | |
| "loss": 0.0002, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.28126100450754166, | |
| "rewards/correct_code_reward_func": 0.5, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 69 | |
| }, | |
| { | |
| "completion_length": 86.77083587646484, | |
| "epoch": 0.9210526315789473, | |
| "grad_norm": 2.3675233810359515, | |
| "kl": 1.0478515625, | |
| "learning_rate": 4.062690576840051e-07, | |
| "loss": 0.001, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.32964862883090973, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 70 | |
| }, | |
| { | |
| "completion_length": 57.66666793823242, | |
| "epoch": 0.9342105263157895, | |
| "grad_norm": 51.022759357987056, | |
| "kl": 8.755859375, | |
| "learning_rate": 4.0347936539286874e-07, | |
| "loss": 0.0088, | |
| "reward": 1.0, | |
| "reward_std": 0.19500282034277916, | |
| "rewards/correct_code_reward_func": 0.5, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 71 | |
| }, | |
| { | |
| "completion_length": 79.37500381469727, | |
| "epoch": 0.9473684210526315, | |
| "grad_norm": 1.1663678155323538, | |
| "kl": 0.1044921875, | |
| "learning_rate": 4.006586590948141e-07, | |
| "loss": 0.0001, | |
| "reward": 1.048139214515686, | |
| "reward_std": 0.17669584602117538, | |
| "rewards/correct_code_reward_func": 0.5625000149011612, | |
| "rewards/len_reward_func": 0.48563915491104126, | |
| "step": 72 | |
| }, | |
| { | |
| "completion_length": 35.62500190734863, | |
| "epoch": 0.9605263157894737, | |
| "grad_norm": 1.1660255366197148, | |
| "kl": 0.2127685546875, | |
| "learning_rate": 3.9780750877789394e-07, | |
| "loss": 0.0002, | |
| "reward": 0.9375000596046448, | |
| "reward_std": 0.24056154489517212, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 73 | |
| }, | |
| { | |
| "completion_length": 63.70833396911621, | |
| "epoch": 0.9736842105263158, | |
| "grad_norm": 1.1033519234822637, | |
| "kl": 0.115234375, | |
| "learning_rate": 3.9492649058206964e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8333333730697632, | |
| "reward_std": 0.3491683229804039, | |
| "rewards/correct_code_reward_func": 0.3541666679084301, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 74 | |
| }, | |
| { | |
| "completion_length": 76.02083396911621, | |
| "epoch": 0.9868421052631579, | |
| "grad_norm": 1.0147417730649864, | |
| "kl": 0.085693359375, | |
| "learning_rate": 3.920161866827889e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9223043918609619, | |
| "reward_std": 0.26814398169517517, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.4848043918609619, | |
| "step": 75 | |
| }, | |
| { | |
| "completion_length": 37.04166793823242, | |
| "epoch": 1.0, | |
| "grad_norm": 53.40321846717754, | |
| "kl": 1.2109375, | |
| "learning_rate": 3.8907718517334396e-07, | |
| "loss": 0.0012, | |
| "reward": 1.125, | |
| "reward_std": 0.2994871288537979, | |
| "rewards/correct_code_reward_func": 0.625, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 76 | |
| }, | |
| { | |
| "completion_length": 109.77083587646484, | |
| "epoch": 1.013157894736842, | |
| "grad_norm": 1.2927063784991473, | |
| "kl": 0.1328125, | |
| "learning_rate": 3.861100799460336e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6458333730697632, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.1458333395421505, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 77 | |
| }, | |
| { | |
| "completion_length": 55.6875, | |
| "epoch": 1.0263157894736843, | |
| "grad_norm": 1.118771460054152, | |
| "kl": 0.130859375, | |
| "learning_rate": 3.831154705721541e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0, | |
| "reward_std": 0.2342708371579647, | |
| "rewards/correct_code_reward_func": 0.5, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 78 | |
| }, | |
| { | |
| "completion_length": 46.25000190734863, | |
| "epoch": 1.0394736842105263, | |
| "grad_norm": 6.523486555479139, | |
| "kl": 0.462158203125, | |
| "learning_rate": 3.800939621808419e-07, | |
| "loss": 0.0005, | |
| "reward": 0.9583333432674408, | |
| "reward_std": 0.22233543917536736, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 79 | |
| }, | |
| { | |
| "completion_length": 77.52083587646484, | |
| "epoch": 1.0526315789473684, | |
| "grad_norm": 1.2611203201996803, | |
| "kl": 0.10211181640625, | |
| "learning_rate": 3.7704616533679334e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7708333432674408, | |
| "reward_std": 0.438380166888237, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 80 | |
| }, | |
| { | |
| "completion_length": 31.666666984558105, | |
| "epoch": 1.0657894736842106, | |
| "grad_norm": 6780.981931743591, | |
| "kl": 2705.0, | |
| "learning_rate": 3.7397269591688663e-07, | |
| "loss": 2.7153, | |
| "reward": 0.9375, | |
| "reward_std": 0.13607724383473396, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 81 | |
| }, | |
| { | |
| "completion_length": 60.08333396911621, | |
| "epoch": 1.0789473684210527, | |
| "grad_norm": 1.4225091724950767, | |
| "kl": 0.09033203125, | |
| "learning_rate": 3.7087417498572944e-07, | |
| "loss": 0.0001, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.34018656611442566, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 82 | |
| }, | |
| { | |
| "completion_length": 47.95833396911621, | |
| "epoch": 1.0921052631578947, | |
| "grad_norm": 39.92181658742887, | |
| "kl": 21.05078125, | |
| "learning_rate": 3.6775122867015865e-07, | |
| "loss": 0.021, | |
| "reward": 1.1666667461395264, | |
| "reward_std": 0.34503278136253357, | |
| "rewards/correct_code_reward_func": 0.6666666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 83 | |
| }, | |
| { | |
| "completion_length": 30.291667938232422, | |
| "epoch": 1.1052631578947367, | |
| "grad_norm": 1.8228407124659103, | |
| "kl": 0.380859375, | |
| "learning_rate": 3.6460448803271754e-07, | |
| "loss": 0.0004, | |
| "reward": 1.1875000596046448, | |
| "reward_std": 0.2931964099407196, | |
| "rewards/correct_code_reward_func": 0.6875000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 84 | |
| }, | |
| { | |
| "completion_length": 48.1875, | |
| "epoch": 1.118421052631579, | |
| "grad_norm": 0.7759500560241989, | |
| "kl": 0.359130859375, | |
| "learning_rate": 3.614345889441346e-07, | |
| "loss": 0.0004, | |
| "reward": 0.7019230872392654, | |
| "reward_std": 0.09528262168169022, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.49358974397182465, | |
| "step": 85 | |
| }, | |
| { | |
| "completion_length": 31.58333396911621, | |
| "epoch": 1.131578947368421, | |
| "grad_norm": 522.7574671201318, | |
| "kl": 133.271484375, | |
| "learning_rate": 3.5824217195483176e-07, | |
| "loss": 0.1335, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.13607724383473396, | |
| "rewards/correct_code_reward_func": 0.3541666679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 86 | |
| }, | |
| { | |
| "completion_length": 41.29166793823242, | |
| "epoch": 1.1447368421052633, | |
| "grad_norm": 1.2754536374690895, | |
| "kl": 0.64208984375, | |
| "learning_rate": 3.550278821654866e-07, | |
| "loss": 0.0006, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.520833358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 87 | |
| }, | |
| { | |
| "completion_length": 49.52083396911621, | |
| "epoch": 1.1578947368421053, | |
| "grad_norm": 1.1388596254339887, | |
| "kl": 0.1265869140625, | |
| "learning_rate": 3.5179236909667464e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0, | |
| "reward_std": 0.1178511306643486, | |
| "rewards/correct_code_reward_func": 0.5, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 88 | |
| }, | |
| { | |
| "completion_length": 56.104169845581055, | |
| "epoch": 1.1710526315789473, | |
| "grad_norm": 4.8354182485526875, | |
| "kl": 1.86669921875, | |
| "learning_rate": 3.485362865576194e-07, | |
| "loss": 0.0019, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.3541666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 89 | |
| }, | |
| { | |
| "completion_length": 55.562503814697266, | |
| "epoch": 1.1842105263157894, | |
| "grad_norm": 1.8509215827690364, | |
| "kl": 0.501953125, | |
| "learning_rate": 3.4526029251407505e-07, | |
| "loss": 0.0005, | |
| "reward": 0.6250000298023224, | |
| "reward_std": 0.22233543917536736, | |
| "rewards/correct_code_reward_func": 0.12500000558793545, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 90 | |
| }, | |
| { | |
| "completion_length": 56.89583396911621, | |
| "epoch": 1.1973684210526316, | |
| "grad_norm": 3.479508392096908, | |
| "kl": 0.2423095703125, | |
| "learning_rate": 3.4196504895536943e-07, | |
| "loss": 0.0002, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.24383928626775742, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 91 | |
| }, | |
| { | |
| "completion_length": 49.79166793823242, | |
| "epoch": 1.2105263157894737, | |
| "grad_norm": 1.436722605439725, | |
| "kl": 0.130126953125, | |
| "learning_rate": 3.3865122176063385e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0625000298023224, | |
| "reward_std": 0.2041093371808529, | |
| "rewards/correct_code_reward_func": 0.5625000223517418, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 92 | |
| }, | |
| { | |
| "completion_length": 57.58333396911621, | |
| "epoch": 1.2236842105263157, | |
| "grad_norm": 3.889581407116151, | |
| "kl": 1.8251953125, | |
| "learning_rate": 3.3531948056424764e-07, | |
| "loss": 0.0018, | |
| "reward": 0.7708333730697632, | |
| "reward_std": 0.204109326004982, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 93 | |
| }, | |
| { | |
| "completion_length": 33.33333492279053, | |
| "epoch": 1.236842105263158, | |
| "grad_norm": 67.13793060279492, | |
| "kl": 26.9716796875, | |
| "learning_rate": 3.319704986205223e-07, | |
| "loss": 0.0269, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.22233543917536736, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 94 | |
| }, | |
| { | |
| "completion_length": 68.54166793823242, | |
| "epoch": 1.25, | |
| "grad_norm": 0.8593629362399875, | |
| "kl": 0.2412109375, | |
| "learning_rate": 3.2860495266765597e-07, | |
| "loss": 0.0002, | |
| "reward": 0.75, | |
| "reward_std": 0.26726123690605164, | |
| "rewards/correct_code_reward_func": 0.25, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 95 | |
| }, | |
| { | |
| "completion_length": 114.70833587646484, | |
| "epoch": 1.263157894736842, | |
| "grad_norm": 0.4762384719502881, | |
| "kl": 0.31884765625, | |
| "learning_rate": 3.252235227909825e-07, | |
| "loss": 0.0003, | |
| "reward": 0.5833333432674408, | |
| "reward_std": 0.1451837606728077, | |
| "rewards/correct_code_reward_func": 0.08333333395421505, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 96 | |
| }, | |
| { | |
| "completion_length": 69.75, | |
| "epoch": 1.2763157894736843, | |
| "grad_norm": 0.9237831880116578, | |
| "kl": 0.156494140625, | |
| "learning_rate": 3.2182689228554515e-07, | |
| "loss": 0.0002, | |
| "reward": 1.0416667461395264, | |
| "reward_std": 0.24966806918382645, | |
| "rewards/correct_code_reward_func": 0.5416666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 97 | |
| }, | |
| { | |
| "completion_length": 56.708335876464844, | |
| "epoch": 1.2894736842105263, | |
| "grad_norm": 86.96758033365275, | |
| "kl": 16.03466796875, | |
| "learning_rate": 3.184157475180207e-07, | |
| "loss": 0.016, | |
| "reward": 1.0416667461395264, | |
| "reward_std": 0.2994871214032173, | |
| "rewards/correct_code_reward_func": 0.5416666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 98 | |
| }, | |
| { | |
| "completion_length": 66.27083587646484, | |
| "epoch": 1.3026315789473684, | |
| "grad_norm": 2.4485962428130286, | |
| "kl": 0.5283203125, | |
| "learning_rate": 3.1499077778802387e-07, | |
| "loss": 0.0005, | |
| "reward": 0.8958333730697632, | |
| "reward_std": 0.4068002998828888, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 99 | |
| }, | |
| { | |
| "completion_length": 53.770835876464844, | |
| "epoch": 1.3157894736842106, | |
| "grad_norm": 2.0445270805974585, | |
| "kl": 1.6337890625, | |
| "learning_rate": 3.115526751888181e-07, | |
| "loss": 0.0016, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.30859364569187164, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 100 | |
| }, | |
| { | |
| "completion_length": 119.14583587646484, | |
| "epoch": 1.3289473684210527, | |
| "grad_norm": 1.2383154787127502, | |
| "kl": 0.08740234375, | |
| "learning_rate": 3.081021344674632e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9476010203361511, | |
| "reward_std": 0.2919590622186661, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.4892676770687103, | |
| "step": 101 | |
| }, | |
| { | |
| "completion_length": 101.10416793823242, | |
| "epoch": 1.3421052631578947, | |
| "grad_norm": 408.9106300781395, | |
| "kl": 77.0654296875, | |
| "learning_rate": 3.0463985288442474e-07, | |
| "loss": 0.0775, | |
| "reward": 0.5625, | |
| "reward_std": 0.08625819534063339, | |
| "rewards/correct_code_reward_func": 0.0625, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 102 | |
| }, | |
| { | |
| "completion_length": 106.85416984558105, | |
| "epoch": 1.3552631578947367, | |
| "grad_norm": 1.2917146685193897, | |
| "kl": 0.157958984375, | |
| "learning_rate": 3.011665300726775e-07, | |
| "loss": 0.0002, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.28126102685928345, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 103 | |
| }, | |
| { | |
| "completion_length": 54.458335876464844, | |
| "epoch": 1.368421052631579, | |
| "grad_norm": 488.53335548972836, | |
| "kl": 259.2265625, | |
| "learning_rate": 2.976828678963284e-07, | |
| "loss": 0.2594, | |
| "reward": 0.6458333432674408, | |
| "reward_std": 0.28126102685928345, | |
| "rewards/correct_code_reward_func": 0.1666666716337204, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 104 | |
| }, | |
| { | |
| "completion_length": 78.89583396911621, | |
| "epoch": 1.381578947368421, | |
| "grad_norm": 2.403194346252636, | |
| "kl": 0.272705078125, | |
| "learning_rate": 2.941895703087887e-07, | |
| "loss": 0.0003, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.2903675436973572, | |
| "rewards/correct_code_reward_func": 0.2083333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 105 | |
| }, | |
| { | |
| "completion_length": 70.89583587646484, | |
| "epoch": 1.3947368421052633, | |
| "grad_norm": 5.590757587987271, | |
| "kl": 0.32861328125, | |
| "learning_rate": 2.906873432105244e-07, | |
| "loss": 0.0003, | |
| "reward": 0.979166716337204, | |
| "reward_std": 0.48676779866218567, | |
| "rewards/correct_code_reward_func": 0.4791666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 106 | |
| }, | |
| { | |
| "completion_length": 66.0625, | |
| "epoch": 1.4078947368421053, | |
| "grad_norm": 1.8305343201041324, | |
| "kl": 0.4862060546875, | |
| "learning_rate": 2.871768943064129e-07, | |
| "loss": 0.0005, | |
| "reward": 1.0000000596046448, | |
| "reward_std": 0.2721545100212097, | |
| "rewards/correct_code_reward_func": 0.5000000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 107 | |
| }, | |
| { | |
| "completion_length": 54.06250190734863, | |
| "epoch": 1.4210526315789473, | |
| "grad_norm": 0.6615096170114014, | |
| "kl": 0.1812744140625, | |
| "learning_rate": 2.8365893296273487e-07, | |
| "loss": 0.0002, | |
| "reward": 0.875, | |
| "reward_std": 0.28215693682432175, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 108 | |
| }, | |
| { | |
| "completion_length": 73.29166984558105, | |
| "epoch": 1.4342105263157894, | |
| "grad_norm": 13.375888897910476, | |
| "kl": 2.66064453125, | |
| "learning_rate": 2.801341700638307e-07, | |
| "loss": 0.0027, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.07715167850255966, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 109 | |
| }, | |
| { | |
| "completion_length": 70.45833587646484, | |
| "epoch": 1.4473684210526316, | |
| "grad_norm": 0.8193690701749492, | |
| "kl": 0.42578125, | |
| "learning_rate": 2.766033178684506e-07, | |
| "loss": 0.0004, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.24339043349027634, | |
| "rewards/correct_code_reward_func": 0.2500000111758709, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 110 | |
| }, | |
| { | |
| "completion_length": 78.39583587646484, | |
| "epoch": 1.4605263157894737, | |
| "grad_norm": 811.8284865728368, | |
| "kl": 149.04150390625, | |
| "learning_rate": 2.730670898658255e-07, | |
| "loss": 0.1496, | |
| "reward": 0.5625, | |
| "reward_std": 0.08625819534063339, | |
| "rewards/correct_code_reward_func": 0.0625, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 111 | |
| }, | |
| { | |
| "completion_length": 91.58333396911621, | |
| "epoch": 1.4736842105263157, | |
| "grad_norm": 44.2662547081144, | |
| "kl": 3.12255859375, | |
| "learning_rate": 2.6952620063149116e-07, | |
| "loss": 0.0031, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.3731769919395447, | |
| "rewards/correct_code_reward_func": 0.4791666865348816, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 112 | |
| }, | |
| { | |
| "completion_length": 58.895835876464844, | |
| "epoch": 1.486842105263158, | |
| "grad_norm": 4.041649610812466, | |
| "kl": 0.2001953125, | |
| "learning_rate": 2.6598136568289144e-07, | |
| "loss": 0.0002, | |
| "reward": 0.7708333730697632, | |
| "reward_std": 0.22516431659460068, | |
| "rewards/correct_code_reward_func": 0.2708333395421505, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 113 | |
| }, | |
| { | |
| "completion_length": 62.20833396911621, | |
| "epoch": 1.5, | |
| "grad_norm": 2.891055687279037, | |
| "kl": 0.6484375, | |
| "learning_rate": 2.624333013347917e-07, | |
| "loss": 0.0006, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.1480126492679119, | |
| "rewards/correct_code_reward_func": 0.3958333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 114 | |
| }, | |
| { | |
| "completion_length": 45.31250190734863, | |
| "epoch": 1.513157894736842, | |
| "grad_norm": 0.7564583724153957, | |
| "kl": 0.769287109375, | |
| "learning_rate": 2.5888272455453133e-07, | |
| "loss": 0.0008, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.0589255653321743, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 115 | |
| }, | |
| { | |
| "completion_length": 59.270835876464844, | |
| "epoch": 1.526315789473684, | |
| "grad_norm": 1.5880223387121624, | |
| "kl": 1.41796875, | |
| "learning_rate": 2.5533035281714365e-07, | |
| "loss": 0.0014, | |
| "reward": 0.6041666865348816, | |
| "reward_std": 0.25392838567495346, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 116 | |
| }, | |
| { | |
| "completion_length": 57.583335876464844, | |
| "epoch": 1.5394736842105263, | |
| "grad_norm": 16.81790414189794, | |
| "kl": 10.380859375, | |
| "learning_rate": 2.5177690396037436e-07, | |
| "loss": 0.0104, | |
| "reward": 1.0625, | |
| "reward_std": 0.1480126492679119, | |
| "rewards/correct_code_reward_func": 0.5625, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 117 | |
| }, | |
| { | |
| "completion_length": 103.41667175292969, | |
| "epoch": 1.5526315789473686, | |
| "grad_norm": 6.8725768759144845, | |
| "kl": 0.1064453125, | |
| "learning_rate": 2.482230960396256e-07, | |
| "loss": 0.0001, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.4173382371664047, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 118 | |
| }, | |
| { | |
| "completion_length": 64.25000190734863, | |
| "epoch": 1.5657894736842106, | |
| "grad_norm": 1.6956587222545918, | |
| "kl": 0.2694091796875, | |
| "learning_rate": 2.4466964718285633e-07, | |
| "loss": 0.0003, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.21322892233729362, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 119 | |
| }, | |
| { | |
| "completion_length": 48.208335876464844, | |
| "epoch": 1.5789473684210527, | |
| "grad_norm": 0.8688231987899243, | |
| "kl": 0.05615234375, | |
| "learning_rate": 2.411172754454688e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0000000298023224, | |
| "reward_std": 0.2994871288537979, | |
| "rewards/correct_code_reward_func": 0.5000000223517418, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 120 | |
| }, | |
| { | |
| "completion_length": 31.479166984558105, | |
| "epoch": 1.5921052631578947, | |
| "grad_norm": 1.0911022619943564, | |
| "kl": 0.193359375, | |
| "learning_rate": 2.375666986652083e-07, | |
| "loss": 0.0002, | |
| "reward": 1.4166666865348816, | |
| "reward_std": 0.19500282034277916, | |
| "rewards/correct_code_reward_func": 0.9166666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 121 | |
| }, | |
| { | |
| "completion_length": 49.91666793823242, | |
| "epoch": 1.6052631578947367, | |
| "grad_norm": 10.939422149518188, | |
| "kl": 0.50537109375, | |
| "learning_rate": 2.3401863431710862e-07, | |
| "loss": 0.0005, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.28408990427851677, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 122 | |
| }, | |
| { | |
| "completion_length": 47.72916793823242, | |
| "epoch": 1.618421052631579, | |
| "grad_norm": 1.2714992325565007, | |
| "kl": 0.097412109375, | |
| "learning_rate": 2.3047379936850882e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1458333730697632, | |
| "reward_std": 0.2931964248418808, | |
| "rewards/correct_code_reward_func": 0.645833358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 123 | |
| }, | |
| { | |
| "completion_length": 76.47916984558105, | |
| "epoch": 1.631578947368421, | |
| "grad_norm": 18.033727216842927, | |
| "kl": 5.46875, | |
| "learning_rate": 2.2693291013417452e-07, | |
| "loss": 0.0055, | |
| "reward": 0.8958333730697632, | |
| "reward_std": 0.3177132308483124, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 124 | |
| }, | |
| { | |
| "completion_length": 57.583335876464844, | |
| "epoch": 1.6447368421052633, | |
| "grad_norm": 10.905825675003781, | |
| "kl": 10.9765625, | |
| "learning_rate": 2.2339668213154941e-07, | |
| "loss": 0.011, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.28408990427851677, | |
| "rewards/correct_code_reward_func": 0.2916666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 125 | |
| }, | |
| { | |
| "completion_length": 38.31250190734863, | |
| "epoch": 1.6578947368421053, | |
| "grad_norm": 1.7809035413529761, | |
| "kl": 0.0531005859375, | |
| "learning_rate": 2.1986582993616925e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.3535194396972656, | |
| "rewards/correct_code_reward_func": 0.520833358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 126 | |
| }, | |
| { | |
| "completion_length": 60.60416793823242, | |
| "epoch": 1.6710526315789473, | |
| "grad_norm": 0.8551909967151841, | |
| "kl": 0.1240234375, | |
| "learning_rate": 2.1634106703726518e-07, | |
| "loss": 0.0001, | |
| "reward": 0.9375, | |
| "reward_std": 0.30231600999832153, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 127 | |
| }, | |
| { | |
| "completion_length": 146.62500762939453, | |
| "epoch": 1.6842105263157894, | |
| "grad_norm": 0.5963722529446751, | |
| "kl": 0.032470703125, | |
| "learning_rate": 2.1282310569358704e-07, | |
| "loss": 0.0, | |
| "reward": 0.854166716337204, | |
| "reward_std": 0.23144196718931198, | |
| "rewards/correct_code_reward_func": 0.354166679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 128 | |
| }, | |
| { | |
| "completion_length": 79.31250381469727, | |
| "epoch": 1.6973684210526314, | |
| "grad_norm": 11.657983637810885, | |
| "kl": 5.33203125, | |
| "learning_rate": 2.093126567894755e-07, | |
| "loss": 0.0053, | |
| "reward": 0.75, | |
| "reward_std": 0.22233545035123825, | |
| "rewards/correct_code_reward_func": 0.25, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 129 | |
| }, | |
| { | |
| "completion_length": 40.54166793823242, | |
| "epoch": 1.7105263157894737, | |
| "grad_norm": 2.9861227183589163, | |
| "kl": 1.69921875, | |
| "learning_rate": 2.0581042969121132e-07, | |
| "loss": 0.0017, | |
| "reward": 0.9375, | |
| "reward_std": 0.320529043674469, | |
| "rewards/correct_code_reward_func": 0.4375, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 130 | |
| }, | |
| { | |
| "completion_length": 35.64583396911621, | |
| "epoch": 1.723684210526316, | |
| "grad_norm": 1.119123230040456, | |
| "kl": 1.509521484375, | |
| "learning_rate": 2.023171321036716e-07, | |
| "loss": 0.0015, | |
| "reward": 1.3541666865348816, | |
| "reward_std": 0.23144196718931198, | |
| "rewards/correct_code_reward_func": 0.8541666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 131 | |
| }, | |
| { | |
| "completion_length": 63.04166793823242, | |
| "epoch": 1.736842105263158, | |
| "grad_norm": 1.328286772215648, | |
| "kl": 0.2412109375, | |
| "learning_rate": 1.9883346992732254e-07, | |
| "loss": 0.0002, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.4096161276102066, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 132 | |
| }, | |
| { | |
| "completion_length": 50.29166793823242, | |
| "epoch": 1.75, | |
| "grad_norm": 16.589627428183928, | |
| "kl": 14.0625, | |
| "learning_rate": 1.9536014711557526e-07, | |
| "loss": 0.0141, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.1451837606728077, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 133 | |
| }, | |
| { | |
| "completion_length": 80.91666793823242, | |
| "epoch": 1.763157894736842, | |
| "grad_norm": 0.6658576872819127, | |
| "kl": 0.183837890625, | |
| "learning_rate": 1.9189786553253687e-07, | |
| "loss": 0.0002, | |
| "reward": 0.9375000298023224, | |
| "reward_std": 0.25392838940024376, | |
| "rewards/correct_code_reward_func": 0.4375000223517418, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 134 | |
| }, | |
| { | |
| "completion_length": 44.06250190734863, | |
| "epoch": 1.776315789473684, | |
| "grad_norm": 20.037402776427545, | |
| "kl": 1.3486328125, | |
| "learning_rate": 1.884473248111818e-07, | |
| "loss": 0.0013, | |
| "reward": 1.0208333432674408, | |
| "reward_std": 0.32964862883090973, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 135 | |
| }, | |
| { | |
| "completion_length": 38.9375, | |
| "epoch": 1.7894736842105263, | |
| "grad_norm": 1.4127858285242418, | |
| "kl": 0.097900390625, | |
| "learning_rate": 1.8500922221197619e-07, | |
| "loss": 0.0001, | |
| "reward": 1.1041666865348816, | |
| "reward_std": 0.320529043674469, | |
| "rewards/correct_code_reward_func": 0.6041666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 136 | |
| }, | |
| { | |
| "completion_length": 61.75, | |
| "epoch": 1.8026315789473686, | |
| "grad_norm": 19.7909578978993, | |
| "kl": 0.2247314453125, | |
| "learning_rate": 1.8158425248197928e-07, | |
| "loss": 0.0002, | |
| "reward": 1.1041667461395264, | |
| "reward_std": 0.3219604715704918, | |
| "rewards/correct_code_reward_func": 0.6250000298023224, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 137 | |
| }, | |
| { | |
| "completion_length": 60.625, | |
| "epoch": 1.8157894736842106, | |
| "grad_norm": 1.5841720040680045, | |
| "kl": 0.15380859375, | |
| "learning_rate": 1.7817310771445488e-07, | |
| "loss": 0.0002, | |
| "reward": 1.0833333730697632, | |
| "reward_std": 0.1451837606728077, | |
| "rewards/correct_code_reward_func": 0.583333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 138 | |
| }, | |
| { | |
| "completion_length": 48.97916793823242, | |
| "epoch": 1.8289473684210527, | |
| "grad_norm": 2.2749307920378903, | |
| "kl": 1.35400390625, | |
| "learning_rate": 1.7477647720901746e-07, | |
| "loss": 0.0014, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.32964862883090973, | |
| "rewards/correct_code_reward_func": 0.5000000298023224, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 139 | |
| }, | |
| { | |
| "completion_length": 29.666667938232422, | |
| "epoch": 1.8421052631578947, | |
| "grad_norm": 40.21756072339013, | |
| "kl": 2.6141357421875, | |
| "learning_rate": 1.7139504733234412e-07, | |
| "loss": 0.0026, | |
| "reward": 1.104166716337204, | |
| "reward_std": 0.30859363824129105, | |
| "rewards/correct_code_reward_func": 0.6041666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 140 | |
| }, | |
| { | |
| "completion_length": 84.70833587646484, | |
| "epoch": 1.8552631578947367, | |
| "grad_norm": 37.379935422229906, | |
| "kl": 7.72314453125, | |
| "learning_rate": 1.680295013794778e-07, | |
| "loss": 0.0078, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.4159068167209625, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 141 | |
| }, | |
| { | |
| "completion_length": 116.58333969116211, | |
| "epoch": 1.868421052631579, | |
| "grad_norm": 21.10627645969623, | |
| "kl": 8.8427734375, | |
| "learning_rate": 1.646805194357524e-07, | |
| "loss": 0.0088, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 142 | |
| }, | |
| { | |
| "completion_length": 47.06250190734863, | |
| "epoch": 1.881578947368421, | |
| "grad_norm": 193.23023765526983, | |
| "kl": 42.0615234375, | |
| "learning_rate": 1.6134877823936607e-07, | |
| "loss": 0.0421, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.3794676959514618, | |
| "rewards/correct_code_reward_func": 0.4791666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 143 | |
| }, | |
| { | |
| "completion_length": 32.20833396911621, | |
| "epoch": 1.8947368421052633, | |
| "grad_norm": 1.3678936634574226, | |
| "kl": 0.09912109375, | |
| "learning_rate": 1.580349510446306e-07, | |
| "loss": 0.0001, | |
| "reward": 1.0000000298023224, | |
| "reward_std": 0.2721545025706291, | |
| "rewards/correct_code_reward_func": 0.5000000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 144 | |
| }, | |
| { | |
| "completion_length": 71.08333587646484, | |
| "epoch": 1.9078947368421053, | |
| "grad_norm": 1.2737509817094077, | |
| "kl": 0.795654296875, | |
| "learning_rate": 1.547397074859249e-07, | |
| "loss": 0.0008, | |
| "reward": 0.8333333730697632, | |
| "reward_std": 0.0, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 145 | |
| }, | |
| { | |
| "completion_length": 65.10416984558105, | |
| "epoch": 1.9210526315789473, | |
| "grad_norm": 1.0612643430461282, | |
| "kl": 0.25439453125, | |
| "learning_rate": 1.514637134423806e-07, | |
| "loss": 0.0003, | |
| "reward": 0.6250000298023224, | |
| "reward_std": 0.1451837606728077, | |
| "rewards/correct_code_reward_func": 0.12500000558793545, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 146 | |
| }, | |
| { | |
| "completion_length": 50.70833396911621, | |
| "epoch": 1.9342105263157894, | |
| "grad_norm": 0.9018149237891075, | |
| "kl": 0.42431640625, | |
| "learning_rate": 1.482076309033254e-07, | |
| "loss": 0.0004, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.19500280916690826, | |
| "rewards/correct_code_reward_func": 0.4791666865348816, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 147 | |
| }, | |
| { | |
| "completion_length": 77.93750381469727, | |
| "epoch": 1.9473684210526314, | |
| "grad_norm": 0.8274466826539039, | |
| "kl": 0.40673828125, | |
| "learning_rate": 1.4497211783451352e-07, | |
| "loss": 0.0004, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.22233543917536736, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 148 | |
| }, | |
| { | |
| "completion_length": 53.458335876464844, | |
| "epoch": 1.9605263157894737, | |
| "grad_norm": 47.18259760797994, | |
| "kl": 21.875, | |
| "learning_rate": 1.4175782804516822e-07, | |
| "loss": 0.0219, | |
| "reward": 1.0625, | |
| "reward_std": 0.42016713321208954, | |
| "rewards/correct_code_reward_func": 0.5625, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 149 | |
| }, | |
| { | |
| "completion_length": 50.5625, | |
| "epoch": 1.973684210526316, | |
| "grad_norm": 3.162565983897937, | |
| "kl": 0.3603515625, | |
| "learning_rate": 1.3856541105586545e-07, | |
| "loss": 0.0004, | |
| "reward": 1.3333333730697632, | |
| "reward_std": 0.2342708334326744, | |
| "rewards/correct_code_reward_func": 0.8333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 150 | |
| }, | |
| { | |
| "completion_length": 43.81250190734863, | |
| "epoch": 1.986842105263158, | |
| "grad_norm": 1.4665389553089878, | |
| "kl": 0.22265625, | |
| "learning_rate": 1.3539551196728249e-07, | |
| "loss": 0.0002, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.3128539249300957, | |
| "rewards/correct_code_reward_func": 0.3750000223517418, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 151 | |
| }, | |
| { | |
| "completion_length": 54.72916793823242, | |
| "epoch": 2.0, | |
| "grad_norm": 1.6783852431327184, | |
| "kl": 0.2939453125, | |
| "learning_rate": 1.322487713298413e-07, | |
| "loss": 0.0003, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.33592625707387924, | |
| "rewards/correct_code_reward_func": 0.4791666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 152 | |
| }, | |
| { | |
| "completion_length": 65.62500381469727, | |
| "epoch": 2.013157894736842, | |
| "grad_norm": 0.870944500815664, | |
| "kl": 0.1328125, | |
| "learning_rate": 1.2912582501427061e-07, | |
| "loss": 0.0001, | |
| "reward": 0.6041666865348816, | |
| "reward_std": 0.21322893351316452, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 153 | |
| }, | |
| { | |
| "completion_length": 48.68750286102295, | |
| "epoch": 2.026315789473684, | |
| "grad_norm": 32.567721507115095, | |
| "kl": 11.8994140625, | |
| "learning_rate": 1.260273040831134e-07, | |
| "loss": 0.0119, | |
| "reward": 0.8058712482452393, | |
| "reward_std": 0.07767461240291595, | |
| "rewards/correct_code_reward_func": 0.3125000149011612, | |
| "rewards/len_reward_func": 0.49337121844291687, | |
| "step": 154 | |
| }, | |
| { | |
| "completion_length": 55.52083396911621, | |
| "epoch": 2.039473684210526, | |
| "grad_norm": 5.082828282284441, | |
| "kl": 1.729736328125, | |
| "learning_rate": 1.2295383466320674e-07, | |
| "loss": 0.0017, | |
| "reward": 0.9278694987297058, | |
| "reward_std": 0.3939250260591507, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.4903694987297058, | |
| "step": 155 | |
| }, | |
| { | |
| "completion_length": 41.666666984558105, | |
| "epoch": 2.0526315789473686, | |
| "grad_norm": 4.028610834972993, | |
| "kl": 0.8974609375, | |
| "learning_rate": 1.1990603781915814e-07, | |
| "loss": 0.0009, | |
| "reward": 1.0000000596046448, | |
| "reward_std": 0.350690558552742, | |
| "rewards/correct_code_reward_func": 0.5000000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 156 | |
| }, | |
| { | |
| "completion_length": 69.0625, | |
| "epoch": 2.0657894736842106, | |
| "grad_norm": 1.314671666949959, | |
| "kl": 0.165283203125, | |
| "learning_rate": 1.1688452942784591e-07, | |
| "loss": 0.0002, | |
| "reward": 0.9375000596046448, | |
| "reward_std": 0.24056155234575272, | |
| "rewards/correct_code_reward_func": 0.4375000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 157 | |
| }, | |
| { | |
| "completion_length": 46.95833492279053, | |
| "epoch": 2.0789473684210527, | |
| "grad_norm": 2.979789760395441, | |
| "kl": 2.431640625, | |
| "learning_rate": 1.138899200539663e-07, | |
| "loss": 0.0024, | |
| "reward": 0.6875, | |
| "reward_std": 0.13607725501060486, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 158 | |
| }, | |
| { | |
| "completion_length": 41.72916793823242, | |
| "epoch": 2.0921052631578947, | |
| "grad_norm": 2.3176148256976137, | |
| "kl": 0.5712890625, | |
| "learning_rate": 1.10922814826656e-07, | |
| "loss": 0.0006, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.2903675064444542, | |
| "rewards/correct_code_reward_func": 0.2083333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 159 | |
| }, | |
| { | |
| "completion_length": 58.625003814697266, | |
| "epoch": 2.1052631578947367, | |
| "grad_norm": 1.4397948211346696, | |
| "kl": 0.56298828125, | |
| "learning_rate": 1.0798381331721107e-07, | |
| "loss": 0.0006, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.39140307903289795, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 160 | |
| }, | |
| { | |
| "completion_length": 106.64583396911621, | |
| "epoch": 2.1184210526315788, | |
| "grad_norm": 16.776871927679135, | |
| "kl": 4.4949951171875, | |
| "learning_rate": 1.0507350941793042e-07, | |
| "loss": 0.0045, | |
| "reward": 0.6666666865348816, | |
| "reward_std": 0.24339043349027634, | |
| "rewards/correct_code_reward_func": 0.1875, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 161 | |
| }, | |
| { | |
| "completion_length": 59.9375, | |
| "epoch": 2.1315789473684212, | |
| "grad_norm": 13.001510018724911, | |
| "kl": 9.50634765625, | |
| "learning_rate": 1.0219249122210619e-07, | |
| "loss": 0.0095, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.2840898856520653, | |
| "rewards/correct_code_reward_func": 0.3750000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 162 | |
| }, | |
| { | |
| "completion_length": 35.39583396911621, | |
| "epoch": 2.1447368421052633, | |
| "grad_norm": 2.008422665082507, | |
| "kl": 1.0521240234375, | |
| "learning_rate": 9.934134090518592e-08, | |
| "loss": 0.001, | |
| "reward": 1.1250000596046448, | |
| "reward_std": 0.2721545100212097, | |
| "rewards/correct_code_reward_func": 0.6250000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 163 | |
| }, | |
| { | |
| "completion_length": 41.72916793823242, | |
| "epoch": 2.1578947368421053, | |
| "grad_norm": 3.008857027150773, | |
| "kl": 0.1142578125, | |
| "learning_rate": 9.652063460713117e-08, | |
| "loss": 0.0001, | |
| "reward": 1.0000000298023224, | |
| "reward_std": 0.15430335700511932, | |
| "rewards/correct_code_reward_func": 0.5000000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 164 | |
| }, | |
| { | |
| "completion_length": 52.54166793823242, | |
| "epoch": 2.1710526315789473, | |
| "grad_norm": 2.2942300832362656, | |
| "kl": 1.458984375, | |
| "learning_rate": 9.37309423159949e-08, | |
| "loss": 0.0015, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.2931964062154293, | |
| "rewards/correct_code_reward_func": 0.520833358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 165 | |
| }, | |
| { | |
| "completion_length": 67.10416793823242, | |
| "epoch": 2.1842105263157894, | |
| "grad_norm": 0.7847964783320711, | |
| "kl": 0.3076171875, | |
| "learning_rate": 9.097282775274238e-08, | |
| "loss": 0.0003, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.22516433894634247, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 166 | |
| }, | |
| { | |
| "completion_length": 62.458335876464844, | |
| "epoch": 2.1973684210526314, | |
| "grad_norm": 67.89371008742586, | |
| "kl": 38.031982421875, | |
| "learning_rate": 8.824684825733863e-08, | |
| "loss": 0.038, | |
| "reward": 1.0833333730697632, | |
| "reward_std": 0.28408990800380707, | |
| "rewards/correct_code_reward_func": 0.5833333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 167 | |
| }, | |
| { | |
| "completion_length": 47.79166793823242, | |
| "epoch": 2.2105263157894735, | |
| "grad_norm": 24.24181613825729, | |
| "kl": 12.8359375, | |
| "learning_rate": 8.555355467612527e-08, | |
| "loss": 0.0128, | |
| "reward": 0.7708333432674408, | |
| "reward_std": 0.2041093371808529, | |
| "rewards/correct_code_reward_func": 0.2708333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 168 | |
| }, | |
| { | |
| "completion_length": 44.68750190734863, | |
| "epoch": 2.223684210526316, | |
| "grad_norm": 237.339379162307, | |
| "kl": 158.1015625, | |
| "learning_rate": 8.289349125050913e-08, | |
| "loss": 0.1585, | |
| "reward": 1.0833333730697632, | |
| "reward_std": 0.36751919239759445, | |
| "rewards/correct_code_reward_func": 0.5833333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 169 | |
| }, | |
| { | |
| "completion_length": 66.95833587646484, | |
| "epoch": 2.236842105263158, | |
| "grad_norm": 1.2301856222728829, | |
| "kl": 0.20703125, | |
| "learning_rate": 8.026719550698627e-08, | |
| "loss": 0.0002, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.1451837606728077, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 170 | |
| }, | |
| { | |
| "completion_length": 40.08333396911621, | |
| "epoch": 2.25, | |
| "grad_norm": 2.792062080070077, | |
| "kl": 0.27099609375, | |
| "learning_rate": 7.767519814852233e-08, | |
| "loss": 0.0003, | |
| "reward": 1.1458333432674408, | |
| "reward_std": 0.34158404916524887, | |
| "rewards/correct_code_reward_func": 0.6458333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 171 | |
| }, | |
| { | |
| "completion_length": 64.04166984558105, | |
| "epoch": 2.263157894736842, | |
| "grad_norm": 1.3110888523745599, | |
| "kl": 0.81640625, | |
| "learning_rate": 7.511802294731159e-08, | |
| "loss": 0.0008, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.33875514566898346, | |
| "rewards/correct_code_reward_func": 0.2916666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 172 | |
| }, | |
| { | |
| "completion_length": 33.85416793823242, | |
| "epoch": 2.276315789473684, | |
| "grad_norm": 1.8190810652171645, | |
| "kl": 0.2119140625, | |
| "learning_rate": 7.259618663893724e-08, | |
| "loss": 0.0002, | |
| "reward": 1.1875000596046448, | |
| "reward_std": 0.2903806045651436, | |
| "rewards/correct_code_reward_func": 0.6875000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 173 | |
| }, | |
| { | |
| "completion_length": 49.06250190734863, | |
| "epoch": 2.2894736842105265, | |
| "grad_norm": 0.9842986413475313, | |
| "kl": 1.07421875, | |
| "learning_rate": 7.011019881795297e-08, | |
| "loss": 0.0011, | |
| "reward": 0.8333333730697632, | |
| "reward_std": 0.17251639068126678, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 174 | |
| }, | |
| { | |
| "completion_length": 65.04166793823242, | |
| "epoch": 2.3026315789473686, | |
| "grad_norm": 4.44211038726181, | |
| "kl": 1.56640625, | |
| "learning_rate": 6.766056183490798e-08, | |
| "loss": 0.0016, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.2931964062154293, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 175 | |
| }, | |
| { | |
| "completion_length": 54.791669845581055, | |
| "epoch": 2.3157894736842106, | |
| "grad_norm": 2.5493155442330098, | |
| "kl": 1.79150390625, | |
| "learning_rate": 6.524777069483525e-08, | |
| "loss": 0.0018, | |
| "reward": 0.8958333730697632, | |
| "reward_std": 0.30231600999832153, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 176 | |
| }, | |
| { | |
| "completion_length": 62.083335876464844, | |
| "epoch": 2.3289473684210527, | |
| "grad_norm": 319.0871377811938, | |
| "kl": 194.21875, | |
| "learning_rate": 6.28723129572247e-08, | |
| "loss": 0.1953, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.31142252683639526, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 177 | |
| }, | |
| { | |
| "completion_length": 46.06250190734863, | |
| "epoch": 2.3421052631578947, | |
| "grad_norm": 6.171508505395978, | |
| "kl": 4.849609375, | |
| "learning_rate": 6.053466863750084e-08, | |
| "loss": 0.0048, | |
| "reward": 0.625, | |
| "reward_std": 0.16623875498771667, | |
| "rewards/correct_code_reward_func": 0.125, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 178 | |
| }, | |
| { | |
| "completion_length": 45.10416793823242, | |
| "epoch": 2.3552631578947367, | |
| "grad_norm": 7.899530101559774, | |
| "kl": 4.39013671875, | |
| "learning_rate": 5.8235310110024224e-08, | |
| "loss": 0.0044, | |
| "reward": 0.979166716337204, | |
| "reward_std": 0.30231600999832153, | |
| "rewards/correct_code_reward_func": 0.479166679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 179 | |
| }, | |
| { | |
| "completion_length": 63.95833396911621, | |
| "epoch": 2.3684210526315788, | |
| "grad_norm": 19.13925515641779, | |
| "kl": 6.4049072265625, | |
| "learning_rate": 5.597470201263782e-08, | |
| "loss": 0.0064, | |
| "reward": 0.9583333432674408, | |
| "reward_std": 0.34018659591674805, | |
| "rewards/correct_code_reward_func": 0.4583333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 180 | |
| }, | |
| { | |
| "completion_length": 95.70833587646484, | |
| "epoch": 2.3815789473684212, | |
| "grad_norm": 2.2349939552948057, | |
| "kl": 0.345458984375, | |
| "learning_rate": 5.3753301152776347e-08, | |
| "loss": 0.0003, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.16623875498771667, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 181 | |
| }, | |
| { | |
| "completion_length": 59.375, | |
| "epoch": 2.3947368421052633, | |
| "grad_norm": 2.542273591175544, | |
| "kl": 0.755615234375, | |
| "learning_rate": 5.1571556415157646e-08, | |
| "loss": 0.0008, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.43129096925258636, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 182 | |
| }, | |
| { | |
| "completion_length": 66.375, | |
| "epoch": 2.4078947368421053, | |
| "grad_norm": 1.1341732814888108, | |
| "kl": 0.12890625, | |
| "learning_rate": 4.942990867107547e-08, | |
| "loss": 0.0001, | |
| "reward": 0.8541666865348816, | |
| "reward_std": 0.3177132308483124, | |
| "rewards/correct_code_reward_func": 0.3541666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 183 | |
| }, | |
| { | |
| "completion_length": 61.29166793823242, | |
| "epoch": 2.4210526315789473, | |
| "grad_norm": 484.5704062949344, | |
| "kl": 301.046875, | |
| "learning_rate": 4.732879068931131e-08, | |
| "loss": 0.3011, | |
| "reward": 1.0416666865348816, | |
| "reward_std": 0.2630349025130272, | |
| "rewards/correct_code_reward_func": 0.5416666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 184 | |
| }, | |
| { | |
| "completion_length": 43.58333396911621, | |
| "epoch": 2.4342105263157894, | |
| "grad_norm": 4.437414420601948, | |
| "kl": 3.1494140625, | |
| "learning_rate": 4.526862704868376e-08, | |
| "loss": 0.0032, | |
| "reward": 0.8750000596046448, | |
| "reward_std": 0.24966806918382645, | |
| "rewards/correct_code_reward_func": 0.3750000111758709, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 185 | |
| }, | |
| { | |
| "completion_length": 71.14583396911621, | |
| "epoch": 2.4473684210526314, | |
| "grad_norm": 90.82877113902249, | |
| "kl": 0.159912109375, | |
| "learning_rate": 4.324983405225235e-08, | |
| "loss": 0.0002, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.19500280916690826, | |
| "rewards/correct_code_reward_func": 0.458333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 186 | |
| }, | |
| { | |
| "completion_length": 78.04166793823242, | |
| "epoch": 2.4605263157894735, | |
| "grad_norm": 283.12841990209046, | |
| "kl": 57.2578125, | |
| "learning_rate": 4.1272819643194456e-08, | |
| "loss": 0.0573, | |
| "reward": 0.9583333432674408, | |
| "reward_std": 0.4958743155002594, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 187 | |
| }, | |
| { | |
| "completion_length": 47.00000190734863, | |
| "epoch": 2.473684210526316, | |
| "grad_norm": 1.1669700382103856, | |
| "kl": 0.20751953125, | |
| "learning_rate": 3.933798332237059e-08, | |
| "loss": 0.0002, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.1451837718486786, | |
| "rewards/correct_code_reward_func": 0.2500000074505806, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 188 | |
| }, | |
| { | |
| "completion_length": 110.87500381469727, | |
| "epoch": 2.486842105263158, | |
| "grad_norm": 1.125444972266526, | |
| "kl": 0.0595703125, | |
| "learning_rate": 3.74457160675965e-08, | |
| "loss": 0.0001, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.22233545035123825, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 189 | |
| }, | |
| { | |
| "completion_length": 122.5625, | |
| "epoch": 2.5, | |
| "grad_norm": 1.2043716850566297, | |
| "kl": 0.3837890625, | |
| "learning_rate": 3.5596400254637035e-08, | |
| "loss": 0.0004, | |
| "reward": 0.6041666865348816, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.1041666679084301, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 190 | |
| }, | |
| { | |
| "completion_length": 65.66666793823242, | |
| "epoch": 2.513157894736842, | |
| "grad_norm": 1.3296548993223685, | |
| "kl": 0.513916015625, | |
| "learning_rate": 3.379040957993834e-08, | |
| "loss": 0.0005, | |
| "reward": 0.8750000298023224, | |
| "reward_std": 0.2630349025130272, | |
| "rewards/correct_code_reward_func": 0.3750000223517418, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 191 | |
| }, | |
| { | |
| "completion_length": 48.041666984558105, | |
| "epoch": 2.526315789473684, | |
| "grad_norm": 284.52007908369717, | |
| "kl": 93.12890625, | |
| "learning_rate": 3.2028108985114235e-08, | |
| "loss": 0.0934, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.2122463434934616, | |
| "rewards/correct_code_reward_func": 0.5000000149011612, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 192 | |
| }, | |
| { | |
| "completion_length": 57.666666984558105, | |
| "epoch": 2.5394736842105265, | |
| "grad_norm": 5.140729023126808, | |
| "kl": 0.083740234375, | |
| "learning_rate": 3.030985458320118e-08, | |
| "loss": 0.0001, | |
| "reward": 1.1041666865348816, | |
| "reward_std": 0.24056155234575272, | |
| "rewards/correct_code_reward_func": 0.6041666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 193 | |
| }, | |
| { | |
| "completion_length": 71.08333587646484, | |
| "epoch": 2.5526315789473686, | |
| "grad_norm": 1.728517309004394, | |
| "kl": 2.6953125, | |
| "learning_rate": 2.863599358669755e-08, | |
| "loss": 0.0027, | |
| "reward": 0.7083333432674408, | |
| "reward_std": 0.2960253022611141, | |
| "rewards/correct_code_reward_func": 0.20833333395421505, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 194 | |
| }, | |
| { | |
| "completion_length": 45.83333396911621, | |
| "epoch": 2.5657894736842106, | |
| "grad_norm": 2.095306352134131, | |
| "kl": 2.639404296875, | |
| "learning_rate": 2.7006864237401423e-08, | |
| "loss": 0.0026, | |
| "reward": 1.0625, | |
| "reward_std": 0.3584126979112625, | |
| "rewards/correct_code_reward_func": 0.5625, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 195 | |
| }, | |
| { | |
| "completion_length": 55.18750190734863, | |
| "epoch": 2.5789473684210527, | |
| "grad_norm": 144.09595199637047, | |
| "kl": 78.4609375, | |
| "learning_rate": 2.542279573806122e-08, | |
| "loss": 0.0784, | |
| "reward": 0.5833333730697632, | |
| "reward_std": 0.15430335700511932, | |
| "rewards/correct_code_reward_func": 0.0833333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 196 | |
| }, | |
| { | |
| "completion_length": 55.37500190734863, | |
| "epoch": 2.5921052631578947, | |
| "grad_norm": 1.595820339822705, | |
| "kl": 0.9705810546875, | |
| "learning_rate": 2.3884108185852626e-08, | |
| "loss": 0.001, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.3233579248189926, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 197 | |
| }, | |
| { | |
| "completion_length": 91.14583587646484, | |
| "epoch": 2.6052631578947367, | |
| "grad_norm": 2.359631401946909, | |
| "kl": 0.7138671875, | |
| "learning_rate": 2.2391112507695876e-08, | |
| "loss": 0.0007, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.4130779355764389, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 198 | |
| }, | |
| { | |
| "completion_length": 31.812500953674316, | |
| "epoch": 2.6184210526315788, | |
| "grad_norm": 9.107326597955607, | |
| "kl": 4.916015625, | |
| "learning_rate": 2.094411039742569e-08, | |
| "loss": 0.0049, | |
| "reward": 1.2500000596046448, | |
| "reward_std": 0.19500280916690826, | |
| "rewards/correct_code_reward_func": 0.7500000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 199 | |
| }, | |
| { | |
| "completion_length": 62.60416793823242, | |
| "epoch": 2.6315789473684212, | |
| "grad_norm": 2.350577703191847, | |
| "kl": 0.904296875, | |
| "learning_rate": 1.954339425482712e-08, | |
| "loss": 0.0009, | |
| "reward": 1.1875000596046448, | |
| "reward_std": 0.25392838567495346, | |
| "rewards/correct_code_reward_func": 0.6875000298023224, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 200 | |
| }, | |
| { | |
| "completion_length": 48.79166793823242, | |
| "epoch": 2.6447368421052633, | |
| "grad_norm": 3.477458112159059, | |
| "kl": 0.35595703125, | |
| "learning_rate": 1.818924712654965e-08, | |
| "loss": 0.0004, | |
| "reward": 0.8125, | |
| "reward_std": 0.23144196718931198, | |
| "rewards/correct_code_reward_func": 0.3125, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 201 | |
| }, | |
| { | |
| "completion_length": 74.52083587646484, | |
| "epoch": 2.6578947368421053, | |
| "grad_norm": 0.7180881930260916, | |
| "kl": 0.181884765625, | |
| "learning_rate": 1.6881942648911074e-08, | |
| "loss": 0.0002, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.16340987384319305, | |
| "rewards/correct_code_reward_func": 0.5416666865348816, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 202 | |
| }, | |
| { | |
| "completion_length": 69.25000381469727, | |
| "epoch": 2.6710526315789473, | |
| "grad_norm": 1.195470821288021, | |
| "kl": 0.26025390625, | |
| "learning_rate": 1.5621744992603047e-08, | |
| "loss": 0.0003, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.4446708858013153, | |
| "rewards/correct_code_reward_func": 0.4166666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 203 | |
| }, | |
| { | |
| "completion_length": 92.64583587646484, | |
| "epoch": 2.6842105263157894, | |
| "grad_norm": 0.5914043963888233, | |
| "kl": 0.1630859375, | |
| "learning_rate": 1.4408908809309422e-08, | |
| "loss": 0.0002, | |
| "reward": 0.7291666865348816, | |
| "reward_std": 0.08625819534063339, | |
| "rewards/correct_code_reward_func": 0.2291666716337204, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 204 | |
| }, | |
| { | |
| "completion_length": 67.5, | |
| "epoch": 2.6973684210526314, | |
| "grad_norm": 0.5891628759615603, | |
| "kl": 0.06036376953125, | |
| "learning_rate": 1.3243679180248075e-08, | |
| "loss": 0.0001, | |
| "reward": 0.9791666865348816, | |
| "reward_std": 0.22516432031989098, | |
| "rewards/correct_code_reward_func": 0.4791666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 205 | |
| }, | |
| { | |
| "completion_length": 37.52083396911621, | |
| "epoch": 2.7105263157894735, | |
| "grad_norm": 3.112281199267652, | |
| "kl": 2.693359375, | |
| "learning_rate": 1.2126291566646462e-08, | |
| "loss": 0.0027, | |
| "reward": 0.875, | |
| "reward_std": 0.1451837718486786, | |
| "rewards/correct_code_reward_func": 0.375, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 206 | |
| }, | |
| { | |
| "completion_length": 69.02083396911621, | |
| "epoch": 2.723684210526316, | |
| "grad_norm": 1.4964559285333816, | |
| "kl": 0.83251953125, | |
| "learning_rate": 1.1056971762161583e-08, | |
| "loss": 0.0008, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.376638799905777, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 207 | |
| }, | |
| { | |
| "completion_length": 41.68750190734863, | |
| "epoch": 2.736842105263158, | |
| "grad_norm": 47.69500240874892, | |
| "kl": 33.90625, | |
| "learning_rate": 1.0035935847253012e-08, | |
| "loss": 0.0339, | |
| "reward": 1.0208333730697632, | |
| "reward_std": 0.28126102685928345, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 208 | |
| }, | |
| { | |
| "completion_length": 49.895835876464844, | |
| "epoch": 2.75, | |
| "grad_norm": 3.62772385527898, | |
| "kl": 0.6689453125, | |
| "learning_rate": 9.063390145519018e-09, | |
| "loss": 0.0007, | |
| "reward": 1.1851695775985718, | |
| "reward_std": 0.3313498795032501, | |
| "rewards/correct_code_reward_func": 0.6875000298023224, | |
| "rewards/len_reward_func": 0.4976695030927658, | |
| "step": 209 | |
| }, | |
| { | |
| "completion_length": 78.97917175292969, | |
| "epoch": 2.763157894736842, | |
| "grad_norm": 13.13764745425958, | |
| "kl": 5.2373046875, | |
| "learning_rate": 8.139531182004222e-09, | |
| "loss": 0.0052, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.2994871288537979, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 210 | |
| }, | |
| { | |
| "completion_length": 33.35416793823242, | |
| "epoch": 2.776315789473684, | |
| "grad_norm": 1.5959152998105355, | |
| "kl": 1.5703125, | |
| "learning_rate": 7.2645456434869965e-09, | |
| "loss": 0.0016, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.2994871214032173, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 211 | |
| }, | |
| { | |
| "completion_length": 64.68750381469727, | |
| "epoch": 2.7894736842105265, | |
| "grad_norm": 109.88019118611281, | |
| "kl": 0.3125, | |
| "learning_rate": 6.438610340755462e-09, | |
| "loss": 0.0003, | |
| "reward": 0.9166666865348816, | |
| "reward_std": 0.376638799905777, | |
| "rewards/correct_code_reward_func": 0.4166666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 212 | |
| }, | |
| { | |
| "completion_length": 73.08333587646484, | |
| "epoch": 2.8026315789473686, | |
| "grad_norm": 0.9800418792331905, | |
| "kl": 0.9185791015625, | |
| "learning_rate": 5.6618921728786015e-09, | |
| "loss": 0.0009, | |
| "reward": 0.8333333432674408, | |
| "reward_std": 0.22233543917536736, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 213 | |
| }, | |
| { | |
| "completion_length": 61.625, | |
| "epoch": 2.8157894736842106, | |
| "grad_norm": 1.5620983389420056, | |
| "kl": 0.1494140625, | |
| "learning_rate": 4.934548093480511e-09, | |
| "loss": 0.0001, | |
| "reward": 0.9583333432674408, | |
| "reward_std": 0.31142252683639526, | |
| "rewards/correct_code_reward_func": 0.4583333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 214 | |
| }, | |
| { | |
| "completion_length": 72.25000190734863, | |
| "epoch": 2.8289473684210527, | |
| "grad_norm": 0.976086181884898, | |
| "kl": 0.1263427734375, | |
| "learning_rate": 4.256725079024553e-09, | |
| "loss": 0.0001, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.3177132233977318, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 215 | |
| }, | |
| { | |
| "completion_length": 44.08333396911621, | |
| "epoch": 2.8421052631578947, | |
| "grad_norm": 1.4377976203336797, | |
| "kl": 0.2220458984375, | |
| "learning_rate": 3.6285600991131095e-09, | |
| "loss": 0.0002, | |
| "reward": 0.70163694024086, | |
| "reward_std": 0.09609203785657883, | |
| "rewards/correct_code_reward_func": 0.2083333395421505, | |
| "rewards/len_reward_func": 0.493303582072258, | |
| "step": 216 | |
| }, | |
| { | |
| "completion_length": 41.10416793823242, | |
| "epoch": 2.8552631578947367, | |
| "grad_norm": 1.728700023377306, | |
| "kl": 0.392578125, | |
| "learning_rate": 3.0501800888099726e-09, | |
| "loss": 0.0004, | |
| "reward": 0.8958333432674408, | |
| "reward_std": 0.21322892233729362, | |
| "rewards/correct_code_reward_func": 0.3958333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 217 | |
| }, | |
| { | |
| "completion_length": 57.58333396911621, | |
| "epoch": 2.8684210526315788, | |
| "grad_norm": 2.083207612873304, | |
| "kl": 0.4912109375, | |
| "learning_rate": 2.5217019229900606e-09, | |
| "loss": 0.0005, | |
| "reward": 1.0833333730697632, | |
| "reward_std": 0.22233545035123825, | |
| "rewards/correct_code_reward_func": 0.5833333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 218 | |
| }, | |
| { | |
| "completion_length": 39.54166793823242, | |
| "epoch": 2.8815789473684212, | |
| "grad_norm": 0.9145475753101857, | |
| "kl": 0.1865234375, | |
| "learning_rate": 2.043232392722388e-09, | |
| "loss": 0.0002, | |
| "reward": 1.0208333432674408, | |
| "reward_std": 0.2041093371808529, | |
| "rewards/correct_code_reward_func": 0.5208333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 219 | |
| }, | |
| { | |
| "completion_length": 49.37500190734863, | |
| "epoch": 2.8947368421052633, | |
| "grad_norm": 1.081844737309092, | |
| "kl": 0.203125, | |
| "learning_rate": 1.614868183690249e-09, | |
| "loss": 0.0002, | |
| "reward": 0.7500000298023224, | |
| "reward_std": 0.1451837606728077, | |
| "rewards/correct_code_reward_func": 0.2500000149011612, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 220 | |
| }, | |
| { | |
| "completion_length": 61.125003814697266, | |
| "epoch": 2.9078947368421053, | |
| "grad_norm": 2.1759674723609286, | |
| "kl": 1.7373046875, | |
| "learning_rate": 1.2366958566538688e-09, | |
| "loss": 0.0017, | |
| "reward": 0.6458333432674408, | |
| "reward_std": 0.13607725501060486, | |
| "rewards/correct_code_reward_func": 0.1458333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 221 | |
| }, | |
| { | |
| "completion_length": 73.47916984558105, | |
| "epoch": 2.9210526315789473, | |
| "grad_norm": 0.8216969877474546, | |
| "kl": 0.073974609375, | |
| "learning_rate": 9.087918299586772e-10, | |
| "loss": 0.0001, | |
| "reward": 1.1458333730697632, | |
| "reward_std": 0.23709972202777863, | |
| "rewards/correct_code_reward_func": 0.6458333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 222 | |
| }, | |
| { | |
| "completion_length": 41.60416793823242, | |
| "epoch": 2.9342105263157894, | |
| "grad_norm": 1.2661888490492268, | |
| "kl": 0.1630859375, | |
| "learning_rate": 6.312223640933811e-10, | |
| "loss": 0.0002, | |
| "reward": 1.166666716337204, | |
| "reward_std": 0.24966806918382645, | |
| "rewards/correct_code_reward_func": 0.6666666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 223 | |
| }, | |
| { | |
| "completion_length": 58.895835876464844, | |
| "epoch": 2.9473684210526314, | |
| "grad_norm": 10.649202570243657, | |
| "kl": 1.75439453125, | |
| "learning_rate": 4.0404354830042563e-10, | |
| "loss": 0.0018, | |
| "reward": 0.6875000298023224, | |
| "reward_std": 0.30231600254774094, | |
| "rewards/correct_code_reward_func": 0.1875000111758709, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 224 | |
| }, | |
| { | |
| "completion_length": 73.64583587646484, | |
| "epoch": 2.9605263157894735, | |
| "grad_norm": 3.7010298062572695, | |
| "kl": 0.350341796875, | |
| "learning_rate": 2.2730128924186574e-10, | |
| "loss": 0.0004, | |
| "reward": 0.7916666865348816, | |
| "reward_std": 0.19500282034277916, | |
| "rewards/correct_code_reward_func": 0.2916666865348816, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 225 | |
| }, | |
| { | |
| "completion_length": 47.64583396911621, | |
| "epoch": 2.973684210526316, | |
| "grad_norm": 2.797986645774147, | |
| "kl": 0.597412109375, | |
| "learning_rate": 1.010313017229536e-10, | |
| "loss": 0.0006, | |
| "reward": 1.2083333730697632, | |
| "reward_std": 0.31142251938581467, | |
| "rewards/correct_code_reward_func": 0.7083333432674408, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 226 | |
| }, | |
| { | |
| "completion_length": 49.18750190734863, | |
| "epoch": 2.986842105263158, | |
| "grad_norm": 29.33011919112973, | |
| "kl": 2.07421875, | |
| "learning_rate": 2.5259101475161304e-11, | |
| "loss": 0.0021, | |
| "reward": 0.9583333730697632, | |
| "reward_std": 0.3268197476863861, | |
| "rewards/correct_code_reward_func": 0.458333358168602, | |
| "rewards/len_reward_func": 0.5, | |
| "step": 227 | |
| }, | |
| { | |
| "completion_length": 75.37500381469727, | |
| "epoch": 3.0, | |
| "grad_norm": 1.4657552454229807, | |
| "kl": 0.4228515625, | |
| "learning_rate": 0.0, | |
| "loss": 0.0004, | |
| "reward": 0.8125, | |
| "reward_std": 0.24185511097311974, | |
| "rewards/correct_code_reward_func": 0.3333333432674408, | |
| "rewards/len_reward_func": 0.4791666716337204, | |
| "step": 228 | |
| }, | |
| { | |
| "epoch": 3.0, | |
| "step": 228, | |
| "total_flos": 0.0, | |
| "train_loss": 0.04159710054096379, | |
| "train_runtime": 3805.4602, | |
| "train_samples_per_second": 0.359, | |
| "train_steps_per_second": 0.06 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 228, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 76, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |