Wenboz's picture
Model save
f9a4454 verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9991111111111111,
"eval_steps": 100,
"global_step": 562,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.0017777777777777779,
"grad_norm": 0.22303288377481736,
"learning_rate": 8.771929824561403e-09,
"logits/chosen": 14.539060592651367,
"logits/rejected": 15.870795249938965,
"logps/chosen": -470.04345703125,
"logps/rejected": -509.49163818359375,
"loss": 0.6931,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.017777777777777778,
"grad_norm": 0.19778346088106144,
"learning_rate": 8.771929824561403e-08,
"logits/chosen": 12.965851783752441,
"logits/rejected": 13.156441688537598,
"logps/chosen": -400.76171875,
"logps/rejected": -399.6529235839844,
"loss": 0.6929,
"rewards/accuracies": 0.3263888955116272,
"rewards/chosen": -0.0005124944727867842,
"rewards/margins": 0.0010498098563402891,
"rewards/rejected": -0.001562304561957717,
"step": 10
},
{
"epoch": 0.035555555555555556,
"grad_norm": 0.2178780514004974,
"learning_rate": 1.7543859649122805e-07,
"logits/chosen": 13.822667121887207,
"logits/rejected": 13.859578132629395,
"logps/chosen": -420.60968017578125,
"logps/rejected": -404.8630676269531,
"loss": 0.6932,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": 0.0033001075498759747,
"rewards/margins": 0.0035031698644161224,
"rewards/rejected": -0.00020306208170950413,
"step": 20
},
{
"epoch": 0.05333333333333334,
"grad_norm": 0.1907566784399537,
"learning_rate": 2.631578947368421e-07,
"logits/chosen": 13.258750915527344,
"logits/rejected": 13.402864456176758,
"logps/chosen": -402.22454833984375,
"logps/rejected": -412.36016845703125,
"loss": 0.6929,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": 0.00020933200721628964,
"rewards/margins": 0.0007921932265162468,
"rewards/rejected": -0.0005828611319884658,
"step": 30
},
{
"epoch": 0.07111111111111111,
"grad_norm": 0.21404300935098977,
"learning_rate": 3.508771929824561e-07,
"logits/chosen": 13.549951553344727,
"logits/rejected": 13.090646743774414,
"logps/chosen": -380.17755126953125,
"logps/rejected": -374.15228271484375,
"loss": 0.694,
"rewards/accuracies": 0.46875,
"rewards/chosen": -0.0018596267327666283,
"rewards/margins": -0.0006307201110757887,
"rewards/rejected": -0.0012289065634831786,
"step": 40
},
{
"epoch": 0.08888888888888889,
"grad_norm": 0.22357155239834353,
"learning_rate": 4.3859649122807013e-07,
"logits/chosen": 13.242405891418457,
"logits/rejected": 13.241366386413574,
"logps/chosen": -396.5928039550781,
"logps/rejected": -430.5787048339844,
"loss": 0.6928,
"rewards/accuracies": 0.4375,
"rewards/chosen": -0.001400044304318726,
"rewards/margins": -0.0008570448262616992,
"rewards/rejected": -0.0005429992452263832,
"step": 50
},
{
"epoch": 0.10666666666666667,
"grad_norm": 0.2723106872396695,
"learning_rate": 4.999564631597801e-07,
"logits/chosen": 13.082662582397461,
"logits/rejected": 13.338605880737305,
"logps/chosen": -388.4220886230469,
"logps/rejected": -414.44000244140625,
"loss": 0.6933,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.0005302426870912313,
"rewards/margins": 0.00033778088982217014,
"rewards/rejected": -0.000868023547809571,
"step": 60
},
{
"epoch": 0.12444444444444444,
"grad_norm": 0.23524853520895073,
"learning_rate": 4.991828966534002e-07,
"logits/chosen": 13.9525728225708,
"logits/rejected": 13.664074897766113,
"logps/chosen": -450.06341552734375,
"logps/rejected": -444.23773193359375,
"loss": 0.6934,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": 0.0007383242482319474,
"rewards/margins": 0.0011414262698963284,
"rewards/rejected": -0.0004031023127026856,
"step": 70
},
{
"epoch": 0.14222222222222222,
"grad_norm": 0.19631754730182652,
"learning_rate": 4.974452899279291e-07,
"logits/chosen": 13.425386428833008,
"logits/rejected": 12.61982536315918,
"logps/chosen": -424.4969177246094,
"logps/rejected": -381.8243103027344,
"loss": 0.6939,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -0.0009060869924724102,
"rewards/margins": -0.0010500686476007104,
"rewards/rejected": 0.00014398139319382608,
"step": 80
},
{
"epoch": 0.16,
"grad_norm": 0.23166744019072769,
"learning_rate": 4.947503654462276e-07,
"logits/chosen": 13.848444938659668,
"logits/rejected": 13.661419868469238,
"logps/chosen": -431.8487243652344,
"logps/rejected": -423.234130859375,
"loss": 0.6936,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": 0.0018427862087264657,
"rewards/margins": 0.0004623296554200351,
"rewards/rejected": 0.001380456262268126,
"step": 90
},
{
"epoch": 0.17777777777777778,
"grad_norm": 0.19863379374123877,
"learning_rate": 4.911085493475802e-07,
"logits/chosen": 12.961809158325195,
"logits/rejected": 12.821878433227539,
"logps/chosen": -407.67791748046875,
"logps/rejected": -407.8581237792969,
"loss": 0.6933,
"rewards/accuracies": 0.5062500238418579,
"rewards/chosen": -0.001652047038078308,
"rewards/margins": 0.0012966005597263575,
"rewards/rejected": -0.002948648063465953,
"step": 100
},
{
"epoch": 0.17777777777777778,
"eval_logits/chosen": 14.322334289550781,
"eval_logits/rejected": 12.53359603881836,
"eval_logps/chosen": -408.54986572265625,
"eval_logps/rejected": -383.8265075683594,
"eval_loss": 0.6931557655334473,
"eval_rewards/accuracies": 0.5277777910232544,
"eval_rewards/chosen": 0.0002005997084779665,
"eval_rewards/margins": 0.0011199678992852569,
"eval_rewards/rejected": -0.0009193681762553751,
"eval_runtime": 90.0845,
"eval_samples_per_second": 11.101,
"eval_steps_per_second": 0.699,
"step": 100
},
{
"epoch": 0.19555555555555557,
"grad_norm": 0.2310171714372319,
"learning_rate": 4.865339311109869e-07,
"logits/chosen": 13.624005317687988,
"logits/rejected": 13.505825996398926,
"logps/chosen": -408.5940856933594,
"logps/rejected": -419.42718505859375,
"loss": 0.6924,
"rewards/accuracies": 0.53125,
"rewards/chosen": 0.003594371024519205,
"rewards/margins": 0.003836499061435461,
"rewards/rejected": -0.0002421284152660519,
"step": 110
},
{
"epoch": 0.21333333333333335,
"grad_norm": 0.2880413214839054,
"learning_rate": 4.810442090457072e-07,
"logits/chosen": 13.073244094848633,
"logits/rejected": 13.229601860046387,
"logps/chosen": -382.7886962890625,
"logps/rejected": -392.2433166503906,
"loss": 0.694,
"rewards/accuracies": 0.48124998807907104,
"rewards/chosen": -0.0008653242257423699,
"rewards/margins": -0.0005229294183664024,
"rewards/rejected": -0.00034239477827213705,
"step": 120
},
{
"epoch": 0.2311111111111111,
"grad_norm": 0.2290858953636122,
"learning_rate": 4.746606218199385e-07,
"logits/chosen": 13.55931568145752,
"logits/rejected": 13.459338188171387,
"logps/chosen": -388.727783203125,
"logps/rejected": -419.49896240234375,
"loss": 0.6936,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.004407087340950966,
"rewards/margins": -0.0013083925005048513,
"rewards/rejected": -0.003098695306107402,
"step": 130
},
{
"epoch": 0.24888888888888888,
"grad_norm": 0.2616397396789134,
"learning_rate": 4.674078662925359e-07,
"logits/chosen": 13.565098762512207,
"logits/rejected": 13.248819351196289,
"logps/chosen": -375.28973388671875,
"logps/rejected": -383.5194091796875,
"loss": 0.6926,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.006319853011518717,
"rewards/margins": -0.0014259483432397246,
"rewards/rejected": -0.004893905017524958,
"step": 140
},
{
"epoch": 0.26666666666666666,
"grad_norm": 0.2525954228621521,
"learning_rate": 4.593140019656625e-07,
"logits/chosen": 13.338470458984375,
"logits/rejected": 13.415323257446289,
"logps/chosen": -413.3804626464844,
"logps/rejected": -431.94268798828125,
"loss": 0.6933,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": 0.0012489401269704103,
"rewards/margins": 0.0019859138410538435,
"rewards/rejected": -0.0007369734812527895,
"step": 150
},
{
"epoch": 0.28444444444444444,
"grad_norm": 0.2809357742154725,
"learning_rate": 4.504103424280266e-07,
"logits/chosen": 13.667657852172852,
"logits/rejected": 13.324671745300293,
"logps/chosen": -423.1854553222656,
"logps/rejected": -417.2071838378906,
"loss": 0.6936,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.0026347413659095764,
"rewards/margins": 0.00047498446656391025,
"rewards/rejected": -0.0031097261235117912,
"step": 160
},
{
"epoch": 0.3022222222222222,
"grad_norm": 0.22151723446382063,
"learning_rate": 4.407313342086905e-07,
"logits/chosen": 13.532007217407227,
"logits/rejected": 13.800847053527832,
"logps/chosen": -410.939208984375,
"logps/rejected": -420.8081970214844,
"loss": 0.6932,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.005603627301752567,
"rewards/margins": -0.0010588798904791474,
"rewards/rejected": -0.004544746596366167,
"step": 170
},
{
"epoch": 0.32,
"grad_norm": 0.2083986934814025,
"learning_rate": 4.3031442351014115e-07,
"logits/chosen": 14.2218599319458,
"logits/rejected": 14.246867179870605,
"logps/chosen": -382.0667419433594,
"logps/rejected": -399.89801025390625,
"loss": 0.6922,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.0030205477960407734,
"rewards/margins": 0.0017284912755712867,
"rewards/rejected": -0.004749038256704807,
"step": 180
},
{
"epoch": 0.3377777777777778,
"grad_norm": 0.24085135433862764,
"learning_rate": 4.19199911336207e-07,
"logits/chosen": 13.331924438476562,
"logits/rejected": 13.189720153808594,
"logps/chosen": -410.3375549316406,
"logps/rejected": -412.0045471191406,
"loss": 0.6928,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -0.0036415320355445147,
"rewards/margins": -0.00039607248618267477,
"rewards/rejected": -0.0032454594038426876,
"step": 190
},
{
"epoch": 0.35555555555555557,
"grad_norm": 0.2306639442798851,
"learning_rate": 4.0743079757530443e-07,
"logits/chosen": 13.834714889526367,
"logits/rejected": 13.460780143737793,
"logps/chosen": -397.07745361328125,
"logps/rejected": -390.42596435546875,
"loss": 0.6942,
"rewards/accuracies": 0.41874998807907104,
"rewards/chosen": -0.005139860790222883,
"rewards/margins": -0.0025263438001275063,
"rewards/rejected": -0.002613516291603446,
"step": 200
},
{
"epoch": 0.35555555555555557,
"eval_logits/chosen": 14.309229850769043,
"eval_logits/rejected": 12.522067070007324,
"eval_logps/chosen": -408.9154357910156,
"eval_logps/rejected": -384.37396240234375,
"eval_loss": 0.6927717328071594,
"eval_rewards/accuracies": 0.6111111044883728,
"eval_rewards/chosen": -0.003454842371866107,
"eval_rewards/margins": 0.0029392761643975973,
"eval_rewards/rejected": -0.006394118070602417,
"eval_runtime": 90.2631,
"eval_samples_per_second": 11.079,
"eval_steps_per_second": 0.698,
"step": 200
},
{
"epoch": 0.37333333333333335,
"grad_norm": 0.22130977335049826,
"learning_rate": 3.9505261464222127e-07,
"logits/chosen": 13.572957992553711,
"logits/rejected": 13.68427848815918,
"logps/chosen": -393.5160217285156,
"logps/rejected": -416.9537658691406,
"loss": 0.6938,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -0.006536452565342188,
"rewards/margins": -0.002487990539520979,
"rewards/rejected": -0.004048462025821209,
"step": 210
},
{
"epoch": 0.39111111111111113,
"grad_norm": 0.21869734573168603,
"learning_rate": 3.821132513220511e-07,
"logits/chosen": 13.86383056640625,
"logits/rejected": 13.388453483581543,
"logps/chosen": -390.048583984375,
"logps/rejected": -393.3787536621094,
"loss": 0.6929,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.006268749479204416,
"rewards/margins": 0.0020352094434201717,
"rewards/rejected": -0.008303959853947163,
"step": 220
},
{
"epoch": 0.4088888888888889,
"grad_norm": 0.2433643205853518,
"learning_rate": 3.6866276749778575e-07,
"logits/chosen": 13.376676559448242,
"logits/rejected": 13.516916275024414,
"logps/chosen": -397.7325744628906,
"logps/rejected": -416.31982421875,
"loss": 0.6921,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.003814761061221361,
"rewards/margins": 0.00433861929923296,
"rewards/rejected": -0.008153380826115608,
"step": 230
},
{
"epoch": 0.4266666666666667,
"grad_norm": 0.27647252316388643,
"learning_rate": 3.5475320047835385e-07,
"logits/chosen": 13.953859329223633,
"logits/rejected": 12.888150215148926,
"logps/chosen": -428.2935485839844,
"logps/rejected": -399.797607421875,
"loss": 0.6938,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.007735622581094503,
"rewards/margins": 0.001966065028682351,
"rewards/rejected": -0.009701686911284924,
"step": 240
},
{
"epoch": 0.4444444444444444,
"grad_norm": 0.19597999951437842,
"learning_rate": 3.4043836367638084e-07,
"logits/chosen": 13.694613456726074,
"logits/rejected": 13.338064193725586,
"logps/chosen": -438.20367431640625,
"logps/rejected": -433.60662841796875,
"loss": 0.6924,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -0.0037851575762033463,
"rewards/margins": 0.00321608642116189,
"rewards/rejected": -0.007001244463026524,
"step": 250
},
{
"epoch": 0.4622222222222222,
"grad_norm": 0.223201986382731,
"learning_rate": 3.257736384145506e-07,
"logits/chosen": 13.863354682922363,
"logits/rejected": 13.879611015319824,
"logps/chosen": -360.39404296875,
"logps/rejected": -373.2799377441406,
"loss": 0.693,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.006231536623090506,
"rewards/margins": 0.0012490267399698496,
"rewards/rejected": -0.0074805631302297115,
"step": 260
},
{
"epoch": 0.48,
"grad_norm": 0.2517853111478367,
"learning_rate": 3.1081575966602624e-07,
"logits/chosen": 13.030715942382812,
"logits/rejected": 12.911158561706543,
"logps/chosen": -420.63885498046875,
"logps/rejected": -403.2894592285156,
"loss": 0.6927,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -0.006683342158794403,
"rewards/margins": -0.00023603746376466006,
"rewards/rejected": -0.006447304971516132,
"step": 270
},
{
"epoch": 0.49777777777777776,
"grad_norm": 0.21470866725023854,
"learning_rate": 2.9562259655786065e-07,
"logits/chosen": 13.380098342895508,
"logits/rejected": 12.719483375549316,
"logps/chosen": -375.6423645019531,
"logps/rejected": -375.3600158691406,
"loss": 0.693,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.008134881034493446,
"rewards/margins": -0.0008266712538897991,
"rewards/rejected": -0.007308208849281073,
"step": 280
},
{
"epoch": 0.5155555555555555,
"grad_norm": 0.2099580010497102,
"learning_rate": 2.8025292848658625e-07,
"logits/chosen": 12.355232238769531,
"logits/rejected": 12.424150466918945,
"logps/chosen": -389.7431640625,
"logps/rejected": -380.82354736328125,
"loss": 0.6932,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.009173677302896976,
"rewards/margins": 0.0001662088616285473,
"rewards/rejected": -0.009339885786175728,
"step": 290
},
{
"epoch": 0.5333333333333333,
"grad_norm": 0.2300411233471207,
"learning_rate": 2.647662177121486e-07,
"logits/chosen": 13.670684814453125,
"logits/rejected": 13.45910358428955,
"logps/chosen": -396.01470947265625,
"logps/rejected": -403.1299743652344,
"loss": 0.6933,
"rewards/accuracies": 0.4437499940395355,
"rewards/chosen": -0.009991997852921486,
"rewards/margins": -0.0007631282205693424,
"rewards/rejected": -0.009228868409991264,
"step": 300
},
{
"epoch": 0.5333333333333333,
"eval_logits/chosen": 14.301990509033203,
"eval_logits/rejected": 12.51347541809082,
"eval_logps/chosen": -409.30389404296875,
"eval_logps/rejected": -384.94805908203125,
"eval_loss": 0.6918109655380249,
"eval_rewards/accuracies": 0.579365074634552,
"eval_rewards/chosen": -0.007339805830270052,
"eval_rewards/margins": 0.0047955019399523735,
"eval_rewards/rejected": -0.012135308235883713,
"eval_runtime": 90.22,
"eval_samples_per_second": 11.084,
"eval_steps_per_second": 0.698,
"step": 300
},
{
"epoch": 0.5511111111111111,
"grad_norm": 0.27059238365031907,
"learning_rate": 2.492223793099743e-07,
"logits/chosen": 13.299054145812988,
"logits/rejected": 13.100789070129395,
"logps/chosen": -394.95269775390625,
"logps/rejected": -414.8412170410156,
"loss": 0.693,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -0.010692128911614418,
"rewards/margins": -0.00010659964755177498,
"rewards/rejected": -0.010585528798401356,
"step": 310
},
{
"epoch": 0.5688888888888889,
"grad_norm": 0.2469034045815531,
"learning_rate": 2.3368154937118352e-07,
"logits/chosen": 12.717276573181152,
"logits/rejected": 12.795941352844238,
"logps/chosen": -402.4614562988281,
"logps/rejected": -420.6311950683594,
"loss": 0.6932,
"rewards/accuracies": 0.53125,
"rewards/chosen": -0.008936749771237373,
"rewards/margins": 0.00016509280249010772,
"rewards/rejected": -0.00910184346139431,
"step": 320
},
{
"epoch": 0.5866666666666667,
"grad_norm": 0.2434679300536542,
"learning_rate": 2.1820385234773604e-07,
"logits/chosen": 12.802389144897461,
"logits/rejected": 12.707880020141602,
"logps/chosen": -358.6187438964844,
"logps/rejected": -362.9839782714844,
"loss": 0.693,
"rewards/accuracies": 0.6187499761581421,
"rewards/chosen": -0.00801909901201725,
"rewards/margins": 0.0028764610178768635,
"rewards/rejected": -0.0108955604955554,
"step": 330
},
{
"epoch": 0.6044444444444445,
"grad_norm": 0.24219888229694975,
"learning_rate": 2.0284916844260697e-07,
"logits/chosen": 13.935209274291992,
"logits/rejected": 14.0303955078125,
"logps/chosen": -393.71795654296875,
"logps/rejected": -398.4595642089844,
"loss": 0.6934,
"rewards/accuracies": 0.512499988079071,
"rewards/chosen": -0.010410056449472904,
"rewards/margins": 0.0008261911571025848,
"rewards/rejected": -0.011236247606575489,
"step": 340
},
{
"epoch": 0.6222222222222222,
"grad_norm": 0.23895219240110752,
"learning_rate": 1.876769019449141e-07,
"logits/chosen": 12.941207885742188,
"logits/rejected": 12.936040878295898,
"logps/chosen": -367.445068359375,
"logps/rejected": -395.6293029785156,
"loss": 0.6929,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.011776421219110489,
"rewards/margins": -0.00041538532241247594,
"rewards/rejected": -0.011361035518348217,
"step": 350
},
{
"epoch": 0.64,
"grad_norm": 0.24890094811299934,
"learning_rate": 1.7274575140626315e-07,
"logits/chosen": 12.9166841506958,
"logits/rejected": 12.816073417663574,
"logps/chosen": -413.01702880859375,
"logps/rejected": -406.90203857421875,
"loss": 0.6927,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.008710355497896671,
"rewards/margins": 0.0026525070425122976,
"rewards/rejected": -0.011362862773239613,
"step": 360
},
{
"epoch": 0.6577777777777778,
"grad_norm": 0.24961814178449623,
"learning_rate": 1.5811348254745572e-07,
"logits/chosen": 13.637972831726074,
"logits/rejected": 13.490551948547363,
"logps/chosen": -405.98931884765625,
"logps/rejected": -415.7749938964844,
"loss": 0.6927,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.011339882388710976,
"rewards/margins": -0.000733031309209764,
"rewards/rejected": -0.010606849566102028,
"step": 370
},
{
"epoch": 0.6755555555555556,
"grad_norm": 0.2261439436299567,
"learning_rate": 1.4383670477413674e-07,
"logits/chosen": 13.193809509277344,
"logits/rejected": 12.558343887329102,
"logps/chosen": -380.3818054199219,
"logps/rejected": -365.39263916015625,
"loss": 0.6924,
"rewards/accuracies": 0.5874999761581421,
"rewards/chosen": -0.01006121002137661,
"rewards/margins": 0.00015344182611443102,
"rewards/rejected": -0.010214651003479958,
"step": 380
},
{
"epoch": 0.6933333333333334,
"grad_norm": 0.21712577203339933,
"learning_rate": 1.2997065216600178e-07,
"logits/chosen": 13.218670845031738,
"logits/rejected": 13.502662658691406,
"logps/chosen": -414.07269287109375,
"logps/rejected": -418.16412353515625,
"loss": 0.6938,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.014597907662391663,
"rewards/margins": -0.0025998970959335566,
"rewards/rejected": -0.0119980089366436,
"step": 390
},
{
"epoch": 0.7111111111111111,
"grad_norm": 0.2243252823451527,
"learning_rate": 1.1656896978687259e-07,
"logits/chosen": 13.136878967285156,
"logits/rejected": 12.531244277954102,
"logps/chosen": -407.50543212890625,
"logps/rejected": -400.00299072265625,
"loss": 0.6928,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.011982095427811146,
"rewards/margins": 0.0002563459565863013,
"rewards/rejected": -0.012238441966474056,
"step": 400
},
{
"epoch": 0.7111111111111111,
"eval_logits/chosen": 14.284664154052734,
"eval_logits/rejected": 12.493403434753418,
"eval_logps/chosen": -409.6051025390625,
"eval_logps/rejected": -385.15447998046875,
"eval_loss": 0.6914495229721069,
"eval_rewards/accuracies": 0.5714285969734192,
"eval_rewards/chosen": -0.010351993143558502,
"eval_rewards/margins": 0.0038474262692034245,
"eval_rewards/rejected": -0.014199419878423214,
"eval_runtime": 90.3281,
"eval_samples_per_second": 11.071,
"eval_steps_per_second": 0.697,
"step": 400
},
{
"epoch": 0.7288888888888889,
"grad_norm": 0.2553822385496833,
"learning_rate": 1.0368350614236685e-07,
"logits/chosen": 12.949267387390137,
"logits/rejected": 13.377090454101562,
"logps/chosen": -389.6181945800781,
"logps/rejected": -409.90875244140625,
"loss": 0.6935,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.013937557116150856,
"rewards/margins": -0.00270536239258945,
"rewards/rejected": -0.011232194490730762,
"step": 410
},
{
"epoch": 0.7466666666666667,
"grad_norm": 0.23007443203253394,
"learning_rate": 9.136411258810229e-08,
"logits/chosen": 13.887911796569824,
"logits/rejected": 14.226011276245117,
"logps/chosen": -384.749755859375,
"logps/rejected": -401.3822326660156,
"loss": 0.6939,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.01530504785478115,
"rewards/margins": -0.0024403613060712814,
"rewards/rejected": -0.012864688411355019,
"step": 420
},
{
"epoch": 0.7644444444444445,
"grad_norm": 0.2287369821575642,
"learning_rate": 7.965845046448657e-08,
"logits/chosen": 13.097994804382324,
"logits/rejected": 13.113497734069824,
"logps/chosen": -400.8845520019531,
"logps/rejected": -405.2341613769531,
"loss": 0.6935,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.010901332832872868,
"rewards/margins": -0.0005171374650672078,
"rewards/rejected": -0.010384196415543556,
"step": 430
},
{
"epoch": 0.7822222222222223,
"grad_norm": 0.23354678846120172,
"learning_rate": 6.861180670424982e-08,
"logits/chosen": 13.59221363067627,
"logits/rejected": 12.869047164916992,
"logps/chosen": -422.7332458496094,
"logps/rejected": -417.83056640625,
"loss": 0.693,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.009619112126529217,
"rewards/margins": 0.0028655820060521364,
"rewards/rejected": -0.012484694831073284,
"step": 440
},
{
"epoch": 0.8,
"grad_norm": 0.2790910188940452,
"learning_rate": 5.826691862609986e-08,
"logits/chosen": 13.35814380645752,
"logits/rejected": 12.920620918273926,
"logps/chosen": -380.17974853515625,
"logps/rejected": -383.1281433105469,
"loss": 0.6927,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.01176600344479084,
"rewards/margins": -0.000654814182780683,
"rewards/rejected": -0.011111188679933548,
"step": 450
},
{
"epoch": 0.8177777777777778,
"grad_norm": 0.24829506659789913,
"learning_rate": 4.86638085923389e-08,
"logits/chosen": 13.321922302246094,
"logits/rejected": 13.535449028015137,
"logps/chosen": -393.94842529296875,
"logps/rejected": -418.94134521484375,
"loss": 0.6925,
"rewards/accuracies": 0.48750001192092896,
"rewards/chosen": -0.011064440943300724,
"rewards/margins": -0.00010039366316050291,
"rewards/rejected": -0.010964048095047474,
"step": 460
},
{
"epoch": 0.8355555555555556,
"grad_norm": 0.24616507601632995,
"learning_rate": 3.983962917011829e-08,
"logits/chosen": 13.560412406921387,
"logits/rejected": 13.323214530944824,
"logps/chosen": -407.70428466796875,
"logps/rejected": -410.4319763183594,
"loss": 0.6933,
"rewards/accuracies": 0.46875,
"rewards/chosen": -0.013220484368503094,
"rewards/margins": -0.0020924489945173264,
"rewards/rejected": -0.011128035373985767,
"step": 470
},
{
"epoch": 0.8533333333333334,
"grad_norm": 0.23918348442377044,
"learning_rate": 3.182851939537409e-08,
"logits/chosen": 13.533491134643555,
"logits/rejected": 13.728436470031738,
"logps/chosen": -413.63714599609375,
"logps/rejected": -430.1578674316406,
"loss": 0.6932,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -0.01208510808646679,
"rewards/margins": -0.0005490529583767056,
"rewards/rejected": -0.01153605617582798,
"step": 480
},
{
"epoch": 0.8711111111111111,
"grad_norm": 0.23930714935454145,
"learning_rate": 2.466147269552893e-08,
"logits/chosen": 13.9771089553833,
"logits/rejected": 13.742532730102539,
"logps/chosen": -383.36968994140625,
"logps/rejected": -387.08831787109375,
"loss": 0.6928,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.012026415206491947,
"rewards/margins": 0.0002618884027469903,
"rewards/rejected": -0.012288304045796394,
"step": 490
},
{
"epoch": 0.8888888888888888,
"grad_norm": 0.21012017177909903,
"learning_rate": 1.8366216981942628e-08,
"logits/chosen": 13.372047424316406,
"logits/rejected": 13.567631721496582,
"logps/chosen": -425.3689880371094,
"logps/rejected": -428.603515625,
"loss": 0.6935,
"rewards/accuracies": 0.5375000238418579,
"rewards/chosen": -0.015114056877791882,
"rewards/margins": 4.679532867157832e-05,
"rewards/rejected": -0.015160853043198586,
"step": 500
},
{
"epoch": 0.8888888888888888,
"eval_logits/chosen": 14.289068222045898,
"eval_logits/rejected": 12.499544143676758,
"eval_logps/chosen": -409.5751037597656,
"eval_logps/rejected": -385.32257080078125,
"eval_loss": 0.6911596059799194,
"eval_rewards/accuracies": 0.591269850730896,
"eval_rewards/chosen": -0.010051789693534374,
"eval_rewards/margins": 0.005827987566590309,
"eval_rewards/rejected": -0.01587977446615696,
"eval_runtime": 90.093,
"eval_samples_per_second": 11.1,
"eval_steps_per_second": 0.699,
"step": 500
},
{
"epoch": 0.9066666666666666,
"grad_norm": 0.3231988284254439,
"learning_rate": 1.296710737600934e-08,
"logits/chosen": 13.321968078613281,
"logits/rejected": 12.900283813476562,
"logps/chosen": -383.46600341796875,
"logps/rejected": -390.33062744140625,
"loss": 0.6929,
"rewards/accuracies": 0.5687500238418579,
"rewards/chosen": -0.012601424939930439,
"rewards/margins": 0.001571696251630783,
"rewards/rejected": -0.014173120260238647,
"step": 510
},
{
"epoch": 0.9244444444444444,
"grad_norm": 0.23193958475506654,
"learning_rate": 8.485031983924557e-09,
"logits/chosen": 14.106877326965332,
"logits/rejected": 14.698068618774414,
"logps/chosen": -394.9110412597656,
"logps/rejected": -412.6573181152344,
"loss": 0.6932,
"rewards/accuracies": 0.4625000059604645,
"rewards/chosen": -0.011982702650129795,
"rewards/margins": -0.0021997804287821054,
"rewards/rejected": -0.009782921522855759,
"step": 520
},
{
"epoch": 0.9422222222222222,
"grad_norm": 0.24532837907459584,
"learning_rate": 4.937331084660129e-09,
"logits/chosen": 13.21332836151123,
"logits/rejected": 13.805132865905762,
"logps/chosen": -414.30206298828125,
"logps/rejected": -441.9681701660156,
"loss": 0.693,
"rewards/accuracies": 0.4937500059604645,
"rewards/chosen": -0.01276436448097229,
"rewards/margins": -0.0009478643769398332,
"rewards/rejected": -0.01181650161743164,
"step": 530
},
{
"epoch": 0.96,
"grad_norm": 0.24429072534270382,
"learning_rate": 2.337730043793423e-09,
"logits/chosen": 14.035207748413086,
"logits/rejected": 13.626538276672363,
"logps/chosen": -371.00714111328125,
"logps/rejected": -362.235107421875,
"loss": 0.6926,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -0.009096421301364899,
"rewards/margins": 0.0019148699939250946,
"rewards/rejected": -0.011011291295289993,
"step": 540
},
{
"epoch": 0.9777777777777777,
"grad_norm": 0.2572226423400756,
"learning_rate": 6.962862127343205e-10,
"logits/chosen": 13.9146089553833,
"logits/rejected": 14.247346878051758,
"logps/chosen": -406.7580871582031,
"logps/rejected": -410.738037109375,
"loss": 0.6927,
"rewards/accuracies": 0.606249988079071,
"rewards/chosen": -0.00879374984651804,
"rewards/margins": 0.0038339956663548946,
"rewards/rejected": -0.012627745047211647,
"step": 550
},
{
"epoch": 0.9955555555555555,
"grad_norm": 0.2354182073801537,
"learning_rate": 1.9350018786556956e-11,
"logits/chosen": 13.487604141235352,
"logits/rejected": 13.729182243347168,
"logps/chosen": -416.7865295410156,
"logps/rejected": -403.67608642578125,
"loss": 0.6933,
"rewards/accuracies": 0.518750011920929,
"rewards/chosen": -0.01150877121835947,
"rewards/margins": -0.0002604381297715008,
"rewards/rejected": -0.011248333379626274,
"step": 560
},
{
"epoch": 0.9991111111111111,
"step": 562,
"total_flos": 0.0,
"train_loss": 0.6931305729197438,
"train_runtime": 7518.6256,
"train_samples_per_second": 4.788,
"train_steps_per_second": 0.075
}
],
"logging_steps": 10,
"max_steps": 562,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}