LoneStriker's picture
Upload folder using huggingface_hub
a986eb6 verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 2.9952,
"eval_steps": 500,
"global_step": 936,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.016,
"grad_norm": 38.78816510379247,
"learning_rate": 2.6595744680851066e-07,
"logits/chosen": -0.9742305874824524,
"logits/rejected": -1.0986480712890625,
"logps/chosen": -1.3971922397613525,
"logps/rejected": -2.0653083324432373,
"loss": 1.7192,
"odds_ratio_loss": 0.5222790241241455,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.06985961645841599,
"rewards/margins": 0.03340579569339752,
"rewards/rejected": -0.1032654196023941,
"sft_loss": 1.3971922397613525,
"step": 5
},
{
"epoch": 0.032,
"grad_norm": 34.938822021789555,
"learning_rate": 5.319148936170213e-07,
"logits/chosen": -0.9504960775375366,
"logits/rejected": -0.9644749760627747,
"logps/chosen": -1.5183446407318115,
"logps/rejected": -1.7423614263534546,
"loss": 1.7,
"odds_ratio_loss": 0.6908475756645203,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.07591722905635834,
"rewards/margins": 0.011200849898159504,
"rewards/rejected": -0.08711807429790497,
"sft_loss": 1.5183446407318115,
"step": 10
},
{
"epoch": 0.048,
"grad_norm": 16.339902176867668,
"learning_rate": 7.97872340425532e-07,
"logits/chosen": -0.9059197306632996,
"logits/rejected": -0.9509723782539368,
"logps/chosen": -1.3207223415374756,
"logps/rejected": -2.0547330379486084,
"loss": 1.6089,
"odds_ratio_loss": 0.6031222343444824,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.06603612005710602,
"rewards/margins": 0.036700539290905,
"rewards/rejected": -0.10273666679859161,
"sft_loss": 1.3207223415374756,
"step": 15
},
{
"epoch": 0.064,
"grad_norm": 20.917533328584984,
"learning_rate": 1.0638297872340427e-06,
"logits/chosen": -0.9647526741027832,
"logits/rejected": -1.0629401206970215,
"logps/chosen": -1.4611456394195557,
"logps/rejected": -2.1091372966766357,
"loss": 1.4991,
"odds_ratio_loss": 0.5197792649269104,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.07305728644132614,
"rewards/margins": 0.03239959105849266,
"rewards/rejected": -0.1054568737745285,
"sft_loss": 1.4611456394195557,
"step": 20
},
{
"epoch": 0.08,
"grad_norm": 18.664826990233166,
"learning_rate": 1.3297872340425533e-06,
"logits/chosen": -1.054176688194275,
"logits/rejected": -1.3495676517486572,
"logps/chosen": -1.3905874490737915,
"logps/rejected": -1.7380883693695068,
"loss": 1.3634,
"odds_ratio_loss": 0.5891755819320679,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.06952936947345734,
"rewards/margins": 0.017375053837895393,
"rewards/rejected": -0.08690442144870758,
"sft_loss": 1.3905874490737915,
"step": 25
},
{
"epoch": 0.096,
"grad_norm": 10.604261736039259,
"learning_rate": 1.595744680851064e-06,
"logits/chosen": -0.9233123660087585,
"logits/rejected": -0.9947792887687683,
"logps/chosen": -1.3364614248275757,
"logps/rejected": -1.4293016195297241,
"loss": 1.3477,
"odds_ratio_loss": 0.7917504906654358,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.0668230801820755,
"rewards/margins": 0.0046419971622526646,
"rewards/rejected": -0.07146507501602173,
"sft_loss": 1.3364614248275757,
"step": 30
},
{
"epoch": 0.112,
"grad_norm": 9.23627597385138,
"learning_rate": 1.8617021276595745e-06,
"logits/chosen": -1.1354405879974365,
"logits/rejected": -1.1504920721054077,
"logps/chosen": -1.2876451015472412,
"logps/rejected": -1.394587755203247,
"loss": 1.3323,
"odds_ratio_loss": 0.7012001276016235,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.06438224762678146,
"rewards/margins": 0.005347140599042177,
"rewards/rejected": -0.06972938776016235,
"sft_loss": 1.2876451015472412,
"step": 35
},
{
"epoch": 0.128,
"grad_norm": 10.737738874884705,
"learning_rate": 2.1276595744680853e-06,
"logits/chosen": -1.0701355934143066,
"logits/rejected": -1.2617167234420776,
"logps/chosen": -1.345517873764038,
"logps/rejected": -1.5351002216339111,
"loss": 1.3194,
"odds_ratio_loss": 0.6834388971328735,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.06727589666843414,
"rewards/margins": 0.009479111060500145,
"rewards/rejected": -0.07675500959157944,
"sft_loss": 1.345517873764038,
"step": 40
},
{
"epoch": 0.144,
"grad_norm": 3.92140979781429,
"learning_rate": 2.393617021276596e-06,
"logits/chosen": -0.8892070651054382,
"logits/rejected": -0.9303566217422485,
"logps/chosen": -1.1515204906463623,
"logps/rejected": -1.311240315437317,
"loss": 1.3284,
"odds_ratio_loss": 0.6429694890975952,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.05757603049278259,
"rewards/margins": 0.007985993288457394,
"rewards/rejected": -0.06556202471256256,
"sft_loss": 1.1515204906463623,
"step": 45
},
{
"epoch": 0.16,
"grad_norm": 12.131879118457386,
"learning_rate": 2.6595744680851065e-06,
"logits/chosen": -0.9589262008666992,
"logits/rejected": -1.0993459224700928,
"logps/chosen": -1.3476245403289795,
"logps/rejected": -1.4529037475585938,
"loss": 1.3627,
"odds_ratio_loss": 0.6757664680480957,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.06738122552633286,
"rewards/margins": 0.005263959523290396,
"rewards/rejected": -0.07264517992734909,
"sft_loss": 1.3476245403289795,
"step": 50
},
{
"epoch": 0.176,
"grad_norm": 6.920195139152275,
"learning_rate": 2.9255319148936174e-06,
"logits/chosen": -0.9547260403633118,
"logits/rejected": -1.137032151222229,
"logps/chosen": -1.3528764247894287,
"logps/rejected": -1.5240767002105713,
"loss": 1.2606,
"odds_ratio_loss": 0.6684664487838745,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.06764382123947144,
"rewards/margins": 0.008560018613934517,
"rewards/rejected": -0.0762038379907608,
"sft_loss": 1.3528764247894287,
"step": 55
},
{
"epoch": 0.192,
"grad_norm": 5.7098162603211104,
"learning_rate": 3.191489361702128e-06,
"logits/chosen": -0.982739269733429,
"logits/rejected": -1.1074873208999634,
"logps/chosen": -1.4583510160446167,
"logps/rejected": -1.5452958345413208,
"loss": 1.3274,
"odds_ratio_loss": 0.7071143388748169,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.07291755080223083,
"rewards/margins": 0.00434724148362875,
"rewards/rejected": -0.07726480066776276,
"sft_loss": 1.4583510160446167,
"step": 60
},
{
"epoch": 0.208,
"grad_norm": 4.824404122964849,
"learning_rate": 3.457446808510639e-06,
"logits/chosen": -0.9817106127738953,
"logits/rejected": -1.111875295639038,
"logps/chosen": -1.3345075845718384,
"logps/rejected": -1.5089738368988037,
"loss": 1.2879,
"odds_ratio_loss": 0.6608881950378418,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.06672537326812744,
"rewards/margins": 0.008723323233425617,
"rewards/rejected": -0.07544869929552078,
"sft_loss": 1.3345075845718384,
"step": 65
},
{
"epoch": 0.224,
"grad_norm": 6.021701349784469,
"learning_rate": 3.723404255319149e-06,
"logits/chosen": -1.1069040298461914,
"logits/rejected": -1.075409173965454,
"logps/chosen": -1.2757076025009155,
"logps/rejected": -1.4533952474594116,
"loss": 1.3134,
"odds_ratio_loss": 0.7258908152580261,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.0637853816151619,
"rewards/margins": 0.008884383365511894,
"rewards/rejected": -0.07266975939273834,
"sft_loss": 1.2757076025009155,
"step": 70
},
{
"epoch": 0.24,
"grad_norm": 5.02350299089495,
"learning_rate": 3.98936170212766e-06,
"logits/chosen": -0.9449170231819153,
"logits/rejected": -1.1291000843048096,
"logps/chosen": -1.2914139032363892,
"logps/rejected": -1.484882116317749,
"loss": 1.2797,
"odds_ratio_loss": 0.6280355453491211,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.06457068026065826,
"rewards/margins": 0.009673424996435642,
"rewards/rejected": -0.07424411177635193,
"sft_loss": 1.2914139032363892,
"step": 75
},
{
"epoch": 0.256,
"grad_norm": 5.308377873815178,
"learning_rate": 4.255319148936171e-06,
"logits/chosen": -0.9751418828964233,
"logits/rejected": -1.094626545906067,
"logps/chosen": -1.0752859115600586,
"logps/rejected": -1.334636926651001,
"loss": 1.2678,
"odds_ratio_loss": 0.5908417701721191,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.05376429483294487,
"rewards/margins": 0.012967551127076149,
"rewards/rejected": -0.06673184782266617,
"sft_loss": 1.0752859115600586,
"step": 80
},
{
"epoch": 0.272,
"grad_norm": 6.346400541372991,
"learning_rate": 4.521276595744681e-06,
"logits/chosen": -0.9909063577651978,
"logits/rejected": -1.0453189611434937,
"logps/chosen": -1.0384023189544678,
"logps/rejected": -1.2738986015319824,
"loss": 1.2615,
"odds_ratio_loss": 0.6149815320968628,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.051920123398303986,
"rewards/margins": 0.011774817481637001,
"rewards/rejected": -0.06369493901729584,
"sft_loss": 1.0384023189544678,
"step": 85
},
{
"epoch": 0.288,
"grad_norm": 6.055157418934933,
"learning_rate": 4.787234042553192e-06,
"logits/chosen": -0.9102107882499695,
"logits/rejected": -0.9935488700866699,
"logps/chosen": -1.1961522102355957,
"logps/rejected": -1.415398359298706,
"loss": 1.212,
"odds_ratio_loss": 0.6041514277458191,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.059807609766721725,
"rewards/margins": 0.010962305590510368,
"rewards/rejected": -0.07076992094516754,
"sft_loss": 1.1961522102355957,
"step": 90
},
{
"epoch": 0.304,
"grad_norm": 5.552038237166853,
"learning_rate": 4.999982598564682e-06,
"logits/chosen": -0.9868474006652832,
"logits/rejected": -1.0573745965957642,
"logps/chosen": -1.2718483209609985,
"logps/rejected": -1.5264861583709717,
"loss": 1.2618,
"odds_ratio_loss": 0.640211820602417,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.06359241157770157,
"rewards/margins": 0.012731892988085747,
"rewards/rejected": -0.07632430642843246,
"sft_loss": 1.2718483209609985,
"step": 95
},
{
"epoch": 0.32,
"grad_norm": 7.727039615086689,
"learning_rate": 4.999373573764188e-06,
"logits/chosen": -1.0175714492797852,
"logits/rejected": -0.8734873533248901,
"logps/chosen": -1.199947476387024,
"logps/rejected": -1.3050801753997803,
"loss": 1.2807,
"odds_ratio_loss": 0.7060422897338867,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.059997379779815674,
"rewards/margins": 0.005256640259176493,
"rewards/rejected": -0.06525401026010513,
"sft_loss": 1.199947476387024,
"step": 100
},
{
"epoch": 0.336,
"grad_norm": 6.365580704740442,
"learning_rate": 4.997894719430564e-06,
"logits/chosen": -0.8132748603820801,
"logits/rejected": -0.9986258745193481,
"logps/chosen": -1.1854135990142822,
"logps/rejected": -1.5025197267532349,
"loss": 1.2805,
"odds_ratio_loss": 0.6134302020072937,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.05927068740129471,
"rewards/margins": 0.015855297446250916,
"rewards/rejected": -0.07512599229812622,
"sft_loss": 1.1854135990142822,
"step": 105
},
{
"epoch": 0.352,
"grad_norm": 6.860280491199452,
"learning_rate": 4.995546550233241e-06,
"logits/chosen": -0.9532275199890137,
"logits/rejected": -1.136918306350708,
"logps/chosen": -1.408894658088684,
"logps/rejected": -1.558379888534546,
"loss": 1.2769,
"odds_ratio_loss": 0.7106048464775085,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.0704447329044342,
"rewards/margins": 0.007474270649254322,
"rewards/rejected": -0.07791899144649506,
"sft_loss": 1.408894658088684,
"step": 110
},
{
"epoch": 0.368,
"grad_norm": 5.854849184237901,
"learning_rate": 4.992329883379755e-06,
"logits/chosen": -0.9923961758613586,
"logits/rejected": -1.2076470851898193,
"logps/chosen": -1.4104652404785156,
"logps/rejected": -1.4293034076690674,
"loss": 1.2514,
"odds_ratio_loss": 0.7473000288009644,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.07052326202392578,
"rewards/margins": 0.0009419171256013215,
"rewards/rejected": -0.07146517932415009,
"sft_loss": 1.4104652404785156,
"step": 115
},
{
"epoch": 0.384,
"grad_norm": 4.334718029265899,
"learning_rate": 4.988245838331339e-06,
"logits/chosen": -0.9599634408950806,
"logits/rejected": -0.9879687428474426,
"logps/chosen": -1.2545793056488037,
"logps/rejected": -1.4001166820526123,
"loss": 1.2204,
"odds_ratio_loss": 0.6919530630111694,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.06272897124290466,
"rewards/margins": 0.007276860065758228,
"rewards/rejected": -0.07000583410263062,
"sft_loss": 1.2545793056488037,
"step": 120
},
{
"epoch": 0.4,
"grad_norm": 6.640886415637415,
"learning_rate": 4.983295836413337e-06,
"logits/chosen": -0.8892742395401001,
"logits/rejected": -0.8854384422302246,
"logps/chosen": -1.1642634868621826,
"logps/rejected": -1.2482268810272217,
"loss": 1.225,
"odds_ratio_loss": 0.7551355361938477,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.05821318179368973,
"rewards/margins": 0.004198164213448763,
"rewards/rejected": -0.0624113492667675,
"sft_loss": 1.1642634868621826,
"step": 125
},
{
"epoch": 0.416,
"grad_norm": 3.3977384471741727,
"learning_rate": 4.977481600320545e-06,
"logits/chosen": -1.0043504238128662,
"logits/rejected": -1.0277382135391235,
"logps/chosen": -1.2277835607528687,
"logps/rejected": -1.3244746923446655,
"loss": 1.2696,
"odds_ratio_loss": 0.726096510887146,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.06138917803764343,
"rewards/margins": 0.0048345597460865974,
"rewards/rejected": -0.06622373312711716,
"sft_loss": 1.2277835607528687,
"step": 130
},
{
"epoch": 0.432,
"grad_norm": 29.84907862640638,
"learning_rate": 4.970805153517692e-06,
"logits/chosen": -0.906798243522644,
"logits/rejected": -1.0836145877838135,
"logps/chosen": -1.273900032043457,
"logps/rejected": -1.6986215114593506,
"loss": 1.2562,
"odds_ratio_loss": 0.6722120642662048,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.06369500607252121,
"rewards/margins": 0.02123607136309147,
"rewards/rejected": -0.08493108302354813,
"sft_loss": 1.273900032043457,
"step": 135
},
{
"epoch": 0.448,
"grad_norm": 9.009223423852758,
"learning_rate": 4.963268819535228e-06,
"logits/chosen": -0.9239026308059692,
"logits/rejected": -0.9300885200500488,
"logps/chosen": -1.0778651237487793,
"logps/rejected": -1.2015340328216553,
"loss": 1.2297,
"odds_ratio_loss": 0.6974226832389832,
"rewards/accuracies": 0.44999998807907104,
"rewards/chosen": -0.053893256932497025,
"rewards/margins": 0.006183440797030926,
"rewards/rejected": -0.06007670238614082,
"sft_loss": 1.0778651237487793,
"step": 140
},
{
"epoch": 0.464,
"grad_norm": 6.881851804175557,
"learning_rate": 4.954875221160695e-06,
"logits/chosen": -0.9062795639038086,
"logits/rejected": -0.9437534213066101,
"logps/chosen": -1.151301622390747,
"logps/rejected": -1.376366376876831,
"loss": 1.1857,
"odds_ratio_loss": 0.6390897035598755,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.05756508558988571,
"rewards/margins": 0.01125323586165905,
"rewards/rejected": -0.06881832331418991,
"sft_loss": 1.151301622390747,
"step": 145
},
{
"epoch": 0.48,
"grad_norm": 6.024680716397588,
"learning_rate": 4.945627279525943e-06,
"logits/chosen": -0.8047590255737305,
"logits/rejected": -1.0275001525878906,
"logps/chosen": -1.1844927072525024,
"logps/rejected": -1.1749188899993896,
"loss": 1.2559,
"odds_ratio_loss": 0.7716752886772156,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.05922463536262512,
"rewards/margins": -0.0004786819336004555,
"rewards/rejected": -0.05874595046043396,
"sft_loss": 1.1844927072525024,
"step": 150
},
{
"epoch": 0.496,
"grad_norm": 4.969594803216186,
"learning_rate": 4.935528213090526e-06,
"logits/chosen": -0.8657525181770325,
"logits/rejected": -0.9782077074050903,
"logps/chosen": -1.2747188806533813,
"logps/rejected": -1.331703543663025,
"loss": 1.2301,
"odds_ratio_loss": 0.720513641834259,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.06373593956232071,
"rewards/margins": 0.0028492447454482317,
"rewards/rejected": -0.06658518314361572,
"sft_loss": 1.2747188806533813,
"step": 155
},
{
"epoch": 0.512,
"grad_norm": 5.691340876461069,
"learning_rate": 4.9245815365216115e-06,
"logits/chosen": -0.7511823773384094,
"logits/rejected": -1.0508475303649902,
"logps/chosen": -1.1776434183120728,
"logps/rejected": -1.6007537841796875,
"loss": 1.221,
"odds_ratio_loss": 0.5590518116950989,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.05888216570019722,
"rewards/margins": 0.021155523136258125,
"rewards/rejected": -0.08003769814968109,
"sft_loss": 1.1776434183120728,
"step": 160
},
{
"epoch": 0.528,
"grad_norm": 6.0040479808042235,
"learning_rate": 4.912791059470815e-06,
"logits/chosen": -0.9242817163467407,
"logits/rejected": -0.8517144918441772,
"logps/chosen": -1.0615520477294922,
"logps/rejected": -1.288195252418518,
"loss": 1.1745,
"odds_ratio_loss": 0.6426531076431274,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.05307760834693909,
"rewards/margins": 0.011332156136631966,
"rewards/rejected": -0.0644097551703453,
"sft_loss": 1.0615520477294922,
"step": 165
},
{
"epoch": 0.544,
"grad_norm": 5.529927712259793,
"learning_rate": 4.900160885248363e-06,
"logits/chosen": -0.9031238555908203,
"logits/rejected": -0.8960297703742981,
"logps/chosen": -1.2523874044418335,
"logps/rejected": -1.4036993980407715,
"loss": 1.194,
"odds_ratio_loss": 0.6845428347587585,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.06261937320232391,
"rewards/margins": 0.007565596606582403,
"rewards/rejected": -0.07018496841192245,
"sft_loss": 1.2523874044418335,
"step": 170
},
{
"epoch": 0.56,
"grad_norm": 4.1316911858664644,
"learning_rate": 4.886695409395068e-06,
"logits/chosen": -0.853635311126709,
"logits/rejected": -1.0007998943328857,
"logps/chosen": -1.158414602279663,
"logps/rejected": -1.1915799379348755,
"loss": 1.2303,
"odds_ratio_loss": 0.7887060046195984,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.05792072415351868,
"rewards/margins": 0.0016582731623202562,
"rewards/rejected": -0.059578996151685715,
"sft_loss": 1.158414602279663,
"step": 175
},
{
"epoch": 0.576,
"grad_norm": 4.185731700055316,
"learning_rate": 4.872399318152594e-06,
"logits/chosen": -0.9244221448898315,
"logits/rejected": -0.8243404626846313,
"logps/chosen": -1.124448537826538,
"logps/rejected": -1.1855733394622803,
"loss": 1.2359,
"odds_ratio_loss": 0.6966110467910767,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.05622243881225586,
"rewards/margins": 0.0030562314204871655,
"rewards/rejected": -0.059278666973114014,
"sft_loss": 1.124448537826538,
"step": 180
},
{
"epoch": 0.592,
"grad_norm": 5.534373355221194,
"learning_rate": 4.857277586832556e-06,
"logits/chosen": -0.8700960278511047,
"logits/rejected": -1.0795601606369019,
"logps/chosen": -1.1355112791061401,
"logps/rejected": -1.3763678073883057,
"loss": 1.2299,
"odds_ratio_loss": 0.61272132396698,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.05677555873990059,
"rewards/margins": 0.012042825110256672,
"rewards/rejected": -0.06881839036941528,
"sft_loss": 1.1355112791061401,
"step": 185
},
{
"epoch": 0.608,
"grad_norm": 8.320294434791975,
"learning_rate": 4.841335478085015e-06,
"logits/chosen": -0.837981104850769,
"logits/rejected": -1.041286587715149,
"logps/chosen": -1.0703322887420654,
"logps/rejected": -1.6396074295043945,
"loss": 1.1951,
"odds_ratio_loss": 0.5467146635055542,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.05351661518216133,
"rewards/margins": 0.028463756665587425,
"rewards/rejected": -0.08198036998510361,
"sft_loss": 1.0703322887420654,
"step": 190
},
{
"epoch": 0.624,
"grad_norm": 4.60123509876027,
"learning_rate": 4.824578540066981e-06,
"logits/chosen": -0.93339604139328,
"logits/rejected": -0.8643118739128113,
"logps/chosen": -1.3019145727157593,
"logps/rejected": -1.2324789762496948,
"loss": 1.2285,
"odds_ratio_loss": 0.8142224550247192,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.06509573012590408,
"rewards/margins": -0.003471782896667719,
"rewards/rejected": -0.061623942106962204,
"sft_loss": 1.3019145727157593,
"step": 195
},
{
"epoch": 0.64,
"grad_norm": 21.66363438026074,
"learning_rate": 4.807012604511542e-06,
"logits/chosen": -0.6915689706802368,
"logits/rejected": -0.897596538066864,
"logps/chosen": -1.1985743045806885,
"logps/rejected": -1.3054250478744507,
"loss": 1.2092,
"odds_ratio_loss": 0.6641479730606079,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.059928715229034424,
"rewards/margins": 0.005342531017959118,
"rewards/rejected": -0.06527124345302582,
"sft_loss": 1.1985743045806885,
"step": 200
},
{
"epoch": 0.656,
"grad_norm": 5.945812709802962,
"learning_rate": 4.788643784698316e-06,
"logits/chosen": -0.8415448069572449,
"logits/rejected": -0.8126734495162964,
"logps/chosen": -1.1990188360214233,
"logps/rejected": -1.1923567056655884,
"loss": 1.1919,
"odds_ratio_loss": 0.7749138474464417,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.05995094031095505,
"rewards/margins": -0.0003331051266286522,
"rewards/rejected": -0.05961783602833748,
"sft_loss": 1.1990188360214233,
"step": 205
},
{
"epoch": 0.672,
"grad_norm": 6.175015091799187,
"learning_rate": 4.769478473325908e-06,
"logits/chosen": -0.9676336050033569,
"logits/rejected": -1.0954523086547852,
"logps/chosen": -1.0937362909317017,
"logps/rejected": -1.169339895248413,
"loss": 1.2244,
"odds_ratio_loss": 0.7255644798278809,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.054686807096004486,
"rewards/margins": 0.003780178725719452,
"rewards/rejected": -0.058466989547014236,
"sft_loss": 1.0937362909317017,
"step": 210
},
{
"epoch": 0.688,
"grad_norm": 4.1222127125265935,
"learning_rate": 4.7495233402871336e-06,
"logits/chosen": -0.7574236392974854,
"logits/rejected": -0.9107638597488403,
"logps/chosen": -1.0908468961715698,
"logps/rejected": -1.3144160509109497,
"loss": 1.2612,
"odds_ratio_loss": 0.6122642159461975,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.05454235151410103,
"rewards/margins": 0.011178453452885151,
"rewards/rejected": -0.0657208114862442,
"sft_loss": 1.0908468961715698,
"step": 215
},
{
"epoch": 0.704,
"grad_norm": 6.629592523219295,
"learning_rate": 4.728785330347771e-06,
"logits/chosen": -1.0126041173934937,
"logits/rejected": -1.1902836561203003,
"logps/chosen": -1.0830891132354736,
"logps/rejected": -1.2832982540130615,
"loss": 1.1849,
"odds_ratio_loss": 0.6457486748695374,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.05415446311235428,
"rewards/margins": 0.010010452009737492,
"rewards/rejected": -0.0641649141907692,
"sft_loss": 1.0830891132354736,
"step": 220
},
{
"epoch": 0.72,
"grad_norm": 4.769856705836216,
"learning_rate": 4.70727166072964e-06,
"logits/chosen": -0.9934102296829224,
"logits/rejected": -1.0441877841949463,
"logps/chosen": -1.1385221481323242,
"logps/rejected": -1.3962942361831665,
"loss": 1.2004,
"odds_ratio_loss": 0.631730318069458,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.05692611262202263,
"rewards/margins": 0.012888607569038868,
"rewards/rejected": -0.06981472671031952,
"sft_loss": 1.1385221481323242,
"step": 225
},
{
"epoch": 0.736,
"grad_norm": 9.602754215145033,
"learning_rate": 4.684989818598887e-06,
"logits/chosen": -0.8686431050300598,
"logits/rejected": -0.9307680130004883,
"logps/chosen": -1.2431721687316895,
"logps/rejected": -1.3505405187606812,
"loss": 1.1679,
"odds_ratio_loss": 0.7377614974975586,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.06215860694646835,
"rewards/margins": 0.005368414800614119,
"rewards/rejected": -0.06752702593803406,
"sft_loss": 1.2431721687316895,
"step": 230
},
{
"epoch": 0.752,
"grad_norm": 4.991520073276977,
"learning_rate": 4.661947558460296e-06,
"logits/chosen": -0.893735408782959,
"logits/rejected": -1.037488579750061,
"logps/chosen": -1.2366321086883545,
"logps/rejected": -1.422572135925293,
"loss": 1.1988,
"odds_ratio_loss": 0.6442903280258179,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.06183161213994026,
"rewards/margins": 0.009297001175582409,
"rewards/rejected": -0.07112861424684525,
"sft_loss": 1.2366321086883545,
"step": 235
},
{
"epoch": 0.768,
"grad_norm": 5.765863831511136,
"learning_rate": 4.63815289945858e-06,
"logits/chosen": -0.8988453149795532,
"logits/rejected": -1.1190879344940186,
"logps/chosen": -1.1124114990234375,
"logps/rejected": -1.2776150703430176,
"loss": 1.2519,
"odds_ratio_loss": 0.697569727897644,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.055620573461055756,
"rewards/margins": 0.008260180242359638,
"rewards/rejected": -0.06388075649738312,
"sft_loss": 1.1124114990234375,
"step": 240
},
{
"epoch": 0.784,
"grad_norm": 4.2480178226924465,
"learning_rate": 4.613614122587563e-06,
"logits/chosen": -0.8890430331230164,
"logits/rejected": -0.993693470954895,
"logps/chosen": -1.1632936000823975,
"logps/rejected": -1.2893292903900146,
"loss": 1.2055,
"odds_ratio_loss": 0.7878369092941284,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.058164678514003754,
"rewards/margins": 0.0063017881475389,
"rewards/rejected": -0.0644664615392685,
"sft_loss": 1.1632936000823975,
"step": 245
},
{
"epoch": 0.8,
"grad_norm": 4.066953723548615,
"learning_rate": 4.5883397678082385e-06,
"logits/chosen": -0.8833068013191223,
"logits/rejected": -1.0138885974884033,
"logps/chosen": -1.2117881774902344,
"logps/rejected": -1.1593796014785767,
"loss": 1.1903,
"odds_ratio_loss": 0.7823435664176941,
"rewards/accuracies": 0.4749999940395355,
"rewards/chosen": -0.06058941036462784,
"rewards/margins": -0.002620431361719966,
"rewards/rejected": -0.05796898156404495,
"sft_loss": 1.2117881774902344,
"step": 250
},
{
"epoch": 0.816,
"grad_norm": 5.662450247054175,
"learning_rate": 4.562338631076703e-06,
"logits/chosen": -1.0948327779769897,
"logits/rejected": -1.1347147226333618,
"logps/chosen": -1.0046937465667725,
"logps/rejected": -1.2093970775604248,
"loss": 1.1825,
"odds_ratio_loss": 0.6374476552009583,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.05023468658328056,
"rewards/margins": 0.010235178284347057,
"rewards/rejected": -0.060469865798950195,
"sft_loss": 1.0046937465667725,
"step": 255
},
{
"epoch": 0.832,
"grad_norm": 5.116135672128333,
"learning_rate": 4.535619761282989e-06,
"logits/chosen": -0.9458154439926147,
"logits/rejected": -1.224017858505249,
"logps/chosen": -1.202074646949768,
"logps/rejected": -1.3588545322418213,
"loss": 1.23,
"odds_ratio_loss": 0.668530285358429,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.06010373681783676,
"rewards/margins": 0.007838994264602661,
"rewards/rejected": -0.06794272363185883,
"sft_loss": 1.202074646949768,
"step": 260
},
{
"epoch": 0.848,
"grad_norm": 8.619415579204428,
"learning_rate": 4.508192457101886e-06,
"logits/chosen": -1.0687763690948486,
"logits/rejected": -1.213895320892334,
"logps/chosen": -1.2021548748016357,
"logps/rejected": -1.541288137435913,
"loss": 1.2482,
"odds_ratio_loss": 0.658495306968689,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.060107748955488205,
"rewards/margins": 0.016956666484475136,
"rewards/rejected": -0.0770644098520279,
"sft_loss": 1.2021548748016357,
"step": 265
},
{
"epoch": 0.864,
"grad_norm": 5.000713262682842,
"learning_rate": 4.480066263756821e-06,
"logits/chosen": -0.9636927843093872,
"logits/rejected": -1.1979575157165527,
"logps/chosen": -1.220902442932129,
"logps/rejected": -1.5320258140563965,
"loss": 1.1889,
"odds_ratio_loss": 0.5880663990974426,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.06104512885212898,
"rewards/margins": 0.01555616408586502,
"rewards/rejected": -0.0766012892127037,
"sft_loss": 1.220902442932129,
"step": 270
},
{
"epoch": 0.88,
"grad_norm": 6.295390583446683,
"learning_rate": 4.451250969697944e-06,
"logits/chosen": -0.9099109768867493,
"logits/rejected": -1.1926413774490356,
"logps/chosen": -1.1539406776428223,
"logps/rejected": -1.4602700471878052,
"loss": 1.2078,
"odds_ratio_loss": 0.5688539743423462,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.05769703909754753,
"rewards/margins": 0.015316471457481384,
"rewards/rejected": -0.07301349937915802,
"sft_loss": 1.1539406776428223,
"step": 275
},
{
"epoch": 0.896,
"grad_norm": 3.602560879317352,
"learning_rate": 4.42175660319555e-06,
"logits/chosen": -1.159501314163208,
"logits/rejected": -1.2406375408172607,
"logps/chosen": -1.192801594734192,
"logps/rejected": -1.3772008419036865,
"loss": 1.1898,
"odds_ratio_loss": 0.7453117966651917,
"rewards/accuracies": 0.5249999761581421,
"rewards/chosen": -0.059640079736709595,
"rewards/margins": 0.00921996496617794,
"rewards/rejected": -0.06886004656553268,
"sft_loss": 1.192801594734192,
"step": 280
},
{
"epoch": 0.912,
"grad_norm": 8.783987785350908,
"learning_rate": 4.391593428850069e-06,
"logits/chosen": -0.9893490076065063,
"logits/rejected": -1.1613223552703857,
"logps/chosen": -1.0789204835891724,
"logps/rejected": -1.1841206550598145,
"loss": 1.2078,
"odds_ratio_loss": 0.7538634538650513,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.05394602566957474,
"rewards/margins": 0.005260012112557888,
"rewards/rejected": -0.0592060312628746,
"sft_loss": 1.0789204835891724,
"step": 285
},
{
"epoch": 0.928,
"grad_norm": 6.736280205252853,
"learning_rate": 4.360771944019767e-06,
"logits/chosen": -1.0662925243377686,
"logits/rejected": -1.1013010740280151,
"logps/chosen": -1.2803829908370972,
"logps/rejected": -1.6290887594223022,
"loss": 1.2225,
"odds_ratio_loss": 0.6528395414352417,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.06401915848255157,
"rewards/margins": 0.017435286194086075,
"rewards/rejected": -0.08145444095134735,
"sft_loss": 1.2803829908370972,
"step": 290
},
{
"epoch": 0.944,
"grad_norm": 7.2196796373916365,
"learning_rate": 4.329302875167486e-06,
"logits/chosen": -1.0543363094329834,
"logits/rejected": -1.2899835109710693,
"logps/chosen": -1.1794860363006592,
"logps/rejected": -1.4053813219070435,
"loss": 1.2094,
"odds_ratio_loss": 0.6699637174606323,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.05897430330514908,
"rewards/margins": 0.011294771917164326,
"rewards/rejected": -0.07026907801628113,
"sft_loss": 1.1794860363006592,
"step": 295
},
{
"epoch": 0.96,
"grad_norm": 5.962697902517566,
"learning_rate": 4.297197174127619e-06,
"logits/chosen": -1.0228321552276611,
"logits/rejected": -1.2345274686813354,
"logps/chosen": -1.0752114057540894,
"logps/rejected": -1.2755658626556396,
"loss": 1.212,
"odds_ratio_loss": 0.6680848002433777,
"rewards/accuracies": 0.550000011920929,
"rewards/chosen": -0.05376056954264641,
"rewards/margins": 0.01001772377640009,
"rewards/rejected": -0.06377829611301422,
"sft_loss": 1.0752114057540894,
"step": 300
},
{
"epoch": 0.976,
"grad_norm": 3.4092575585850917,
"learning_rate": 4.2644660142946685e-06,
"logits/chosen": -0.9500937461853027,
"logits/rejected": -1.0103486776351929,
"logps/chosen": -1.0463281869888306,
"logps/rejected": -1.4567468166351318,
"loss": 1.2082,
"odds_ratio_loss": 0.5394900441169739,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.05231640487909317,
"rewards/margins": 0.020520929247140884,
"rewards/rejected": -0.07283733785152435,
"sft_loss": 1.0463281869888306,
"step": 305
},
{
"epoch": 0.992,
"grad_norm": 4.944690721660475,
"learning_rate": 4.231120786734689e-06,
"logits/chosen": -1.0554213523864746,
"logits/rejected": -1.2123026847839355,
"logps/chosen": -1.0677235126495361,
"logps/rejected": -1.427178978919983,
"loss": 1.1548,
"odds_ratio_loss": 0.5765484571456909,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.05338617414236069,
"rewards/margins": 0.01797277294099331,
"rewards/rejected": -0.07135894149541855,
"sft_loss": 1.0677235126495361,
"step": 310
},
{
"epoch": 1.008,
"grad_norm": 3.9331589062338344,
"learning_rate": 4.197173096220983e-06,
"logits/chosen": -1.0465214252471924,
"logits/rejected": -1.0950441360473633,
"logps/chosen": -1.0325384140014648,
"logps/rejected": -1.4658787250518799,
"loss": 1.1007,
"odds_ratio_loss": 0.5475068688392639,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.05162692070007324,
"rewards/margins": 0.02166702225804329,
"rewards/rejected": -0.07329393923282623,
"sft_loss": 1.0325384140014648,
"step": 315
},
{
"epoch": 1.024,
"grad_norm": 2.810636096476599,
"learning_rate": 4.162634757195418e-06,
"logits/chosen": -0.9247652888298035,
"logits/rejected": -1.031118392944336,
"logps/chosen": -0.9556388854980469,
"logps/rejected": -1.2451565265655518,
"loss": 0.9386,
"odds_ratio_loss": 0.594650149345398,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.047781944274902344,
"rewards/margins": 0.014475872740149498,
"rewards/rejected": -0.06225781887769699,
"sft_loss": 0.9556388854980469,
"step": 320
},
{
"epoch": 1.04,
"grad_norm": 3.0448051628925454,
"learning_rate": 4.127517789656772e-06,
"logits/chosen": -0.9287153482437134,
"logits/rejected": -1.0283979177474976,
"logps/chosen": -1.0202507972717285,
"logps/rejected": -1.2847683429718018,
"loss": 1.0353,
"odds_ratio_loss": 0.5959427952766418,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.051012538373470306,
"rewards/margins": 0.01322587113827467,
"rewards/rejected": -0.06423841416835785,
"sft_loss": 1.0202507972717285,
"step": 325
},
{
"epoch": 1.056,
"grad_norm": 4.1771500190151825,
"learning_rate": 4.091834414977556e-06,
"logits/chosen": -1.0129518508911133,
"logits/rejected": -1.271144986152649,
"logps/chosen": -1.018913984298706,
"logps/rejected": -1.554570198059082,
"loss": 0.9511,
"odds_ratio_loss": 0.47632694244384766,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.0509456992149353,
"rewards/margins": 0.026782814413309097,
"rewards/rejected": -0.0777285099029541,
"sft_loss": 1.018913984298706,
"step": 330
},
{
"epoch": 1.072,
"grad_norm": 6.50448197903886,
"learning_rate": 4.055597051650731e-06,
"logits/chosen": -1.0193812847137451,
"logits/rejected": -1.1772067546844482,
"logps/chosen": -0.9281299710273743,
"logps/rejected": -1.3907569646835327,
"loss": 0.9662,
"odds_ratio_loss": 0.5411213040351868,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.046406496316194534,
"rewards/margins": 0.023131350055336952,
"rewards/rejected": -0.06953784823417664,
"sft_loss": 0.9281299710273743,
"step": 335
},
{
"epoch": 1.088,
"grad_norm": 5.600060917092442,
"learning_rate": 4.018818310967843e-06,
"logits/chosen": -1.1713335514068604,
"logits/rejected": -1.2236130237579346,
"logps/chosen": -1.0068949460983276,
"logps/rejected": -1.1553858518600464,
"loss": 0.9817,
"odds_ratio_loss": 0.7148981094360352,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.05034474655985832,
"rewards/margins": 0.007424544543027878,
"rewards/rejected": -0.0577692911028862,
"sft_loss": 1.0068949460983276,
"step": 340
},
{
"epoch": 1.104,
"grad_norm": 5.092643762749835,
"learning_rate": 3.981510992630055e-06,
"logits/chosen": -0.9599639177322388,
"logits/rejected": -1.189029574394226,
"logps/chosen": -0.9695978164672852,
"logps/rejected": -1.4548832178115845,
"loss": 0.9953,
"odds_ratio_loss": 0.5450643301010132,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.048479896038770676,
"rewards/margins": 0.024264268577098846,
"rewards/rejected": -0.07274416834115982,
"sft_loss": 0.9695978164672852,
"step": 345
},
{
"epoch": 1.12,
"grad_norm": 4.234304914475125,
"learning_rate": 3.943688080293607e-06,
"logits/chosen": -0.9754691123962402,
"logits/rejected": -1.1199839115142822,
"logps/chosen": -0.894666314125061,
"logps/rejected": -1.577906847000122,
"loss": 0.9707,
"odds_ratio_loss": 0.45880669355392456,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.044733308255672455,
"rewards/margins": 0.03416203707456589,
"rewards/rejected": -0.07889535278081894,
"sft_loss": 0.894666314125061,
"step": 350
},
{
"epoch": 1.1360000000000001,
"grad_norm": 4.426257939120987,
"learning_rate": 3.905362737051252e-06,
"logits/chosen": -1.0120149850845337,
"logits/rejected": -1.1284050941467285,
"logps/chosen": -0.8394268751144409,
"logps/rejected": -1.3487886190414429,
"loss": 0.9606,
"odds_ratio_loss": 0.5031847357749939,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.041971348226070404,
"rewards/margins": 0.02546808496117592,
"rewards/rejected": -0.06743943691253662,
"sft_loss": 0.8394268751144409,
"step": 355
},
{
"epoch": 1.152,
"grad_norm": 6.846072733066629,
"learning_rate": 3.866548300851254e-06,
"logits/chosen": -1.0661356449127197,
"logits/rejected": -1.1536469459533691,
"logps/chosen": -0.9550708532333374,
"logps/rejected": -1.400836706161499,
"loss": 0.9989,
"odds_ratio_loss": 0.5404989123344421,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.04775355011224747,
"rewards/margins": 0.02228829450905323,
"rewards/rejected": -0.07004183530807495,
"sft_loss": 0.9550708532333374,
"step": 360
},
{
"epoch": 1.168,
"grad_norm": 4.238159230888541,
"learning_rate": 3.827258279855527e-06,
"logits/chosen": -1.1802518367767334,
"logits/rejected": -1.206186056137085,
"logps/chosen": -1.0155624151229858,
"logps/rejected": -1.3900431394577026,
"loss": 0.9456,
"odds_ratio_loss": 0.6698687672615051,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.05077812820672989,
"rewards/margins": 0.01872403547167778,
"rewards/rejected": -0.06950215995311737,
"sft_loss": 1.0155624151229858,
"step": 365
},
{
"epoch": 1.184,
"grad_norm": 3.7353385077354337,
"learning_rate": 3.787506347738538e-06,
"logits/chosen": -0.9678732752799988,
"logits/rejected": -1.1043659448623657,
"logps/chosen": -0.9759182929992676,
"logps/rejected": -1.3616999387741089,
"loss": 0.9721,
"odds_ratio_loss": 0.6270595788955688,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.0487959161400795,
"rewards/margins": 0.019289087504148483,
"rewards/rejected": -0.06808499991893768,
"sft_loss": 0.9759182929992676,
"step": 370
},
{
"epoch": 1.2,
"grad_norm": 4.7540211314956204,
"learning_rate": 3.747306338928609e-06,
"logits/chosen": -1.049630045890808,
"logits/rejected": -1.2382904291152954,
"logps/chosen": -0.938238263130188,
"logps/rejected": -1.2308955192565918,
"loss": 1.0158,
"odds_ratio_loss": 0.5973559021949768,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.04691191390156746,
"rewards/margins": 0.01463286578655243,
"rewards/rejected": -0.06154477596282959,
"sft_loss": 0.938238263130188,
"step": 375
},
{
"epoch": 1.216,
"grad_norm": 5.565360284878935,
"learning_rate": 3.706672243793271e-06,
"logits/chosen": -0.9532748460769653,
"logits/rejected": -1.03403902053833,
"logps/chosen": -0.9681148529052734,
"logps/rejected": -1.2792482376098633,
"loss": 0.9464,
"odds_ratio_loss": 0.5441717505455017,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.04840574413537979,
"rewards/margins": 0.015556666068732738,
"rewards/rejected": -0.0639624148607254,
"sft_loss": 0.9681148529052734,
"step": 380
},
{
"epoch": 1.232,
"grad_norm": 3.447185504214593,
"learning_rate": 3.665618203770352e-06,
"logits/chosen": -1.0210788249969482,
"logits/rejected": -1.1396093368530273,
"logps/chosen": -0.8457318544387817,
"logps/rejected": -1.2301608324050903,
"loss": 0.9478,
"odds_ratio_loss": 0.5166088342666626,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.04228659346699715,
"rewards/margins": 0.01922144927084446,
"rewards/rejected": -0.06150804087519646,
"sft_loss": 0.8457318544387817,
"step": 385
},
{
"epoch": 1.248,
"grad_norm": 3.3338981917851047,
"learning_rate": 3.6241585064464846e-06,
"logits/chosen": -1.2911275625228882,
"logits/rejected": -1.1232960224151611,
"logps/chosen": -0.8720508813858032,
"logps/rejected": -1.23488187789917,
"loss": 0.9435,
"odds_ratio_loss": 0.5066360831260681,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.04360254481434822,
"rewards/margins": 0.01814154163002968,
"rewards/rejected": -0.0617440864443779,
"sft_loss": 0.8720508813858032,
"step": 390
},
{
"epoch": 1.264,
"grad_norm": 3.567835515660478,
"learning_rate": 3.582307580584759e-06,
"logits/chosen": -0.959577739238739,
"logits/rejected": -1.2563647031784058,
"logps/chosen": -0.8968993425369263,
"logps/rejected": -1.2443525791168213,
"loss": 0.9669,
"odds_ratio_loss": 0.521682620048523,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04484497010707855,
"rewards/margins": 0.01737266220152378,
"rewards/rejected": -0.062217630445957184,
"sft_loss": 0.8968993425369263,
"step": 395
},
{
"epoch": 1.28,
"grad_norm": 3.9755484861044956,
"learning_rate": 3.5400799911032357e-06,
"logits/chosen": -1.054391860961914,
"logits/rejected": -1.017757534980774,
"logps/chosen": -0.8325561285018921,
"logps/rejected": -1.2529734373092651,
"loss": 0.9323,
"odds_ratio_loss": 0.4909485876560211,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.041627805680036545,
"rewards/margins": 0.021020859479904175,
"rewards/rejected": -0.06264867633581161,
"sft_loss": 0.8325561285018921,
"step": 400
},
{
"epoch": 1.296,
"grad_norm": 3.4685607506272476,
"learning_rate": 3.4974904340060756e-06,
"logits/chosen": -1.004164218902588,
"logits/rejected": -1.320536732673645,
"logps/chosen": -1.0670472383499146,
"logps/rejected": -1.2909588813781738,
"loss": 0.9978,
"odds_ratio_loss": 0.6580184102058411,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.05335236340761185,
"rewards/margins": 0.011195586062967777,
"rewards/rejected": -0.06454795598983765,
"sft_loss": 1.0670472383499146,
"step": 405
},
{
"epoch": 1.312,
"grad_norm": 4.3019598153539516,
"learning_rate": 3.4545537312690565e-06,
"logits/chosen": -1.1336114406585693,
"logits/rejected": -1.1372759342193604,
"logps/chosen": -0.9065093994140625,
"logps/rejected": -1.415405511856079,
"loss": 0.9679,
"odds_ratio_loss": 0.4388805031776428,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.045325469225645065,
"rewards/margins": 0.02544480562210083,
"rewards/rejected": -0.0707702785730362,
"sft_loss": 0.9065093994140625,
"step": 410
},
{
"epoch": 1.328,
"grad_norm": 3.490824264107295,
"learning_rate": 3.4112848256812374e-06,
"logits/chosen": -1.0414103269577026,
"logits/rejected": -1.164736032485962,
"logps/chosen": -0.8454300761222839,
"logps/rejected": -1.311577558517456,
"loss": 0.9274,
"odds_ratio_loss": 0.503560483455658,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.04227150231599808,
"rewards/margins": 0.02330738492310047,
"rewards/rejected": -0.0655788853764534,
"sft_loss": 0.8454300761222839,
"step": 415
},
{
"epoch": 1.3439999999999999,
"grad_norm": 8.789242453958117,
"learning_rate": 3.3676987756445894e-06,
"logits/chosen": -0.935681164264679,
"logits/rejected": -1.1432523727416992,
"logps/chosen": -0.9042528867721558,
"logps/rejected": -1.587890386581421,
"loss": 0.9137,
"odds_ratio_loss": 0.47100192308425903,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04521264508366585,
"rewards/margins": 0.0341818742454052,
"rewards/rejected": -0.07939452677965164,
"sft_loss": 0.9042528867721558,
"step": 420
},
{
"epoch": 1.3599999999999999,
"grad_norm": 4.930177739224116,
"learning_rate": 3.323810749933381e-06,
"logits/chosen": -1.0698130130767822,
"logits/rejected": -1.226738691329956,
"logps/chosen": -0.8717530369758606,
"logps/rejected": -1.4678089618682861,
"loss": 0.9124,
"odds_ratio_loss": 0.4427013397216797,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.04358765110373497,
"rewards/margins": 0.02980278991162777,
"rewards/rejected": -0.07339043915271759,
"sft_loss": 0.8717530369758606,
"step": 425
},
{
"epoch": 1.376,
"grad_norm": 7.570978097521412,
"learning_rate": 3.2796360224151587e-06,
"logits/chosen": -1.2224234342575073,
"logits/rejected": -1.3439141511917114,
"logps/chosen": -0.9194822311401367,
"logps/rejected": -1.3092845678329468,
"loss": 0.9594,
"odds_ratio_loss": 0.5272402763366699,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.045974113047122955,
"rewards/margins": 0.019490113481879234,
"rewards/rejected": -0.06546422094106674,
"sft_loss": 0.9194822311401367,
"step": 430
},
{
"epoch": 1.392,
"grad_norm": 4.982061148370284,
"learning_rate": 3.235189966735148e-06,
"logits/chosen": -1.2072560787200928,
"logits/rejected": -1.1339049339294434,
"logps/chosen": -0.805392861366272,
"logps/rejected": -1.594913125038147,
"loss": 0.9259,
"odds_ratio_loss": 0.406252384185791,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.0402696393430233,
"rewards/margins": 0.03947601094841957,
"rewards/rejected": -0.07974565029144287,
"sft_loss": 0.805392861366272,
"step": 435
},
{
"epoch": 1.408,
"grad_norm": 7.692151482525084,
"learning_rate": 3.1904880509659397e-06,
"logits/chosen": -1.0890783071517944,
"logits/rejected": -1.177278757095337,
"logps/chosen": -0.9858812093734741,
"logps/rejected": -1.204777479171753,
"loss": 0.9667,
"odds_ratio_loss": 0.637945830821991,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.049294065684080124,
"rewards/margins": 0.010944806039333344,
"rewards/rejected": -0.060238875448703766,
"sft_loss": 0.9858812093734741,
"step": 440
},
{
"epoch": 1.424,
"grad_norm": 4.492542501706703,
"learning_rate": 3.1455458322242943e-06,
"logits/chosen": -1.004298448562622,
"logits/rejected": -1.1840426921844482,
"logps/chosen": -0.9921313524246216,
"logps/rejected": -1.2852861881256104,
"loss": 0.9938,
"odds_ratio_loss": 0.6350023150444031,
"rewards/accuracies": 0.6000000238418579,
"rewards/chosen": -0.0496065691113472,
"rewards/margins": 0.014657738618552685,
"rewards/rejected": -0.06426431238651276,
"sft_loss": 0.9921313524246216,
"step": 445
},
{
"epoch": 1.44,
"grad_norm": 4.069388708509856,
"learning_rate": 3.100378951256981e-06,
"logits/chosen": -1.2047039270401,
"logits/rejected": -1.2526731491088867,
"logps/chosen": -0.8970128297805786,
"logps/rejected": -1.087720274925232,
"loss": 0.9649,
"odds_ratio_loss": 0.6119378805160522,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.04485064372420311,
"rewards/margins": 0.009535368531942368,
"rewards/rejected": -0.05438600853085518,
"sft_loss": 0.8970128297805786,
"step": 450
},
{
"epoch": 1.456,
"grad_norm": 4.1796047584641745,
"learning_rate": 3.055003126997495e-06,
"logits/chosen": -1.100693941116333,
"logits/rejected": -1.1093151569366455,
"logps/chosen": -0.9937971234321594,
"logps/rejected": -1.3282194137573242,
"loss": 0.9642,
"odds_ratio_loss": 0.5750898122787476,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.04968985915184021,
"rewards/margins": 0.01672111637890339,
"rewards/rejected": -0.06641098111867905,
"sft_loss": 0.9937971234321594,
"step": 455
},
{
"epoch": 1.472,
"grad_norm": 4.0507990154789075,
"learning_rate": 3.0094341510955697e-06,
"logits/chosen": -1.1363385915756226,
"logits/rejected": -1.1738272905349731,
"logps/chosen": -0.877064049243927,
"logps/rejected": -1.4007574319839478,
"loss": 0.959,
"odds_ratio_loss": 0.5275009870529175,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04385320469737053,
"rewards/margins": 0.026184672489762306,
"rewards/rejected": -0.07003787159919739,
"sft_loss": 0.877064049243927,
"step": 460
},
{
"epoch": 1.488,
"grad_norm": 5.088249905544024,
"learning_rate": 2.963687882421383e-06,
"logits/chosen": -1.096651554107666,
"logits/rejected": -1.1972352266311646,
"logps/chosen": -0.9147326350212097,
"logps/rejected": -1.253187894821167,
"loss": 0.9591,
"odds_ratio_loss": 0.5869191288948059,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.04573662951588631,
"rewards/margins": 0.01692276820540428,
"rewards/rejected": -0.06265939772129059,
"sft_loss": 0.9147326350212097,
"step": 465
},
{
"epoch": 1.504,
"grad_norm": 4.206721558657063,
"learning_rate": 2.9177802415463714e-06,
"logits/chosen": -0.9876214265823364,
"logits/rejected": -1.073717713356018,
"logps/chosen": -0.7661324143409729,
"logps/rejected": -1.2461497783660889,
"loss": 0.9619,
"odds_ratio_loss": 0.45304179191589355,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.038306623697280884,
"rewards/margins": 0.0240008607506752,
"rewards/rejected": -0.062307484447956085,
"sft_loss": 0.7661324143409729,
"step": 470
},
{
"epoch": 1.52,
"grad_norm": 3.67250528125703,
"learning_rate": 2.871727205202563e-06,
"logits/chosen": -1.0482373237609863,
"logits/rejected": -1.1120043992996216,
"logps/chosen": -0.950273871421814,
"logps/rejected": -1.2787425518035889,
"loss": 0.9498,
"odds_ratio_loss": 0.599994421005249,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.047513701021671295,
"rewards/margins": 0.016423430293798447,
"rewards/rejected": -0.06393712759017944,
"sft_loss": 0.950273871421814,
"step": 475
},
{
"epoch": 1.536,
"grad_norm": 4.735321963888987,
"learning_rate": 2.825544800722376e-06,
"logits/chosen": -1.1098222732543945,
"logits/rejected": -1.0207080841064453,
"logps/chosen": -0.9046823382377625,
"logps/rejected": -1.3911302089691162,
"loss": 0.9629,
"odds_ratio_loss": 0.5132350921630859,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.04523410648107529,
"rewards/margins": 0.0243223924189806,
"rewards/rejected": -0.06955650448799133,
"sft_loss": 0.9046823382377625,
"step": 480
},
{
"epoch": 1.552,
"grad_norm": 3.159767853907868,
"learning_rate": 2.7792491004607984e-06,
"logits/chosen": -1.0096070766448975,
"logits/rejected": -1.0770386457443237,
"logps/chosen": -0.9277147054672241,
"logps/rejected": -1.344773530960083,
"loss": 0.9485,
"odds_ratio_loss": 0.5044125318527222,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.046385735273361206,
"rewards/margins": 0.020852940157055855,
"rewards/rejected": -0.06723867356777191,
"sft_loss": 0.9277147054672241,
"step": 485
},
{
"epoch": 1.568,
"grad_norm": 4.013671913402214,
"learning_rate": 2.732856216201906e-06,
"logits/chosen": -1.063245415687561,
"logits/rejected": -1.1233893632888794,
"logps/chosen": -0.9281940460205078,
"logps/rejected": -1.3444063663482666,
"loss": 0.9529,
"odds_ratio_loss": 0.489690363407135,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04640969634056091,
"rewards/margins": 0.020810618996620178,
"rewards/rejected": -0.06722031533718109,
"sft_loss": 0.9281940460205078,
"step": 490
},
{
"epoch": 1.584,
"grad_norm": 4.31314553790671,
"learning_rate": 2.6863822935516546e-06,
"logits/chosen": -1.0234626531600952,
"logits/rejected": -1.1978650093078613,
"logps/chosen": -0.9632134437561035,
"logps/rejected": -1.206189751625061,
"loss": 0.9733,
"odds_ratio_loss": 0.5884974002838135,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.048160675913095474,
"rewards/margins": 0.012148816138505936,
"rewards/rejected": -0.06030949205160141,
"sft_loss": 0.9632134437561035,
"step": 495
},
{
"epoch": 1.6,
"grad_norm": 3.3125411606876445,
"learning_rate": 2.639843506318899e-06,
"logits/chosen": -0.9256556630134583,
"logits/rejected": -1.1392202377319336,
"logps/chosen": -0.8793741464614868,
"logps/rejected": -1.3441039323806763,
"loss": 0.9562,
"odds_ratio_loss": 0.510412335395813,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04396871104836464,
"rewards/margins": 0.023236487060785294,
"rewards/rejected": -0.06720519065856934,
"sft_loss": 0.8793741464614868,
"step": 500
},
{
"epoch": 1.616,
"grad_norm": 5.866811891887456,
"learning_rate": 2.593256050886603e-06,
"logits/chosen": -0.8647163510322571,
"logits/rejected": -1.0798470973968506,
"logps/chosen": -0.9747751951217651,
"logps/rejected": -1.2537850141525269,
"loss": 0.9531,
"odds_ratio_loss": 0.5470961332321167,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.0487387590110302,
"rewards/margins": 0.01395049411803484,
"rewards/rejected": -0.06268925964832306,
"sft_loss": 0.9747751951217651,
"step": 505
},
{
"epoch": 1.6320000000000001,
"grad_norm": 4.074172203592654,
"learning_rate": 2.5466361405751914e-06,
"logits/chosen": -0.9186518788337708,
"logits/rejected": -1.1423577070236206,
"logps/chosen": -0.9840022921562195,
"logps/rejected": -1.3202232122421265,
"loss": 0.949,
"odds_ratio_loss": 0.564594566822052,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.049200110137462616,
"rewards/margins": 0.01681104302406311,
"rewards/rejected": -0.06601114571094513,
"sft_loss": 0.9840022921562195,
"step": 510
},
{
"epoch": 1.6480000000000001,
"grad_norm": 5.002204843627367,
"learning_rate": 2.5e-06,
"logits/chosen": -0.8605943918228149,
"logits/rejected": -0.9710782766342163,
"logps/chosen": -0.8691279292106628,
"logps/rejected": -1.2410802841186523,
"loss": 0.9795,
"odds_ratio_loss": 0.5254562497138977,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.04345639795064926,
"rewards/margins": 0.018597617745399475,
"rewards/rejected": -0.06205401569604874,
"sft_loss": 0.8691279292106628,
"step": 515
},
{
"epoch": 1.6640000000000001,
"grad_norm": 8.858448372423531,
"learning_rate": 2.4533638594248094e-06,
"logits/chosen": -0.9867407083511353,
"logits/rejected": -1.0888783931732178,
"logps/chosen": -0.9163697957992554,
"logps/rejected": -1.1703190803527832,
"loss": 0.9221,
"odds_ratio_loss": 0.5501306056976318,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.045818496495485306,
"rewards/margins": 0.012697461061179638,
"rewards/rejected": -0.05851595476269722,
"sft_loss": 0.9163697957992554,
"step": 520
},
{
"epoch": 1.6800000000000002,
"grad_norm": 4.445755031880296,
"learning_rate": 2.406743949113397e-06,
"logits/chosen": -0.9282479286193848,
"logits/rejected": -1.1179710626602173,
"logps/chosen": -0.872515082359314,
"logps/rejected": -1.3933911323547363,
"loss": 0.9773,
"odds_ratio_loss": 0.5091356039047241,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.04362575337290764,
"rewards/margins": 0.026043808087706566,
"rewards/rejected": -0.06966955959796906,
"sft_loss": 0.872515082359314,
"step": 525
},
{
"epoch": 1.696,
"grad_norm": 3.6413598914526486,
"learning_rate": 2.360156493681102e-06,
"logits/chosen": -0.910436749458313,
"logits/rejected": -1.0247265100479126,
"logps/chosen": -0.9496439695358276,
"logps/rejected": -1.3968040943145752,
"loss": 0.9491,
"odds_ratio_loss": 0.48868894577026367,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.0474822036921978,
"rewards/margins": 0.0223580040037632,
"rewards/rejected": -0.069840207695961,
"sft_loss": 0.9496439695358276,
"step": 530
},
{
"epoch": 1.712,
"grad_norm": 4.294695857238959,
"learning_rate": 2.3136177064483462e-06,
"logits/chosen": -1.0878788232803345,
"logits/rejected": -1.283992052078247,
"logps/chosen": -1.003148078918457,
"logps/rejected": -1.4421515464782715,
"loss": 0.9586,
"odds_ratio_loss": 0.49956098198890686,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.05015740543603897,
"rewards/margins": 0.021950174123048782,
"rewards/rejected": -0.07210757583379745,
"sft_loss": 1.003148078918457,
"step": 535
},
{
"epoch": 1.728,
"grad_norm": 4.183928338953188,
"learning_rate": 2.2671437837980943e-06,
"logits/chosen": -1.0223979949951172,
"logits/rejected": -0.9681426286697388,
"logps/chosen": -0.8269376754760742,
"logps/rejected": -1.141745924949646,
"loss": 0.8999,
"odds_ratio_loss": 0.5333009958267212,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.04134688898921013,
"rewards/margins": 0.0157404113560915,
"rewards/rejected": -0.05708730220794678,
"sft_loss": 0.8269376754760742,
"step": 540
},
{
"epoch": 1.744,
"grad_norm": 4.65432905249252,
"learning_rate": 2.2207508995392024e-06,
"logits/chosen": -0.9679327011108398,
"logits/rejected": -1.0990087985992432,
"logps/chosen": -0.9268666505813599,
"logps/rejected": -1.2932831048965454,
"loss": 0.9613,
"odds_ratio_loss": 0.5248704552650452,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.04634333401918411,
"rewards/margins": 0.018320811912417412,
"rewards/rejected": -0.06466414779424667,
"sft_loss": 0.9268666505813599,
"step": 545
},
{
"epoch": 1.76,
"grad_norm": 3.9586982529561205,
"learning_rate": 2.1744551992776247e-06,
"logits/chosen": -1.0059576034545898,
"logits/rejected": -1.0268685817718506,
"logps/chosen": -0.9964353442192078,
"logps/rejected": -1.6557565927505493,
"loss": 0.9815,
"odds_ratio_loss": 0.49777206778526306,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.04982176423072815,
"rewards/margins": 0.032966069877147675,
"rewards/rejected": -0.08278782665729523,
"sft_loss": 0.9964353442192078,
"step": 550
},
{
"epoch": 1.776,
"grad_norm": 9.562472852188419,
"learning_rate": 2.1282727947974373e-06,
"logits/chosen": -0.9853806495666504,
"logits/rejected": -1.1351251602172852,
"logps/chosen": -0.8706343770027161,
"logps/rejected": -1.4279866218566895,
"loss": 0.9807,
"odds_ratio_loss": 0.4323902130126953,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.043531715869903564,
"rewards/margins": 0.02786761149764061,
"rewards/rejected": -0.07139933109283447,
"sft_loss": 0.8706343770027161,
"step": 555
},
{
"epoch": 1.792,
"grad_norm": 4.3083896592948845,
"learning_rate": 2.082219758453629e-06,
"logits/chosen": -0.8532622456550598,
"logits/rejected": -1.0556373596191406,
"logps/chosen": -0.9265901446342468,
"logps/rejected": -1.6611661911010742,
"loss": 0.9808,
"odds_ratio_loss": 0.47050419449806213,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.04632951319217682,
"rewards/margins": 0.03672880306839943,
"rewards/rejected": -0.08305831253528595,
"sft_loss": 0.9265901446342468,
"step": 560
},
{
"epoch": 1.808,
"grad_norm": 4.988596811372773,
"learning_rate": 2.036312117578617e-06,
"logits/chosen": -0.8543407320976257,
"logits/rejected": -1.035388708114624,
"logps/chosen": -0.9570035934448242,
"logps/rejected": -1.4836640357971191,
"loss": 0.9573,
"odds_ratio_loss": 0.4832231104373932,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04785018041729927,
"rewards/margins": 0.02633301541209221,
"rewards/rejected": -0.07418319582939148,
"sft_loss": 0.9570035934448242,
"step": 565
},
{
"epoch": 1.8239999999999998,
"grad_norm": 3.4762921171914463,
"learning_rate": 1.990565848904431e-06,
"logits/chosen": -0.8208405375480652,
"logits/rejected": -0.940590500831604,
"logps/chosen": -0.8431119918823242,
"logps/rejected": -1.3455002307891846,
"loss": 0.9561,
"odds_ratio_loss": 0.4739263653755188,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04215560108423233,
"rewards/margins": 0.025119412690401077,
"rewards/rejected": -0.06727501004934311,
"sft_loss": 0.8431119918823242,
"step": 570
},
{
"epoch": 1.8399999999999999,
"grad_norm": 4.955237689773966,
"learning_rate": 1.9449968730025055e-06,
"logits/chosen": -0.9368654489517212,
"logits/rejected": -1.018002986907959,
"logps/chosen": -1.0393065214157104,
"logps/rejected": -1.3489172458648682,
"loss": 0.9946,
"odds_ratio_loss": 0.5891560316085815,
"rewards/accuracies": 0.574999988079071,
"rewards/chosen": -0.05196532607078552,
"rewards/margins": 0.01548053603619337,
"rewards/rejected": -0.06744585931301117,
"sft_loss": 1.0393065214157104,
"step": 575
},
{
"epoch": 1.8559999999999999,
"grad_norm": 4.122518133723736,
"learning_rate": 1.899621048743019e-06,
"logits/chosen": -0.8472858667373657,
"logits/rejected": -0.9123756289482117,
"logps/chosen": -0.8696669340133667,
"logps/rejected": -1.3730722665786743,
"loss": 0.9332,
"odds_ratio_loss": 0.504239559173584,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.043483346700668335,
"rewards/margins": 0.025170262902975082,
"rewards/rejected": -0.06865362077951431,
"sft_loss": 0.8696669340133667,
"step": 580
},
{
"epoch": 1.8719999999999999,
"grad_norm": 5.090413419223807,
"learning_rate": 1.854454167775706e-06,
"logits/chosen": -0.8803795576095581,
"logits/rejected": -1.011498212814331,
"logps/chosen": -0.8195604085922241,
"logps/rejected": -1.5066629648208618,
"loss": 0.9053,
"odds_ratio_loss": 0.4221072793006897,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.040978025645017624,
"rewards/margins": 0.03435512259602547,
"rewards/rejected": -0.07533314824104309,
"sft_loss": 0.8195604085922241,
"step": 585
},
{
"epoch": 1.888,
"grad_norm": 3.962979618406243,
"learning_rate": 1.8095119490340618e-06,
"logits/chosen": -0.9934213757514954,
"logits/rejected": -1.028438925743103,
"logps/chosen": -0.9060677289962769,
"logps/rejected": -1.4857866764068604,
"loss": 0.9753,
"odds_ratio_loss": 0.6394127011299133,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.04530338943004608,
"rewards/margins": 0.028985943645238876,
"rewards/rejected": -0.07428933680057526,
"sft_loss": 0.9060677289962769,
"step": 590
},
{
"epoch": 1.904,
"grad_norm": 4.384891130460857,
"learning_rate": 1.764810033264852e-06,
"logits/chosen": -0.907455563545227,
"logits/rejected": -1.0589368343353271,
"logps/chosen": -1.0206191539764404,
"logps/rejected": -1.4559767246246338,
"loss": 0.9771,
"odds_ratio_loss": 0.5931872129440308,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.0510309636592865,
"rewards/margins": 0.02176786959171295,
"rewards/rejected": -0.07279883325099945,
"sft_loss": 1.0206191539764404,
"step": 595
},
{
"epoch": 1.92,
"grad_norm": 3.4134689622305383,
"learning_rate": 1.7203639775848423e-06,
"logits/chosen": -1.0145207643508911,
"logits/rejected": -0.9972650408744812,
"logps/chosen": -0.956381618976593,
"logps/rejected": -1.3823316097259521,
"loss": 0.9631,
"odds_ratio_loss": 0.5243911743164062,
"rewards/accuracies": 0.675000011920929,
"rewards/chosen": -0.04781908541917801,
"rewards/margins": 0.021297505125403404,
"rewards/rejected": -0.06911659240722656,
"sft_loss": 0.956381618976593,
"step": 600
},
{
"epoch": 1.936,
"grad_norm": 4.505148742615646,
"learning_rate": 1.6761892500666195e-06,
"logits/chosen": -1.023581624031067,
"logits/rejected": -1.027942419052124,
"logps/chosen": -0.933574378490448,
"logps/rejected": -1.2963403463363647,
"loss": 0.9786,
"odds_ratio_loss": 0.5001336336135864,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.04667872190475464,
"rewards/margins": 0.018138296902179718,
"rewards/rejected": -0.06481701880693436,
"sft_loss": 0.933574378490448,
"step": 605
},
{
"epoch": 1.952,
"grad_norm": 4.83889942986056,
"learning_rate": 1.632301224355411e-06,
"logits/chosen": -0.8887473940849304,
"logits/rejected": -0.9829289317131042,
"logps/chosen": -1.0025461912155151,
"logps/rejected": -1.2573713064193726,
"loss": 0.9546,
"odds_ratio_loss": 0.6103852987289429,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.0501273088157177,
"rewards/margins": 0.012741250917315483,
"rewards/rejected": -0.06286855787038803,
"sft_loss": 1.0025461912155151,
"step": 610
},
{
"epoch": 1.968,
"grad_norm": 4.517273923883564,
"learning_rate": 1.5887151743187634e-06,
"logits/chosen": -0.9073827862739563,
"logits/rejected": -1.0887978076934814,
"logps/chosen": -0.9604595899581909,
"logps/rejected": -1.4289729595184326,
"loss": 0.9811,
"odds_ratio_loss": 0.545965850353241,
"rewards/accuracies": 0.699999988079071,
"rewards/chosen": -0.048022981733083725,
"rewards/margins": 0.023425664752721786,
"rewards/rejected": -0.07144864648580551,
"sft_loss": 0.9604595899581909,
"step": 615
},
{
"epoch": 1.984,
"grad_norm": 3.1757477627169908,
"learning_rate": 1.5454462687309445e-06,
"logits/chosen": -1.015500783920288,
"logits/rejected": -1.0785764455795288,
"logps/chosen": -0.9297005534172058,
"logps/rejected": -1.5386453866958618,
"loss": 0.9563,
"odds_ratio_loss": 0.542974054813385,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.04648502543568611,
"rewards/margins": 0.03044723905622959,
"rewards/rejected": -0.07693226635456085,
"sft_loss": 0.9297005534172058,
"step": 620
},
{
"epoch": 2.0,
"grad_norm": 4.242978905390128,
"learning_rate": 1.502509565993925e-06,
"logits/chosen": -0.9173771142959595,
"logits/rejected": -1.0122209787368774,
"logps/chosen": -0.9267832040786743,
"logps/rejected": -1.5153340101242065,
"loss": 0.8898,
"odds_ratio_loss": 0.45579010248184204,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.04633915796875954,
"rewards/margins": 0.02942754328250885,
"rewards/rejected": -0.07576669752597809,
"sft_loss": 0.9267832040786743,
"step": 625
},
{
"epoch": 2.016,
"grad_norm": 3.553798975041404,
"learning_rate": 1.4599200088967652e-06,
"logits/chosen": -0.8619723320007324,
"logits/rejected": -1.0356481075286865,
"logps/chosen": -0.8099665641784668,
"logps/rejected": -1.3428175449371338,
"loss": 0.8054,
"odds_ratio_loss": 0.42676934599876404,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.04049833118915558,
"rewards/margins": 0.02664254978299141,
"rewards/rejected": -0.06714087724685669,
"sft_loss": 0.8099665641784668,
"step": 630
},
{
"epoch": 2.032,
"grad_norm": 4.569902596370669,
"learning_rate": 1.4176924194152405e-06,
"logits/chosen": -0.9947516322135925,
"logits/rejected": -1.0760139226913452,
"logps/chosen": -0.822666347026825,
"logps/rejected": -1.4580928087234497,
"loss": 0.8194,
"odds_ratio_loss": 0.408845990896225,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.041133321821689606,
"rewards/margins": 0.03177132084965706,
"rewards/rejected": -0.07290463894605637,
"sft_loss": 0.822666347026825,
"step": 635
},
{
"epoch": 2.048,
"grad_norm": 3.58850101082042,
"learning_rate": 1.3758414935535147e-06,
"logits/chosen": -0.8080762624740601,
"logits/rejected": -0.8731328845024109,
"logps/chosen": -0.6912453174591064,
"logps/rejected": -1.2947412729263306,
"loss": 0.7991,
"odds_ratio_loss": 0.39434918761253357,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.03456226736307144,
"rewards/margins": 0.030174797400832176,
"rewards/rejected": -0.06473706662654877,
"sft_loss": 0.6912453174591064,
"step": 640
},
{
"epoch": 2.064,
"grad_norm": 3.1095303302424204,
"learning_rate": 1.3343817962296485e-06,
"logits/chosen": -0.9426826238632202,
"logits/rejected": -0.9611290097236633,
"logps/chosen": -0.7455964088439941,
"logps/rejected": -1.3733656406402588,
"loss": 0.8219,
"odds_ratio_loss": 0.400754451751709,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.037279821932315826,
"rewards/margins": 0.03138846904039383,
"rewards/rejected": -0.06866829097270966,
"sft_loss": 0.7455964088439941,
"step": 645
},
{
"epoch": 2.08,
"grad_norm": 3.045256483819498,
"learning_rate": 1.293327756206729e-06,
"logits/chosen": -0.8983833193778992,
"logits/rejected": -0.9661597013473511,
"logps/chosen": -0.6603522300720215,
"logps/rejected": -1.5160930156707764,
"loss": 0.7579,
"odds_ratio_loss": 0.3125734031200409,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.033017612993717194,
"rewards/margins": 0.042787034064531326,
"rewards/rejected": -0.07580464333295822,
"sft_loss": 0.6603522300720215,
"step": 650
},
{
"epoch": 2.096,
"grad_norm": 3.1717779800652592,
"learning_rate": 1.252693661071391e-06,
"logits/chosen": -1.157251238822937,
"logits/rejected": -1.0623748302459717,
"logps/chosen": -0.7086284160614014,
"logps/rejected": -1.514593243598938,
"loss": 0.7865,
"odds_ratio_loss": 0.33622443675994873,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.035431426018476486,
"rewards/margins": 0.04029824212193489,
"rewards/rejected": -0.07572966068983078,
"sft_loss": 0.7086284160614014,
"step": 655
},
{
"epoch": 2.112,
"grad_norm": 2.7205852811983093,
"learning_rate": 1.2124936522614622e-06,
"logits/chosen": -1.0289185047149658,
"logits/rejected": -1.098934292793274,
"logps/chosen": -0.7381170988082886,
"logps/rejected": -1.4106049537658691,
"loss": 0.7903,
"odds_ratio_loss": 0.45428943634033203,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.03690585121512413,
"rewards/margins": 0.033624399453401566,
"rewards/rejected": -0.0705302506685257,
"sft_loss": 0.7381170988082886,
"step": 660
},
{
"epoch": 2.128,
"grad_norm": 2.8528967143088666,
"learning_rate": 1.1727417201444735e-06,
"logits/chosen": -1.0716520547866821,
"logits/rejected": -1.0720608234405518,
"logps/chosen": -0.7356234788894653,
"logps/rejected": -1.2771320343017578,
"loss": 0.797,
"odds_ratio_loss": 0.41821280121803284,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.036781176924705505,
"rewards/margins": 0.027075421065092087,
"rewards/rejected": -0.0638565942645073,
"sft_loss": 0.7356234788894653,
"step": 665
},
{
"epoch": 2.144,
"grad_norm": 3.7706399394026615,
"learning_rate": 1.1334516991487473e-06,
"logits/chosen": -0.7660879492759705,
"logits/rejected": -1.0078188180923462,
"logps/chosen": -0.7483173608779907,
"logps/rejected": -1.2434841394424438,
"loss": 0.7863,
"odds_ratio_loss": 0.3927895426750183,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.037415869534015656,
"rewards/margins": 0.02475833147764206,
"rewards/rejected": -0.062174201011657715,
"sft_loss": 0.7483173608779907,
"step": 670
},
{
"epoch": 2.16,
"grad_norm": 3.5703943571070482,
"learning_rate": 1.094637262948749e-06,
"logits/chosen": -0.9008601307868958,
"logits/rejected": -1.0412832498550415,
"logps/chosen": -0.821381688117981,
"logps/rejected": -1.3886059522628784,
"loss": 0.7675,
"odds_ratio_loss": 0.4320302903652191,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04106908664107323,
"rewards/margins": 0.02836121991276741,
"rewards/rejected": -0.06943030655384064,
"sft_loss": 0.821381688117981,
"step": 675
},
{
"epoch": 2.176,
"grad_norm": 2.9837572151294065,
"learning_rate": 1.0563119197063934e-06,
"logits/chosen": -1.058257818222046,
"logits/rejected": -1.0895675420761108,
"logps/chosen": -0.914572536945343,
"logps/rejected": -1.3927881717681885,
"loss": 0.7969,
"odds_ratio_loss": 0.46075814962387085,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.04572862759232521,
"rewards/margins": 0.023910781368613243,
"rewards/rejected": -0.0696394145488739,
"sft_loss": 0.914572536945343,
"step": 680
},
{
"epoch": 2.192,
"grad_norm": 3.2497337072742414,
"learning_rate": 1.018489007369945e-06,
"logits/chosen": -0.8853160738945007,
"logits/rejected": -0.8999375104904175,
"logps/chosen": -0.7777100801467896,
"logps/rejected": -1.448999047279358,
"loss": 0.804,
"odds_ratio_loss": 0.39251843094825745,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.03888550028204918,
"rewards/margins": 0.03356444090604782,
"rewards/rejected": -0.0724499449133873,
"sft_loss": 0.7777100801467896,
"step": 685
},
{
"epoch": 2.208,
"grad_norm": 3.029373367507165,
"learning_rate": 9.81181689032158e-07,
"logits/chosen": -0.8918782472610474,
"logits/rejected": -0.8753920793533325,
"logps/chosen": -0.6909304857254028,
"logps/rejected": -1.2901936769485474,
"loss": 0.7751,
"odds_ratio_loss": 0.4143063426017761,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.03454653173685074,
"rewards/margins": 0.029963159933686256,
"rewards/rejected": -0.06450968980789185,
"sft_loss": 0.6909304857254028,
"step": 690
},
{
"epoch": 2.224,
"grad_norm": 4.5746404005970405,
"learning_rate": 9.444029483492703e-07,
"logits/chosen": -1.0063061714172363,
"logits/rejected": -1.0239533185958862,
"logps/chosen": -0.6867491006851196,
"logps/rejected": -1.245091199874878,
"loss": 0.7732,
"odds_ratio_loss": 0.4617982804775238,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.03433745354413986,
"rewards/margins": 0.027917101979255676,
"rewards/rejected": -0.06225455552339554,
"sft_loss": 0.6867491006851196,
"step": 695
},
{
"epoch": 2.24,
"grad_norm": 3.694255891644154,
"learning_rate": 9.081655850224449e-07,
"logits/chosen": -1.0207760334014893,
"logits/rejected": -1.1230969429016113,
"logps/chosen": -0.8350968360900879,
"logps/rejected": -1.5500547885894775,
"loss": 0.7583,
"odds_ratio_loss": 0.42910847067832947,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.041754838079214096,
"rewards/margins": 0.03574790060520172,
"rewards/rejected": -0.07750274240970612,
"sft_loss": 0.8350968360900879,
"step": 700
},
{
"epoch": 2.2560000000000002,
"grad_norm": 3.994487822092531,
"learning_rate": 8.724822103432282e-07,
"logits/chosen": -0.8827289342880249,
"logits/rejected": -0.9654695391654968,
"logps/chosen": -0.7088466286659241,
"logps/rejected": -1.281082034111023,
"loss": 0.7482,
"odds_ratio_loss": 0.3990407884120941,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.035442329943180084,
"rewards/margins": 0.028611773625016212,
"rewards/rejected": -0.06405410915613174,
"sft_loss": 0.7088466286659241,
"step": 705
},
{
"epoch": 2.2720000000000002,
"grad_norm": 3.3050191320581166,
"learning_rate": 8.373652428045831e-07,
"logits/chosen": -1.0878264904022217,
"logits/rejected": -1.0405784845352173,
"logps/chosen": -0.65968257188797,
"logps/rejected": -1.3744075298309326,
"loss": 0.7722,
"odds_ratio_loss": 0.3181590139865875,
"rewards/accuracies": 0.949999988079071,
"rewards/chosen": -0.03298413008451462,
"rewards/margins": 0.03573625162243843,
"rewards/rejected": -0.06872038543224335,
"sft_loss": 0.65968257188797,
"step": 710
},
{
"epoch": 2.288,
"grad_norm": 3.8117555833594943,
"learning_rate": 8.028269037790171e-07,
"logits/chosen": -0.8991669416427612,
"logits/rejected": -0.9795140027999878,
"logps/chosen": -0.8547693490982056,
"logps/rejected": -1.5010745525360107,
"loss": 0.7688,
"odds_ratio_loss": 0.42176875472068787,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.04273846372961998,
"rewards/margins": 0.03231526166200638,
"rewards/rejected": -0.07505372911691666,
"sft_loss": 0.8547693490982056,
"step": 715
},
{
"epoch": 2.304,
"grad_norm": 6.060881007391729,
"learning_rate": 7.688792132653111e-07,
"logits/chosen": -0.8518077731132507,
"logits/rejected": -0.9652704000473022,
"logps/chosen": -0.7316969633102417,
"logps/rejected": -1.5251853466033936,
"loss": 0.7621,
"odds_ratio_loss": 0.3367399573326111,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.03658485412597656,
"rewards/margins": 0.03967442363500595,
"rewards/rejected": -0.07625927031040192,
"sft_loss": 0.7316969633102417,
"step": 720
},
{
"epoch": 2.32,
"grad_norm": 2.9281573250934363,
"learning_rate": 7.355339857053318e-07,
"logits/chosen": -0.8477774858474731,
"logits/rejected": -1.018747329711914,
"logps/chosen": -0.7693859934806824,
"logps/rejected": -1.3998419046401978,
"loss": 0.7714,
"odds_ratio_loss": 0.3711521625518799,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.03846929594874382,
"rewards/margins": 0.031522803008556366,
"rewards/rejected": -0.06999210268259048,
"sft_loss": 0.7693859934806824,
"step": 725
},
{
"epoch": 2.336,
"grad_norm": 3.3575685063670213,
"learning_rate": 7.028028258723818e-07,
"logits/chosen": -0.9588859677314758,
"logits/rejected": -1.0327568054199219,
"logps/chosen": -0.7077163457870483,
"logps/rejected": -1.3171709775924683,
"loss": 0.7932,
"odds_ratio_loss": 0.3643357455730438,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.03538581356406212,
"rewards/margins": 0.030472736805677414,
"rewards/rejected": -0.06585855782032013,
"sft_loss": 0.7077163457870483,
"step": 730
},
{
"epoch": 2.352,
"grad_norm": 3.07955633424655,
"learning_rate": 6.706971248325151e-07,
"logits/chosen": -0.8883659243583679,
"logits/rejected": -1.0441168546676636,
"logps/chosen": -0.8074674606323242,
"logps/rejected": -1.4699620008468628,
"loss": 0.776,
"odds_ratio_loss": 0.39879804849624634,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.04037337377667427,
"rewards/margins": 0.033124733716249466,
"rewards/rejected": -0.07349809259176254,
"sft_loss": 0.8074674606323242,
"step": 735
},
{
"epoch": 2.368,
"grad_norm": 3.6213614434043593,
"learning_rate": 6.392280559802341e-07,
"logits/chosen": -0.9162646532058716,
"logits/rejected": -0.9279802441596985,
"logps/chosen": -0.7218812704086304,
"logps/rejected": -1.4897241592407227,
"loss": 0.7637,
"odds_ratio_loss": 0.411416620016098,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.0360940620303154,
"rewards/margins": 0.03839214891195297,
"rewards/rejected": -0.07448621094226837,
"sft_loss": 0.7218812704086304,
"step": 740
},
{
"epoch": 2.384,
"grad_norm": 4.1017905530653245,
"learning_rate": 6.084065711499326e-07,
"logits/chosen": -0.84368896484375,
"logits/rejected": -0.9179951548576355,
"logps/chosen": -0.6008902192115784,
"logps/rejected": -1.4554692506790161,
"loss": 0.7279,
"odds_ratio_loss": 0.3157138526439667,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.030044514685869217,
"rewards/margins": 0.04272894933819771,
"rewards/rejected": -0.07277346402406693,
"sft_loss": 0.6008902192115784,
"step": 745
},
{
"epoch": 2.4,
"grad_norm": 3.8487814271047758,
"learning_rate": 5.782433968044495e-07,
"logits/chosen": -0.8399477005004883,
"logits/rejected": -0.9413140416145325,
"logps/chosen": -0.8139437437057495,
"logps/rejected": -1.5214980840682983,
"loss": 0.7662,
"odds_ratio_loss": 0.5000792741775513,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.040697190910577774,
"rewards/margins": 0.03537772223353386,
"rewards/rejected": -0.07607491314411163,
"sft_loss": 0.8139437437057495,
"step": 750
},
{
"epoch": 2.416,
"grad_norm": 3.305488495552487,
"learning_rate": 5.487490303020576e-07,
"logits/chosen": -0.9991554021835327,
"logits/rejected": -0.944214940071106,
"logps/chosen": -0.7551851272583008,
"logps/rejected": -1.2736141681671143,
"loss": 0.7806,
"odds_ratio_loss": 0.4188924729824066,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.03775925561785698,
"rewards/margins": 0.025921454653143883,
"rewards/rejected": -0.06368071585893631,
"sft_loss": 0.7551851272583008,
"step": 755
},
{
"epoch": 2.432,
"grad_norm": 3.6380522040854566,
"learning_rate": 5.199337362431792e-07,
"logits/chosen": -0.8715981245040894,
"logits/rejected": -1.0095371007919312,
"logps/chosen": -0.716741681098938,
"logps/rejected": -1.7061433792114258,
"loss": 0.7998,
"odds_ratio_loss": 0.40209442377090454,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.0358370803296566,
"rewards/margins": 0.04947008565068245,
"rewards/rejected": -0.08530716598033905,
"sft_loss": 0.716741681098938,
"step": 760
},
{
"epoch": 2.448,
"grad_norm": 4.799248860716653,
"learning_rate": 4.918075428981148e-07,
"logits/chosen": -0.9698484539985657,
"logits/rejected": -1.0252515077590942,
"logps/chosen": -0.7276674509048462,
"logps/rejected": -1.2364145517349243,
"loss": 0.7519,
"odds_ratio_loss": 0.4512792229652405,
"rewards/accuracies": 0.7250000238418579,
"rewards/chosen": -0.03638336807489395,
"rewards/margins": 0.025437360629439354,
"rewards/rejected": -0.06182073429226875,
"sft_loss": 0.7276674509048462,
"step": 765
},
{
"epoch": 2.464,
"grad_norm": 3.975888652914653,
"learning_rate": 4.643802387170118e-07,
"logits/chosen": -0.784801185131073,
"logits/rejected": -0.8751401901245117,
"logps/chosen": -0.736864447593689,
"logps/rejected": -1.4113752841949463,
"loss": 0.7689,
"odds_ratio_loss": 0.41703367233276367,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.03684321790933609,
"rewards/margins": 0.0337255522608757,
"rewards/rejected": -0.07056877762079239,
"sft_loss": 0.736864447593689,
"step": 770
},
{
"epoch": 2.48,
"grad_norm": 3.7017186080610873,
"learning_rate": 4.376613689232978e-07,
"logits/chosen": -0.9608728289604187,
"logits/rejected": -1.000018835067749,
"logps/chosen": -0.8199416399002075,
"logps/rejected": -1.469847321510315,
"loss": 0.7588,
"odds_ratio_loss": 0.43195098638534546,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.04099708050489426,
"rewards/margins": 0.03249528631567955,
"rewards/rejected": -0.0734923705458641,
"sft_loss": 0.8199416399002075,
"step": 775
},
{
"epoch": 2.496,
"grad_norm": 3.1981116471330764,
"learning_rate": 4.1166023219176176e-07,
"logits/chosen": -0.8377124667167664,
"logits/rejected": -1.0116181373596191,
"logps/chosen": -0.7824350595474243,
"logps/rejected": -1.6547685861587524,
"loss": 0.7523,
"odds_ratio_loss": 0.32834020256996155,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.039121754467487335,
"rewards/margins": 0.043616678565740585,
"rewards/rejected": -0.08273844420909882,
"sft_loss": 0.7824350595474243,
"step": 780
},
{
"epoch": 2.512,
"grad_norm": 3.7082914287151945,
"learning_rate": 3.863858774124385e-07,
"logits/chosen": -1.0026713609695435,
"logits/rejected": -1.0367608070373535,
"logps/chosen": -0.6959080696105957,
"logps/rejected": -1.4358288049697876,
"loss": 0.7563,
"odds_ratio_loss": 0.3291171193122864,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.03479539602994919,
"rewards/margins": 0.036996036767959595,
"rewards/rejected": -0.07179144024848938,
"sft_loss": 0.6959080696105957,
"step": 785
},
{
"epoch": 2.528,
"grad_norm": 2.9819183882511004,
"learning_rate": 3.618471005414215e-07,
"logits/chosen": -1.1593316793441772,
"logits/rejected": -1.1186132431030273,
"logps/chosen": -0.5521804094314575,
"logps/rejected": -1.3308180570602417,
"loss": 0.7382,
"odds_ratio_loss": 0.40159231424331665,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.027609020471572876,
"rewards/margins": 0.03893188014626503,
"rewards/rejected": -0.0665409117937088,
"sft_loss": 0.5521804094314575,
"step": 790
},
{
"epoch": 2.544,
"grad_norm": 3.3567110818810755,
"learning_rate": 3.380524415397049e-07,
"logits/chosen": -0.8402732014656067,
"logits/rejected": -1.0374484062194824,
"logps/chosen": -0.803017795085907,
"logps/rejected": -1.2986352443695068,
"loss": 0.7742,
"odds_ratio_loss": 0.46145597100257874,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.04015089198946953,
"rewards/margins": 0.0247808750718832,
"rewards/rejected": -0.06493176519870758,
"sft_loss": 0.803017795085907,
"step": 795
},
{
"epoch": 2.56,
"grad_norm": 3.3420577148113924,
"learning_rate": 3.150101814011136e-07,
"logits/chosen": -1.031750202178955,
"logits/rejected": -1.02663254737854,
"logps/chosen": -0.7611913681030273,
"logps/rejected": -1.1853117942810059,
"loss": 0.7785,
"odds_ratio_loss": 0.5039734244346619,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.03805956989526749,
"rewards/margins": 0.021206015720963478,
"rewards/rejected": -0.05926559120416641,
"sft_loss": 0.7611913681030273,
"step": 800
},
{
"epoch": 2.576,
"grad_norm": 3.246898163699306,
"learning_rate": 2.927283392703606e-07,
"logits/chosen": -0.8511770367622375,
"logits/rejected": -0.9470258951187134,
"logps/chosen": -0.7670743465423584,
"logps/rejected": -1.1585172414779663,
"loss": 0.7977,
"odds_ratio_loss": 0.4781486988067627,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.03835371881723404,
"rewards/margins": 0.019572149962186813,
"rewards/rejected": -0.057925861328840256,
"sft_loss": 0.7670743465423584,
"step": 805
},
{
"epoch": 2.592,
"grad_norm": 3.9992460548702438,
"learning_rate": 2.712146696522305e-07,
"logits/chosen": -1.022859811782837,
"logits/rejected": -1.129640817642212,
"logps/chosen": -0.716964840888977,
"logps/rejected": -1.246201992034912,
"loss": 0.7552,
"odds_ratio_loss": 0.4771757125854492,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.03584824129939079,
"rewards/margins": 0.026461860164999962,
"rewards/rejected": -0.062310099601745605,
"sft_loss": 0.716964840888977,
"step": 810
},
{
"epoch": 2.608,
"grad_norm": 3.5945402118846403,
"learning_rate": 2.504766597128666e-07,
"logits/chosen": -0.9580585360527039,
"logits/rejected": -0.9554399251937866,
"logps/chosen": -0.8108547925949097,
"logps/rejected": -1.315863847732544,
"loss": 0.7575,
"odds_ratio_loss": 0.44800853729248047,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.040542736649513245,
"rewards/margins": 0.025250453501939774,
"rewards/rejected": -0.06579320132732391,
"sft_loss": 0.8108547925949097,
"step": 815
},
{
"epoch": 2.624,
"grad_norm": 3.288445101598415,
"learning_rate": 2.3052152667409289e-07,
"logits/chosen": -0.9488247036933899,
"logits/rejected": -0.8991857767105103,
"logps/chosen": -0.7179723978042603,
"logps/rejected": -1.3861563205718994,
"loss": 0.7752,
"odds_ratio_loss": 0.3779214322566986,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.03589861840009689,
"rewards/margins": 0.03340919688344002,
"rewards/rejected": -0.06930781900882721,
"sft_loss": 0.7179723978042603,
"step": 820
},
{
"epoch": 2.64,
"grad_norm": 4.333903778456222,
"learning_rate": 2.1135621530168488e-07,
"logits/chosen": -1.028704285621643,
"logits/rejected": -0.9637089967727661,
"logps/chosen": -0.6309131383895874,
"logps/rejected": -1.2894703149795532,
"loss": 0.7935,
"odds_ratio_loss": 0.3709859549999237,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.03154565393924713,
"rewards/margins": 0.03292786702513695,
"rewards/rejected": -0.06447352468967438,
"sft_loss": 0.6309131383895874,
"step": 825
},
{
"epoch": 2.656,
"grad_norm": 3.1671393797255414,
"learning_rate": 1.9298739548845813e-07,
"logits/chosen": -0.9645661115646362,
"logits/rejected": -0.9816401600837708,
"logps/chosen": -0.8160637021064758,
"logps/rejected": -1.374236822128296,
"loss": 0.8091,
"odds_ratio_loss": 0.4034562110900879,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.04080318659543991,
"rewards/margins": 0.027908671647310257,
"rewards/rejected": -0.06871185451745987,
"sft_loss": 0.8160637021064758,
"step": 830
},
{
"epoch": 2.672,
"grad_norm": 3.093675577889326,
"learning_rate": 1.7542145993301896e-07,
"logits/chosen": -0.9770193099975586,
"logits/rejected": -1.0430421829223633,
"logps/chosen": -0.6873968839645386,
"logps/rejected": -1.183712124824524,
"loss": 0.7636,
"odds_ratio_loss": 0.42723220586776733,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.03436984866857529,
"rewards/margins": 0.02481575682759285,
"rewards/rejected": -0.05918560549616814,
"sft_loss": 0.6873968839645386,
"step": 835
},
{
"epoch": 2.6879999999999997,
"grad_norm": 3.044985818298562,
"learning_rate": 1.5866452191498488e-07,
"logits/chosen": -0.8661419153213501,
"logits/rejected": -1.0225311517715454,
"logps/chosen": -0.7636631727218628,
"logps/rejected": -1.324355959892273,
"loss": 0.7742,
"odds_ratio_loss": 0.3950451910495758,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.03818316012620926,
"rewards/margins": 0.028034633025527,
"rewards/rejected": -0.06621779501438141,
"sft_loss": 0.7636631727218628,
"step": 840
},
{
"epoch": 2.7039999999999997,
"grad_norm": 3.49139680447691,
"learning_rate": 1.4272241316744456e-07,
"logits/chosen": -0.9768983125686646,
"logits/rejected": -1.0676887035369873,
"logps/chosen": -0.8691689372062683,
"logps/rejected": -1.5023279190063477,
"loss": 0.7386,
"odds_ratio_loss": 0.36283236742019653,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.043458450585603714,
"rewards/margins": 0.03165794909000397,
"rewards/rejected": -0.07511639595031738,
"sft_loss": 0.8691689372062683,
"step": 845
},
{
"epoch": 2.7199999999999998,
"grad_norm": 3.4065842852697674,
"learning_rate": 1.2760068184740597e-07,
"logits/chosen": -0.9277766942977905,
"logits/rejected": -0.9539043307304382,
"logps/chosen": -0.6972528100013733,
"logps/rejected": -1.4623371362686157,
"loss": 0.711,
"odds_ratio_loss": 0.32903754711151123,
"rewards/accuracies": 0.9750000238418579,
"rewards/chosen": -0.03486264497041702,
"rewards/margins": 0.03825421258807182,
"rewards/rejected": -0.07311685383319855,
"sft_loss": 0.6972528100013733,
"step": 850
},
{
"epoch": 2.7359999999999998,
"grad_norm": 4.258492525068464,
"learning_rate": 1.133045906049321e-07,
"logits/chosen": -0.9535223245620728,
"logits/rejected": -0.967192530632019,
"logps/chosen": -0.7654585242271423,
"logps/rejected": -1.2852718830108643,
"loss": 0.7165,
"odds_ratio_loss": 0.4885888695716858,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.038272932171821594,
"rewards/margins": 0.025990664958953857,
"rewards/rejected": -0.06426359713077545,
"sft_loss": 0.7654585242271423,
"step": 855
},
{
"epoch": 2.752,
"grad_norm": 3.0541533378277403,
"learning_rate": 9.983911475163727e-08,
"logits/chosen": -0.9215459823608398,
"logits/rejected": -1.025499939918518,
"logps/chosen": -0.6536693572998047,
"logps/rejected": -1.464961051940918,
"loss": 0.7491,
"odds_ratio_loss": 0.32264453172683716,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.032683465629816055,
"rewards/margins": 0.04056458920240402,
"rewards/rejected": -0.07324805110692978,
"sft_loss": 0.6536693572998047,
"step": 860
},
{
"epoch": 2.768,
"grad_norm": 3.5262579401831413,
"learning_rate": 8.720894052918566e-08,
"logits/chosen": -0.9398951530456543,
"logits/rejected": -1.0038102865219116,
"logps/chosen": -0.6747088432312012,
"logps/rejected": -1.5684466361999512,
"loss": 0.7404,
"odds_ratio_loss": 0.35855698585510254,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.03373543918132782,
"rewards/margins": 0.044686902314424515,
"rewards/rejected": -0.07842233031988144,
"sft_loss": 0.6747088432312012,
"step": 865
},
{
"epoch": 2.784,
"grad_norm": 4.66998911832725,
"learning_rate": 7.541846347838915e-08,
"logits/chosen": -0.9948604702949524,
"logits/rejected": -0.9804432988166809,
"logps/chosen": -0.7318006753921509,
"logps/rejected": -1.360406517982483,
"loss": 0.7496,
"odds_ratio_loss": 0.4134978652000427,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.036590032279491425,
"rewards/margins": 0.03143029659986496,
"rewards/rejected": -0.06802032887935638,
"sft_loss": 0.7318006753921509,
"step": 870
},
{
"epoch": 2.8,
"grad_norm": 3.657269423224737,
"learning_rate": 6.447178690947492e-08,
"logits/chosen": -0.8408433794975281,
"logits/rejected": -1.0504786968231201,
"logps/chosen": -0.6820616126060486,
"logps/rejected": -1.6608028411865234,
"loss": 0.7865,
"odds_ratio_loss": 0.2992148697376251,
"rewards/accuracies": 0.8999999761581421,
"rewards/chosen": -0.03410308063030243,
"rewards/margins": 0.04893706366419792,
"rewards/rejected": -0.08304014801979065,
"sft_loss": 0.6820616126060486,
"step": 875
},
{
"epoch": 2.816,
"grad_norm": 3.377129562618715,
"learning_rate": 5.437272047405712e-08,
"logits/chosen": -0.8684722781181335,
"logits/rejected": -0.871597945690155,
"logps/chosen": -0.7219066619873047,
"logps/rejected": -1.350375771522522,
"loss": 0.7529,
"odds_ratio_loss": 0.3972117006778717,
"rewards/accuracies": 0.925000011920929,
"rewards/chosen": -0.036095328629016876,
"rewards/margins": 0.031423456966876984,
"rewards/rejected": -0.06751878559589386,
"sft_loss": 0.7219066619873047,
"step": 880
},
{
"epoch": 2.832,
"grad_norm": 3.1663560374696633,
"learning_rate": 4.512477883930527e-08,
"logits/chosen": -0.9551021456718445,
"logits/rejected": -1.0277870893478394,
"logps/chosen": -0.7899004817008972,
"logps/rejected": -1.5054200887680054,
"loss": 0.8151,
"odds_ratio_loss": 0.44721174240112305,
"rewards/accuracies": 0.800000011920929,
"rewards/chosen": -0.03949502483010292,
"rewards/margins": 0.03577598184347153,
"rewards/rejected": -0.07527101784944534,
"sft_loss": 0.7899004817008972,
"step": 885
},
{
"epoch": 2.848,
"grad_norm": 4.553277973933315,
"learning_rate": 3.673118046477159e-08,
"logits/chosen": -0.9583977460861206,
"logits/rejected": -0.9128350019454956,
"logps/chosen": -0.7262309789657593,
"logps/rejected": -1.2458374500274658,
"loss": 0.7505,
"odds_ratio_loss": 0.4251146912574768,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.0363115519285202,
"rewards/margins": 0.025980323553085327,
"rewards/rejected": -0.06229187175631523,
"sft_loss": 0.7262309789657593,
"step": 890
},
{
"epoch": 2.864,
"grad_norm": 3.38523058441021,
"learning_rate": 2.9194846482308026e-08,
"logits/chosen": -0.9198935627937317,
"logits/rejected": -1.0665795803070068,
"logps/chosen": -0.7491655349731445,
"logps/rejected": -1.439427137374878,
"loss": 0.7622,
"odds_ratio_loss": 0.3993758261203766,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.037458278238773346,
"rewards/margins": 0.03451308235526085,
"rewards/rejected": -0.0719713643193245,
"sft_loss": 0.7491655349731445,
"step": 895
},
{
"epoch": 2.88,
"grad_norm": 5.053660983515992,
"learning_rate": 2.251839967945535e-08,
"logits/chosen": -0.9375904202461243,
"logits/rejected": -1.060706377029419,
"logps/chosen": -0.8767011761665344,
"logps/rejected": -1.344328761100769,
"loss": 0.766,
"odds_ratio_loss": 0.5356343984603882,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.04383506253361702,
"rewards/margins": 0.02338137850165367,
"rewards/rejected": -0.06721644103527069,
"sft_loss": 0.8767011761665344,
"step": 900
},
{
"epoch": 2.896,
"grad_norm": 4.052840811239487,
"learning_rate": 1.6704163586663825e-08,
"logits/chosen": -0.9922626614570618,
"logits/rejected": -0.952767014503479,
"logps/chosen": -0.7710371017456055,
"logps/rejected": -1.4237831830978394,
"loss": 0.8119,
"odds_ratio_loss": 0.37999871373176575,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.03855185955762863,
"rewards/margins": 0.032637301832437515,
"rewards/rejected": -0.07118916511535645,
"sft_loss": 0.7710371017456055,
"step": 905
},
{
"epoch": 2.912,
"grad_norm": 3.5836749421401146,
"learning_rate": 1.1754161668660612e-08,
"logits/chosen": -0.8529298901557922,
"logits/rejected": -0.905608057975769,
"logps/chosen": -0.7730112671852112,
"logps/rejected": -1.4924733638763428,
"loss": 0.7818,
"odds_ratio_loss": 0.40126991271972656,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.03865056484937668,
"rewards/margins": 0.035973113030195236,
"rewards/rejected": -0.07462368160486221,
"sft_loss": 0.7730112671852112,
"step": 910
},
{
"epoch": 2.928,
"grad_norm": 4.257912492562235,
"learning_rate": 7.670116620245304e-09,
"logits/chosen": -0.9177722930908203,
"logits/rejected": -1.0528053045272827,
"logps/chosen": -0.7789991497993469,
"logps/rejected": -1.4093081951141357,
"loss": 0.7862,
"odds_ratio_loss": 0.41386303305625916,
"rewards/accuracies": 0.824999988079071,
"rewards/chosen": -0.03894995525479317,
"rewards/margins": 0.0315154567360878,
"rewards/rejected": -0.07046540826559067,
"sft_loss": 0.7789991497993469,
"step": 915
},
{
"epoch": 2.944,
"grad_norm": 4.5782153540538575,
"learning_rate": 4.453449766758933e-09,
"logits/chosen": -1.0013253688812256,
"logits/rejected": -0.9696464538574219,
"logps/chosen": -0.7406843900680542,
"logps/rejected": -1.4851348400115967,
"loss": 0.7435,
"odds_ratio_loss": 0.3701634407043457,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.037034209817647934,
"rewards/margins": 0.03722252696752548,
"rewards/rejected": -0.07425674051046371,
"sft_loss": 0.7406843900680542,
"step": 920
},
{
"epoch": 2.96,
"grad_norm": 3.8405956220288906,
"learning_rate": 2.1052805694365964e-09,
"logits/chosen": -0.9112046360969543,
"logits/rejected": -0.953746497631073,
"logps/chosen": -0.9097617864608765,
"logps/rejected": -1.2347737550735474,
"loss": 0.787,
"odds_ratio_loss": 0.5556334853172302,
"rewards/accuracies": 0.6499999761581421,
"rewards/chosen": -0.04548809304833412,
"rewards/margins": 0.016250593587756157,
"rewards/rejected": -0.061738692224025726,
"sft_loss": 0.9097617864608765,
"step": 925
},
{
"epoch": 2.976,
"grad_norm": 3.3132646219123676,
"learning_rate": 6.264262358129936e-10,
"logits/chosen": -0.8271144032478333,
"logits/rejected": -1.076704502105713,
"logps/chosen": -0.8304189443588257,
"logps/rejected": -1.3862361907958984,
"loss": 0.7539,
"odds_ratio_loss": 0.4186479449272156,
"rewards/accuracies": 0.7749999761581421,
"rewards/chosen": -0.04152094945311546,
"rewards/margins": 0.027790868654847145,
"rewards/rejected": -0.06931181252002716,
"sft_loss": 0.8304189443588257,
"step": 930
},
{
"epoch": 2.992,
"grad_norm": 3.846388656199648,
"learning_rate": 1.7401435318531444e-11,
"logits/chosen": -1.0561763048171997,
"logits/rejected": -1.0017073154449463,
"logps/chosen": -0.6843768358230591,
"logps/rejected": -1.4440100193023682,
"loss": 0.7977,
"odds_ratio_loss": 0.4135671555995941,
"rewards/accuracies": 0.8500000238418579,
"rewards/chosen": -0.034218840301036835,
"rewards/margins": 0.03798165172338486,
"rewards/rejected": -0.07220049947500229,
"sft_loss": 0.6843768358230591,
"step": 935
},
{
"epoch": 2.9952,
"step": 936,
"total_flos": 172904605286400.0,
"train_loss": 0.9994224820636276,
"train_runtime": 29817.6595,
"train_samples_per_second": 2.012,
"train_steps_per_second": 0.031
}
],
"logging_steps": 5,
"max_steps": 936,
"num_input_tokens_seen": 0,
"num_train_epochs": 3,
"save_steps": 100.0,
"total_flos": 172904605286400.0,
"train_batch_size": 2,
"trial_name": null,
"trial_params": null
}