Model save

README.md

@@ -2,13 +2,9 @@
 license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
-- generated_from_trainer
 - trl
 - dpo
 - generated_from_trainer
-datasets:
-- HuggingFaceH4/ultrafeedback_binarized
 base_model: mistralai/Mistral-7B-v0.1
 model-index:
 - name: zephyr-7b-dpo-qlora
@@ -20,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # zephyr-7b-dpo-qlora
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-qlora](https://huggingface.co/alignment-handbook/zephyr-7b-sft-qlora) on the HuggingFaceH4/ultrafeedback_binarized dataset.
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2121.0452
-- Rewards/chosen: 0.0578
-- Rewards/rejected: -0.0912
-- Rewards/accuracies: 0.7599
-- Rewards/margins: 0.1490
-- Logps/rejected: -253.8891
-- Logps/chosen: -259.2458
-- Logits/rejected: -2.2028
-- Logits/chosen: -2.2552
+- Loss: 1721.1201
+- Rewards/chosen: -0.0627
+- Rewards/rejected: -0.2250
+- Rewards/accuracies: 0.7738
+- Rewards/margins: 0.1623
+- Logps/rejected: -267.2721
+- Logps/chosen: -271.2979
+- Logits/rejected: -2.0354
+- Logits/chosen: -2.0918
 
 ## Model description
 
@@ -67,10 +63,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 2149.4746 | 0.21 | 100 | 2190.7666 | 0.0445 | -0.0848 | 0.7460 | 0.1293 | -253.2523 | -260.5782 | -2.1770 | -2.2229 |
-| 2105.1256 | 0.42 | 200 | 2151.1555 | 0.0543 | -0.0961 | 0.7599 | 0.1504 | -254.3840 | -259.5941 | -2.2074 | -2.2603 |
-| 2135.4973 | 0.63 | 300 | 2129.0896 | 0.0626 | -0.0799 | 0.7560 | 0.1425 | -252.7585 | -258.7624 | -2.2232 | -2.2765 |
-| 2099.8018 | 0.84 | 400 | 2121.6672 | 0.0538 | -0.0959 | 0.7540 | 0.1497 | -254.3591 | -259.6440 | -2.2016 | -2.2541 |
+| 1797.9404 | 0.21 | 100 | 1887.4103 | 0.0131 | -0.1197 | 0.7520 | 0.1328 | -256.7424 | -263.7133 | -2.1486 | -2.1969 |
+| 1700.9055 | 0.42 | 200 | 1784.6598 | -0.0464 | -0.2062 | 0.7619 | 0.1598 | -265.3905 | -269.6655 | -2.1081 | -2.1618 |
+| 1767.2219 | 0.63 | 300 | 1735.5183 | -0.0467 | -0.2001 | 0.7698 | 0.1534 | -264.7795 | -269.6956 | -2.1057 | -2.1587 |
+| 1717.4336 | 0.84 | 400 | 1721.6765 | -0.0691 | -0.2309 | 0.7718 | 0.1618 | -267.8569 | -271.9333 | -2.0322 | -2.0885 |
 
 
 ### Framework versions

adapter_config.json

@@ -19,13 +19,13 @@
  "rank_pattern": {},
  "revision": null,
  "target_modules": [
- "up_proj",
- "v_proj",
  "q_proj",
  "gate_proj",
+ "up_proj",
+ "k_proj",
  "down_proj",
  "o_proj",
- "k_proj"
+ "v_proj"
  ],
  "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c6f7d1a95415a6c03799926b1b3b0647c3602207bcc4fb5c48fa957c5b2fea04
+oid sha256:58513bf1529e315eda3b88d4c9cacb2897ba3fd8a6c935b6b16975253aa6b856
 size 671150064

all_results.json

@@ -1,21 +1,21 @@
 {
  "epoch": 1.0,
- "eval_logits/chosen": -2.255185127258301,
- "eval_logits/rejected": -2.2027812004089355,
- "eval_logps/chosen": -259.245849609375,
- "eval_logps/rejected": -253.8891143798828,
- "eval_loss": 2121.045166015625,
- "eval_rewards/accuracies": 0.7599206566810608,
- "eval_rewards/chosen": 0.057787273079156876,
- "eval_rewards/margins": 0.14896924793720245,
- "eval_rewards/rejected": -0.09118196368217468,
- "eval_runtime": 547.839,
+ "eval_logits/chosen": -2.0918362140655518,
+ "eval_logits/rejected": -2.03544020652771,
+ "eval_logps/chosen": -271.2979431152344,
+ "eval_logps/rejected": -267.2720642089844,
+ "eval_loss": 1721.1201171875,
+ "eval_rewards/accuracies": 0.773809552192688,
+ "eval_rewards/chosen": -0.06273359060287476,
+ "eval_rewards/margins": 0.16227789223194122,
+ "eval_rewards/rejected": -0.22501146793365479,
+ "eval_runtime": 548.8776,
  "eval_samples": 2000,
- "eval_samples_per_second": 3.651,
+ "eval_samples_per_second": 3.644,
  "eval_steps_per_second": 0.115,
- "train_loss": 2164.5614415454666,
- "train_runtime": 32346.8016,
+ "train_loss": 1826.8015694608227,
+ "train_runtime": 32379.7062,
  "train_samples": 61135,
- "train_samples_per_second": 1.89,
+ "train_samples_per_second": 1.888,
  "train_steps_per_second": 0.015
 }

eval_results.json

@@ -1,16 +1,16 @@
 {
  "epoch": 1.0,
- "eval_logits/chosen": -2.255185127258301,
- "eval_logits/rejected": -2.2027812004089355,
- "eval_logps/chosen": -259.245849609375,
- "eval_logps/rejected": -253.8891143798828,
- "eval_loss": 2121.045166015625,
- "eval_rewards/accuracies": 0.7599206566810608,
- "eval_rewards/chosen": 0.057787273079156876,
- "eval_rewards/margins": 0.14896924793720245,
- "eval_rewards/rejected": -0.09118196368217468,
- "eval_runtime": 547.839,
+ "eval_logits/chosen": -2.0918362140655518,
+ "eval_logits/rejected": -2.03544020652771,
+ "eval_logps/chosen": -271.2979431152344,
+ "eval_logps/rejected": -267.2720642089844,
+ "eval_loss": 1721.1201171875,
+ "eval_rewards/accuracies": 0.773809552192688,
+ "eval_rewards/chosen": -0.06273359060287476,
+ "eval_rewards/margins": 0.16227789223194122,
+ "eval_rewards/rejected": -0.22501146793365479,
+ "eval_runtime": 548.8776,
  "eval_samples": 2000,
- "eval_samples_per_second": 3.651,
+ "eval_samples_per_second": 3.644,
  "eval_steps_per_second": 0.115
 }

train_results.json

@@ -1,8 +1,8 @@
 {
  "epoch": 1.0,
- "train_loss": 2164.5614415454666,
- "train_runtime": 32346.8016,
+ "train_loss": 1826.8015694608227,
+ "train_runtime": 32379.7062,
  "train_samples": 61135,
- "train_samples_per_second": 1.89,
+ "train_samples_per_second": 1.888,
  "train_steps_per_second": 0.015
 }

trainer_state.json

@@ -25,732 +25,732 @@
  {
  "epoch": 0.02,
  "learning_rate": 1.0416666666666667e-06,
- "logits/chosen": -2.5851330757141113,
- "logits/rejected": -2.6188478469848633,
- "logps/chosen": -265.6952209472656,
- "logps/rejected": -261.4213562011719,
- "loss": 2495.385,
- "rewards/accuracies": 0.4375,
- "rewards/chosen": 0.005977082531899214,
- "rewards/margins": 0.0005994850071147084,
- "rewards/rejected": 0.005377596709877253,
+ "logits/chosen": -2.585383176803589,
+ "logits/rejected": -2.6190898418426514,
+ "logps/chosen": -265.6199035644531,
+ "logps/rejected": -261.3590393066406,
+ "loss": 2489.4685,
+ "rewards/accuracies": 0.4548611044883728,
+ "rewards/chosen": 0.006730278953909874,
+ "rewards/margins": 0.0007296364055946469,
+ "rewards/rejected": 0.006000642664730549,
  "step": 10
  },
  {
  "epoch": 0.04,
  "learning_rate": 2.0833333333333334e-06,
- "logits/chosen": -2.6101512908935547,
- "logits/rejected": -2.5939109325408936,
- "logps/chosen": -255.68185424804688,
- "logps/rejected": -248.1254119873047,
- "loss": 2457.86,
- "rewards/accuracies": 0.628125011920929,
- "rewards/chosen": 0.013690793886780739,
- "rewards/margins": 0.00916606467217207,
- "rewards/rejected": 0.004524729214608669,
+ "logits/chosen": -2.616151809692383,
+ "logits/rejected": -2.599904775619507,
+ "logps/chosen": -253.3858184814453,
+ "logps/rejected": -245.82345581054688,
+ "loss": 2411.3754,
+ "rewards/accuracies": 0.6000000238418579,
+ "rewards/chosen": 0.036651305854320526,
+ "rewards/margins": 0.009106594137847424,
+ "rewards/rejected": 0.02754470705986023,
  "step": 20
  },
  {
  "epoch": 0.06,
  "learning_rate": 3.125e-06,
- "logits/chosen": -2.604323148727417,
- "logits/rejected": -2.598053455352783,
- "logps/chosen": -254.423095703125,
- "logps/rejected": -226.73153686523438,
- "loss": 2402.3988,
- "rewards/accuracies": 0.703125,
- "rewards/chosen": 0.01266755722463131,
- "rewards/margins": 0.024019470438361168,
- "rewards/rejected": -0.01135191135108471,
+ "logits/chosen": -2.617845058441162,
+ "logits/rejected": -2.6118521690368652,
+ "logps/chosen": -250.7469482421875,
+ "logps/rejected": -223.05172729492188,
+ "loss": 2306.1311,
+ "rewards/accuracies": 0.671875,
+ "rewards/chosen": 0.04942930489778519,
+ "rewards/margins": 0.023983022198081017,
+ "rewards/rejected": 0.02544628083705902,
  "step": 30
  },
  {
  "epoch": 0.08,
  "learning_rate": 4.166666666666667e-06,
- "logits/chosen": -2.6043972969055176,
- "logits/rejected": -2.582412004470825,
- "logps/chosen": -279.12042236328125,
- "logps/rejected": -241.2065887451172,
- "loss": 2290.4264,
+ "logits/chosen": -2.6323208808898926,
+ "logits/rejected": -2.608524799346924,
+ "logps/chosen": -276.45947265625,
+ "logps/rejected": -238.35391235351562,
+ "loss": 2100.6182,
  "rewards/accuracies": 0.6968749761581421,
- "rewards/chosen": 0.024520257487893105,
- "rewards/margins": 0.0557018406689167,
- "rewards/rejected": -0.031181585043668747,
+ "rewards/chosen": 0.05112973973155022,
+ "rewards/margins": 0.05378426983952522,
+ "rewards/rejected": -0.002654529409483075,
  "step": 40
  },
  {
  "epoch": 0.1,
  "learning_rate": 4.999731868769027e-06,
- "logits/chosen": -2.531161308288574,
- "logits/rejected": -2.5264387130737305,
- "logps/chosen": -252.51846313476562,
- "logps/rejected": -247.7227325439453,
- "loss": 2291.9322,
- "rewards/accuracies": 0.6812499761581421,
- "rewards/chosen": 0.029673133045434952,
- "rewards/margins": 0.08245684206485748,
- "rewards/rejected": -0.05278371647000313,
+ "logits/chosen": -2.552873373031616,
+ "logits/rejected": -2.5477213859558105,
+ "logps/chosen": -253.2111358642578,
+ "logps/rejected": -248.1074676513672,
+ "loss": 2103.8223,
+ "rewards/accuracies": 0.6781250238418579,
+ "rewards/chosen": 0.022746428847312927,
+ "rewards/margins": 0.07937721163034439,
+ "rewards/rejected": -0.05663077160716057,
  "step": 50
  },
  {
  "epoch": 0.13,
  "learning_rate": 4.9903533134293035e-06,
- "logits/chosen": -2.545037031173706,
- "logits/rejected": -2.5416412353515625,
- "logps/chosen": -260.83905029296875,
- "logps/rejected": -239.8417205810547,
- "loss": 2269.9371,
- "rewards/accuracies": 0.6937500238418579,
- "rewards/chosen": 0.03231300041079521,
- "rewards/margins": 0.09112317860126495,
- "rewards/rejected": -0.05881017446517944,
+ "logits/chosen": -2.556926727294922,
+ "logits/rejected": -2.551504611968994,
+ "logps/chosen": -261.6982116699219,
+ "logps/rejected": -240.27059936523438,
+ "loss": 2054.3434,
+ "rewards/accuracies": 0.6781250238418579,
+ "rewards/chosen": 0.023721303790807724,
+ "rewards/margins": 0.08682042360305786,
+ "rewards/rejected": -0.06309913098812103,
  "step": 60
  },
  {
  "epoch": 0.15,
  "learning_rate": 4.967625656594782e-06,
- "logits/chosen": -2.5832419395446777,
- "logits/rejected": -2.564356565475464,
- "logps/chosen": -275.95452880859375,
- "logps/rejected": -264.7611083984375,
- "loss": 2236.1113,
- "rewards/accuracies": 0.6968749761581421,
- "rewards/chosen": 0.036882974207401276,
- "rewards/margins": 0.08578891307115555,
- "rewards/rejected": -0.048905935138463974,
+ "logits/chosen": -2.5740597248077393,
+ "logits/rejected": -2.553145408630371,
+ "logps/chosen": -278.0965270996094,
+ "logps/rejected": -267.19586181640625,
+ "loss": 1971.1375,
+ "rewards/accuracies": 0.6875,
+ "rewards/chosen": 0.015462947078049183,
+ "rewards/margins": 0.08871600031852722,
+ "rewards/rejected": -0.07325305044651031,
  "step": 70
  },
  {
  "epoch": 0.17,
  "learning_rate": 4.93167072587771e-06,
- "logits/chosen": -2.552919864654541,
- "logits/rejected": -2.524970293045044,
- "logps/chosen": -257.78448486328125,
- "logps/rejected": -262.3812561035156,
- "loss": 2220.0893,
- "rewards/accuracies": 0.737500011920929,
- "rewards/chosen": 0.037375591695308685,
- "rewards/margins": 0.11339374631643295,
- "rewards/rejected": -0.07601816952228546,
+ "logits/chosen": -2.5298993587493896,
+ "logits/rejected": -2.5009925365448,
+ "logps/chosen": -258.5903015136719,
+ "logps/rejected": -263.52850341796875,
+ "loss": 1933.0076,
+ "rewards/accuracies": 0.731249988079071,
+ "rewards/chosen": 0.029317494481801987,
+ "rewards/margins": 0.11680855602025986,
+ "rewards/rejected": -0.08749105781316757,
  "step": 80
  },
  {
  "epoch": 0.19,
  "learning_rate": 4.882681251368549e-06,
- "logits/chosen": -2.56257963180542,
- "logits/rejected": -2.5289363861083984,
- "logps/chosen": -239.4860382080078,
- "logps/rejected": -252.36196899414062,
- "loss": 2167.3848,
- "rewards/accuracies": 0.7250000238418579,
- "rewards/chosen": 0.04182355850934982,
- "rewards/margins": 0.10886694490909576,
- "rewards/rejected": -0.06704337894916534,
+ "logits/chosen": -2.5273003578186035,
+ "logits/rejected": -2.493241548538208,
+ "logps/chosen": -247.47506713867188,
+ "logps/rejected": -260.76678466796875,
+ "loss": 1845.5582,
+ "rewards/accuracies": 0.7281249761581421,
+ "rewards/chosen": -0.03806694597005844,
+ "rewards/margins": 0.11302463710308075,
+ "rewards/rejected": -0.15109160542488098,
  "step": 90
  },
  {
  "epoch": 0.21,
  "learning_rate": 4.8209198325401815e-06,
- "logits/chosen": -2.5551962852478027,
- "logits/rejected": -2.562063455581665,
- "logps/chosen": -266.8739013671875,
- "logps/rejected": -269.649169921875,
- "loss": 2149.4746,
+ "logits/chosen": -2.5287628173828125,
+ "logits/rejected": -2.5358829498291016,
+ "logps/chosen": -272.0884704589844,
+ "logps/rejected": -275.2580871582031,
+ "loss": 1797.9404,
  "rewards/accuracies": 0.762499988079071,
- "rewards/chosen": 0.04759662598371506,
- "rewards/margins": 0.1307816356420517,
- "rewards/rejected": -0.08318501710891724,
+ "rewards/chosen": -0.0045492262579500675,
+ "rewards/margins": 0.13472509384155273,
+ "rewards/rejected": -0.13927432894706726,
  "step": 100
  },
  {
  "epoch": 0.21,
- "eval_logits/chosen": -2.222931385040283,
- "eval_logits/rejected": -2.1770126819610596,
- "eval_logps/chosen": -260.57818603515625,
- "eval_logps/rejected": -253.25228881835938,
- "eval_loss": 2190.7666015625,
- "eval_rewards/accuracies": 0.7460317611694336,
- "eval_rewards/chosen": 0.044464047998189926,
- "eval_rewards/margins": 0.12927772104740143,
- "eval_rewards/rejected": -0.0848136618733406,
- "eval_runtime": 549.355,
- "eval_samples_per_second": 3.641,
+ "eval_logits/chosen": -2.196876287460327,
+ "eval_logits/rejected": -2.1486356258392334,
+ "eval_logps/chosen": -263.71331787109375,
+ "eval_logps/rejected": -256.7424011230469,
+ "eval_loss": 1887.4102783203125,
+ "eval_rewards/accuracies": 0.7519841194152832,
+ "eval_rewards/chosen": 0.013112416490912437,
+ "eval_rewards/margins": 0.13282696902751923,
+ "eval_rewards/rejected": -0.11971456557512283,
+ "eval_runtime": 549.9966,
+ "eval_samples_per_second": 3.636,
  "eval_steps_per_second": 0.115,
  "step": 100
  },
  {
  "epoch": 0.23,
  "learning_rate": 4.746717530629565e-06,
- "logits/chosen": -2.5229454040527344,
- "logits/rejected": -2.5105621814727783,
- "logps/chosen": -261.46649169921875,
- "logps/rejected": -256.37835693359375,
- "loss": 2174.1184,
- "rewards/accuracies": 0.746874988079071,
- "rewards/chosen": 0.03517655283212662,
- "rewards/margins": 0.11897265911102295,
- "rewards/rejected": -0.08379611372947693,
+ "logits/chosen": -2.480510711669922,
+ "logits/rejected": -2.4668211936950684,
+ "logps/chosen": -267.04180908203125,
+ "logps/rejected": -262.5838317871094,
+ "loss": 1870.9051,
+ "rewards/accuracies": 0.734375,
+ "rewards/chosen": -0.020576762035489082,
+ "rewards/margins": 0.125274196267128,
+ "rewards/rejected": -0.14585095643997192,
  "step": 110
  },
  {
  "epoch": 0.25,
  "learning_rate": 4.660472094042121e-06,
- "logits/chosen": -2.5114097595214844,
- "logits/rejected": -2.481840133666992,
- "logps/chosen": -246.70370483398438,
- "logps/rejected": -238.27621459960938,
- "loss": 2181.3053,
- "rewards/accuracies": 0.699999988079071,
- "rewards/chosen": 0.044524095952510834,
- "rewards/margins": 0.10293309390544891,
- "rewards/rejected": -0.05840899422764778,
+ "logits/chosen": -2.44077205657959,
+ "logits/rejected": -2.4053845405578613,
+ "logps/chosen": -256.12939453125,
+ "logps/rejected": -248.28060913085938,
+ "loss": 1855.318,
+ "rewards/accuracies": 0.684374988079071,
+ "rewards/chosen": -0.049732744693756104,
+ "rewards/margins": 0.10872016102075577,
+ "rewards/rejected": -0.15845291316509247,
  "step": 120
  },
  {
  "epoch": 0.27,
  "learning_rate": 4.5626458262912745e-06,
- "logits/chosen": -2.4726600646972656,
- "logits/rejected": -2.46514630317688,
- "logps/chosen": -271.7862548828125,
- "logps/rejected": -260.61676025390625,
- "loss": 2175.3252,
- "rewards/accuracies": 0.762499988079071,
- "rewards/chosen": 0.06200919300317764,
- "rewards/margins": 0.12613125145435333,
- "rewards/rejected": -0.06412206590175629,
+ "logits/chosen": -2.395805597305298,
+ "logits/rejected": -2.383305311203003,
+ "logps/chosen": -280.74053955078125,
+ "logps/rejected": -270.37860107421875,
+ "loss": 1811.4148,
+ "rewards/accuracies": 0.7437499761581421,
+ "rewards/chosen": -0.027533594518899918,
+ "rewards/margins": 0.1342071145772934,
+ "rewards/rejected": -0.1617407202720642,
  "step": 130
  },
  {
  "epoch": 0.29,
  "learning_rate": 4.453763107901676e-06,
- "logits/chosen": -2.506436586380005,
- "logits/rejected": -2.5005128383636475,
- "logps/chosen": -237.8655242919922,
- "logps/rejected": -249.9298553466797,
- "loss": 2167.2516,
+ "logits/chosen": -2.4485344886779785,
+ "logits/rejected": -2.43884015083313,
+ "logps/chosen": -243.1454315185547,
+ "logps/rejected": -255.15432739257812,
+ "loss": 1803.225,
  "rewards/accuracies": 0.734375,
- "rewards/chosen": 0.024008702486753464,
- "rewards/margins": 0.1495535969734192,
- "rewards/rejected": -0.12554487586021423,
+ "rewards/chosen": -0.028790492564439774,
+ "rewards/margins": 0.1489991694688797,
+ "rewards/rejected": -0.17778967320919037,
  "step": 140
  },
  {
  "epoch": 0.31,
  "learning_rate": 4.33440758555951e-06,
- "logits/chosen": -2.5227842330932617,
- "logits/rejected": -2.536785364151001,
- "logps/chosen": -260.7518005371094,
- "logps/rejected": -235.9630889892578,
- "loss": 2119.4062,
- "rewards/accuracies": 0.715624988079071,
- "rewards/chosen": 0.04733316972851753,
- "rewards/margins": 0.12345732748508453,
- "rewards/rejected": -0.0761241465806961,
+ "logits/chosen": -2.459658622741699,
+ "logits/rejected": -2.483065605163574,
+ "logps/chosen": -267.7740478515625,
+ "logps/rejected": -243.34609985351562,
+ "loss": 1781.1752,
+ "rewards/accuracies": 0.699999988079071,
+ "rewards/chosen": -0.02288922667503357,
+ "rewards/margins": 0.12706486880779266,
+ "rewards/rejected": -0.14995409548282623,
  "step": 150
  },
  {
  "epoch": 0.33,
  "learning_rate": 4.205219043576955e-06,
- "logits/chosen": -2.5534234046936035,
- "logits/rejected": -2.4914207458496094,
- "logps/chosen": -254.14065551757812,
- "logps/rejected": -250.95700073242188,
- "loss": 2114.7645,
- "rewards/accuracies": 0.778124988079071,
- "rewards/chosen": 0.06031092256307602,
- "rewards/margins": 0.15202957391738892,
- "rewards/rejected": -0.09171866625547409,
+ "logits/chosen": -2.483583688735962,
+ "logits/rejected": -2.4244942665100098,
+ "logps/chosen": -260.3743896484375,
+ "logps/rejected": -258.7478332519531,
+ "loss": 1754.5766,
+ "rewards/accuracies": 0.784375011920929,
+ "rewards/chosen": -0.0020265295170247555,
+ "rewards/margins": 0.16760031878948212,
+ "rewards/rejected": -0.169626846909523,
  "step": 160
  },
  {
  "epoch": 0.36,
  "learning_rate": 4.066889974440757e-06,
- "logits/chosen": -2.5092320442199707,
- "logits/rejected": -2.4965577125549316,
- "logps/chosen": -254.91439819335938,
- "logps/rejected": -242.8040008544922,
- "loss": 2229.8135,
- "rewards/accuracies": 0.675000011920929,
- "rewards/chosen": 0.034448813647031784,
- "rewards/margins": 0.12951095402240753,
- "rewards/rejected": -0.09506212174892426,
+ "logits/chosen": -2.4374189376831055,
+ "logits/rejected": -2.428433656692505,
+ "logps/chosen": -264.5699768066406,
+ "logps/rejected": -252.79421997070312,
+ "loss": 1953.8818,
+ "rewards/accuracies": 0.668749988079071,
+ "rewards/chosen": -0.06210694834589958,
+ "rewards/margins": 0.13285748660564423,
+ "rewards/rejected": -0.1949644386768341,
  "step": 170
  },
  {
  "epoch": 0.38,
  "learning_rate": 3.92016186682789e-06,
- "logits/chosen": -2.521221399307251,
- "logits/rejected": -2.533686399459839,
- "logps/chosen": -251.4235382080078,
- "logps/rejected": -259.76220703125,
- "loss": 2175.5213,
+ "logits/chosen": -2.467085361480713,
+ "logits/rejected": -2.487204074859619,
+ "logps/chosen": -262.995361328125,
+ "logps/rejected": -271.94183349609375,
+ "loss": 1848.9945,
  "rewards/accuracies": 0.721875011920929,
- "rewards/chosen": 0.04062749817967415,
- "rewards/margins": 0.120635487139225,
- "rewards/rejected": -0.08000798523426056,
+ "rewards/chosen": -0.07509048283100128,
+ "rewards/margins": 0.12671387195587158,
+ "rewards/rejected": -0.20180435478687286,
  "step": 180
  },
  {
  "epoch": 0.4,
  "learning_rate": 3.7658212309857576e-06,
- "logits/chosen": -2.5192363262176514,
- "logits/rejected": -2.4917151927948,
- "logps/chosen": -255.2060089111328,
- "logps/rejected": -250.82022094726562,
- "loss": 2099.443,
- "rewards/accuracies": 0.715624988079071,
- "rewards/chosen": 0.0492943711578846,
- "rewards/margins": 0.14053165912628174,
- "rewards/rejected": -0.09123729914426804,
+ "logits/chosen": -2.450601816177368,
+ "logits/rejected": -2.4304168224334717,
+ "logps/chosen": -269.1886901855469,
+ "logps/rejected": -265.7490539550781,
+ "loss": 1698.6666,
+ "rewards/accuracies": 0.746874988079071,
+ "rewards/chosen": -0.09053254127502441,
+ "rewards/margins": 0.14999321103096008,
+ "rewards/rejected": -0.2405257225036621,
  "step": 190
  },
  {
  "epoch": 0.42,
  "learning_rate": 3.604695382782159e-06,
- "logits/chosen": -2.5251801013946533,
- "logits/rejected": -2.5034642219543457,
- "logps/chosen": -269.3675537109375,
- "logps/rejected": -262.86376953125,
- "loss": 2105.1256,
- "rewards/accuracies": 0.7562500238418579,
- "rewards/chosen": 0.0575677752494812,
- "rewards/margins": 0.14340198040008545,
- "rewards/rejected": -0.08583419024944305,
+ "logits/chosen": -2.447007179260254,
+ "logits/rejected": -2.419039726257324,
+ "logps/chosen": -282.8253479003906,
+ "logps/rejected": -278.14508056640625,
+ "loss": 1700.9055,
+ "rewards/accuracies": 0.784375011920929,
+ "rewards/chosen": -0.07701022177934647,
+ "rewards/margins": 0.16163742542266846,
+ "rewards/rejected": -0.23864765465259552,
  "step": 200
  },
  {
  "epoch": 0.42,
- "eval_logits/chosen": -2.260270833969116,
- "eval_logits/rejected": -2.2073864936828613,
- "eval_logps/chosen": -259.5941467285156,
- "eval_logps/rejected": -254.3839874267578,
- "eval_loss": 2151.155517578125,
- "eval_rewards/accuracies": 0.7599206566810608,
- "eval_rewards/chosen": 0.05430443957448006,
- "eval_rewards/margins": 0.15043501555919647,
- "eval_rewards/rejected": -0.09613056480884552,
- "eval_runtime": 548.195,
- "eval_samples_per_second": 3.648,
+ "eval_logits/chosen": -2.161839485168457,
+ "eval_logits/rejected": -2.1081268787384033,
+ "eval_logps/chosen": -269.66546630859375,
+ "eval_logps/rejected": -265.3905029296875,
+ "eval_loss": 1784.6597900390625,
+ "eval_rewards/accuracies": 0.761904776096344,
+ "eval_rewards/chosen": -0.0464087538421154,
+ "eval_rewards/margins": 0.15978708863258362,
+ "eval_rewards/rejected": -0.2061958611011505,
+ "eval_runtime": 549.0189,
+ "eval_samples_per_second": 3.643,
  "eval_steps_per_second": 0.115,
  "step": 200
  },
  {
  "epoch": 0.44,
  "learning_rate": 3.437648009023905e-06,
- "logits/chosen": -2.533383369445801,
- "logits/rejected": -2.4935860633850098,
- "logps/chosen": -243.6236114501953,
- "logps/rejected": -238.85140991210938,
- "loss": 2145.5416,
- "rewards/accuracies": 0.7437499761581421,
- "rewards/chosen": 0.06410142779350281,
- "rewards/margins": 0.14374245703220367,
- "rewards/rejected": -0.07964102178812027,
+ "logits/chosen": -2.458688259124756,
+ "logits/rejected": -2.4217796325683594,
+ "logps/chosen": -252.5647430419922,
+ "logps/rejected": -248.326416015625,
+ "loss": 1806.0594,
+ "rewards/accuracies": 0.7562500238418579,
+ "rewards/chosen": -0.02530970238149166,
+ "rewards/margins": 0.14908090233802795,
+ "rewards/rejected": -0.17439061403274536,
  "step": 210
  },
  {
  "epoch": 0.46,
  "learning_rate": 3.265574537815398e-06,
- "logits/chosen": -2.554565906524658,
- "logits/rejected": -2.56289005279541,
- "logps/chosen": -277.4061584472656,
- "logps/rejected": -253.40048217773438,
- "loss": 2196.8484,
- "rewards/accuracies": 0.675000011920929,
- "rewards/chosen": 0.052330613136291504,
- "rewards/margins": 0.11339585483074188,
- "rewards/rejected": -0.06106524541974068,
+ "logits/chosen": -2.4742610454559326,
+ "logits/rejected": -2.4789376258850098,
+ "logps/chosen": -286.0444030761719,
+ "logps/rejected": -261.9767150878906,
+ "loss": 1855.6273,
+ "rewards/accuracies": 0.706250011920929,
+ "rewards/chosen": -0.03405206650495529,
+ "rewards/margins": 0.11277566105127335,
+ "rewards/rejected": -0.14682772755622864,
  "step": 220
  },
  {
  "epoch": 0.48,
  "learning_rate": 3.089397338773569e-06,
- "logits/chosen": -2.4857611656188965,
- "logits/rejected": -2.473193407058716,
- "logps/chosen": -247.3427276611328,
- "logps/rejected": -241.8627471923828,
- "loss": 2160.1729,
- "rewards/accuracies": 0.7124999761581421,
- "rewards/chosen": 0.03845102712512016,
- "rewards/margins": 0.11976752430200577,
- "rewards/rejected": -0.0813164934515953,
+ "logits/chosen": -2.38773775100708,
+ "logits/rejected": -2.3718185424804688,
+ "logps/chosen": -257.7181701660156,
+ "logps/rejected": -253.3428955078125,
+ "loss": 1797.9486,
+ "rewards/accuracies": 0.721875011920929,
+ "rewards/chosen": -0.06530335545539856,
+ "rewards/margins": 0.1308148354291916,
+ "rewards/rejected": -0.19611820578575134,
  "step": 230
  },
  {
  "epoch": 0.5,
  "learning_rate": 2.9100607788275547e-06,
- "logits/chosen": -2.5121560096740723,
- "logits/rejected": -2.516338586807251,
- "logps/chosen": -257.1769714355469,
- "logps/rejected": -247.3695068359375,
- "loss": 2185.7641,
- "rewards/accuracies": 0.684374988079071,
- "rewards/chosen": 0.0379050187766552,
- "rewards/margins": 0.11140499264001846,
- "rewards/rejected": -0.07349997013807297,
+ "logits/chosen": -2.4125852584838867,
+ "logits/rejected": -2.414628267288208,
+ "logps/chosen": -265.2156066894531,
+ "logps/rejected": -257.0289001464844,
+ "loss": 1850.0729,
+ "rewards/accuracies": 0.690625011920929,
+ "rewards/chosen": -0.04248107224702835,
+ "rewards/margins": 0.12761279940605164,
+ "rewards/rejected": -0.17009387910366058,
  "step": 240
  },
  {
  "epoch": 0.52,
  "learning_rate": 2.72852616010567e-06,
- "logits/chosen": -2.5092978477478027,
- "logits/rejected": -2.487090826034546,
- "logps/chosen": -264.5955505371094,
- "logps/rejected": -246.3382110595703,
- "loss": 2136.6197,
- "rewards/accuracies": 0.7406250238418579,
- "rewards/chosen": 0.039962492883205414,
- "rewards/margins": 0.1403963267803192,
- "rewards/rejected": -0.1004338413476944,
+ "logits/chosen": -2.4339253902435303,
+ "logits/rejected": -2.4054951667785645,
+ "logps/chosen": -271.8371276855469,
+ "logps/rejected": -255.33438110351562,
+ "loss": 1766.1885,
+ "rewards/accuracies": 0.753125011920929,
+ "rewards/chosen": -0.0324532687664032,
+ "rewards/margins": 0.1579422652721405,
+ "rewards/rejected": -0.1903955340385437,
  "step": 250
  },
  {
  "epoch": 0.54,
  "learning_rate": 2.5457665670441937e-06,
- "logits/chosen": -2.5069711208343506,
- "logits/rejected": -2.5030505657196045,
- "logps/chosen": -257.4859619140625,
- "logps/rejected": -231.91958618164062,
- "loss": 2085.2795,
- "rewards/accuracies": 0.762499988079071,
- "rewards/chosen": 0.05723271518945694,
- "rewards/margins": 0.15024301409721375,
- "rewards/rejected": -0.0930103212594986,
+ "logits/chosen": -2.4216437339782715,
+ "logits/rejected": -2.4156367778778076,
+ "logps/chosen": -266.7996520996094,
+ "logps/rejected": -243.180419921875,
+ "loss": 1710.8809,
+ "rewards/accuracies": 0.753125011920929,
+ "rewards/chosen": -0.035904210060834885,
+ "rewards/margins": 0.16971439123153687,
+ "rewards/rejected": -0.20561861991882324,
  "step": 260
  },
  {
  "epoch": 0.57,
  "learning_rate": 2.3627616503391813e-06,
- "logits/chosen": -2.525665760040283,
- "logits/rejected": -2.5043163299560547,
- "logps/chosen": -280.7471618652344,
- "logps/rejected": -267.36712646484375,
- "loss": 2089.859,
- "rewards/accuracies": 0.737500011920929,
- "rewards/chosen": 0.05569761246442795,
- "rewards/margins": 0.179846853017807,
- "rewards/rejected": -0.12414924055337906,
+ "logits/chosen": -2.4438915252685547,
+ "logits/rejected": -2.416748285293579,
+ "logps/chosen": -290.58453369140625,
+ "logps/rejected": -277.0739440917969,
+ "loss": 1714.5062,
+ "rewards/accuracies": 0.765625,
+ "rewards/chosen": -0.042676471173763275,
+ "rewards/margins": 0.17854078114032745,
+ "rewards/rejected": -0.22121724486351013,
  "step": 270
  },
  {
  "epoch": 0.59,
  "learning_rate": 2.1804923757009885e-06,
- "logits/chosen": -2.500837564468384,
- "logits/rejected": -2.501950740814209,
- "logps/chosen": -270.04193115234375,
- "logps/rejected": -248.61978149414062,
- "loss": 2111.6906,
- "rewards/accuracies": 0.7124999761581421,
- "rewards/chosen": 0.05320361256599426,
- "rewards/margins": 0.1410333216190338,
- "rewards/rejected": -0.08782971650362015,
+ "logits/chosen": -2.414602756500244,
+ "logits/rejected": -2.4200820922851562,
+ "logps/chosen": -282.95147705078125,
+ "logps/rejected": -261.1886291503906,
+ "loss": 1764.4607,
+ "rewards/accuracies": 0.7437499761581421,
+ "rewards/chosen": -0.07589195668697357,
+ "rewards/margins": 0.13762618601322174,
+ "rewards/rejected": -0.2135181427001953,
  "step": 280
  },
  {
  "epoch": 0.61,
  "learning_rate": 1.9999357655598894e-06,
- "logits/chosen": -2.5122292041778564,
- "logits/rejected": -2.50368070602417,
- "logps/chosen": -258.72686767578125,
- "logps/rejected": -256.91387939453125,
- "loss": 2137.0592,
- "rewards/accuracies": 0.737500011920929,
- "rewards/chosen": 0.053160279989242554,
- "rewards/margins": 0.15454119443893433,
- "rewards/rejected": -0.10138092190027237,
+ "logits/chosen": -2.430169105529785,
+ "logits/rejected": -2.4057881832122803,
+ "logps/chosen": -265.06805419921875,
+ "logps/rejected": -263.2739562988281,
+ "loss": 1786.2846,
+ "rewards/accuracies": 0.746874988079071,
+ "rewards/chosen": -0.010251840576529503,
+ "rewards/margins": 0.15472975373268127,
+ "rewards/rejected": -0.16498157382011414,
  "step": 290
  },
  {
  "epoch": 0.63,
  "learning_rate": 1.8220596619089576e-06,
- "logits/chosen": -2.471623659133911,
- "logits/rejected": -2.4690403938293457,
- "logps/chosen": -246.51766967773438,
- "logps/rejected": -251.79257202148438,
- "loss": 2135.4973,
- "rewards/accuracies": 0.71875,
- "rewards/chosen": 0.0453377440571785,
- "rewards/margins": 0.12641170620918274,
- "rewards/rejected": -0.08107397705316544,
+ "logits/chosen": -2.392138957977295,
+ "logits/rejected": -2.3823294639587402,
+ "logps/chosen": -255.75393676757812,
+ "logps/rejected": -261.84271240234375,
+ "loss": 1767.2219,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": -0.04702477902173996,
+ "rewards/margins": 0.13455010950565338,
+ "rewards/rejected": -0.18157489597797394,
  "step": 300
  },
  {
  "epoch": 0.63,
- "eval_logits/chosen": -2.2764506340026855,
- "eval_logits/rejected": -2.2231767177581787,
- "eval_logps/chosen": -258.7624206542969,
- "eval_logps/rejected": -252.75852966308594,
- "eval_loss": 2129.089599609375,
- "eval_rewards/accuracies": 0.7559523582458496,
- "eval_rewards/chosen": 0.06262180209159851,
- "eval_rewards/margins": 0.14249789714813232,
- "eval_rewards/rejected": -0.07987607270479202,
- "eval_runtime": 547.9938,
- "eval_samples_per_second": 3.65,
+ "eval_logits/chosen": -2.158698797225952,
+ "eval_logits/rejected": -2.1057095527648926,
+ "eval_logps/chosen": -269.6955871582031,
+ "eval_logps/rejected": -264.77947998046875,
+ "eval_loss": 1735.518310546875,
+ "eval_rewards/accuracies": 0.7698412537574768,
+ "eval_rewards/chosen": -0.04671022295951843,
+ "eval_rewards/margins": 0.15337513387203217,
+ "eval_rewards/rejected": -0.2000853717327118,
+ "eval_runtime": 548.7136,
+ "eval_samples_per_second": 3.645,
  "eval_steps_per_second": 0.115,
  "step": 300
  },
  {
  "epoch": 0.65,
  "learning_rate": 1.647817538357072e-06,
- "logits/chosen": -2.5041086673736572,
- "logits/rejected": -2.495436191558838,
- "logps/chosen": -264.5109558105469,
- "logps/rejected": -248.3275604248047,
- "loss": 2107.123,
- "rewards/accuracies": 0.731249988079071,
- "rewards/chosen": 0.05480458214879036,
- "rewards/margins": 0.13964474201202393,
- "rewards/rejected": -0.08484016358852386,
+ "logits/chosen": -2.4140188694000244,
+ "logits/rejected": -2.4031002521514893,
+ "logps/chosen": -274.3767395019531,
+ "logps/rejected": -259.40155029296875,
+ "loss": 1673.9693,
+ "rewards/accuracies": 0.746874988079071,
+ "rewards/chosen": -0.04385297745466232,
+ "rewards/margins": 0.1517268717288971,
+ "rewards/rejected": -0.19557985663414001,
  "step": 310
  },
  {
  "epoch": 0.67,
  "learning_rate": 1.4781433892011132e-06,
- "logits/chosen": -2.53191876411438,
- "logits/rejected": -2.4989166259765625,
- "logps/chosen": -242.36599731445312,
- "logps/rejected": -243.78067016601562,
- "loss": 2076.0621,
- "rewards/accuracies": 0.7718750238418579,
- "rewards/chosen": 0.05456935614347458,
- "rewards/margins": 0.14978976547718048,
- "rewards/rejected": -0.0952204093337059,
+ "logits/chosen": -2.416640520095825,
+ "logits/rejected": -2.370535135269165,
+ "logps/chosen": -252.216064453125,
+ "logps/rejected": -255.1393280029297,
+ "loss": 1673.8594,
+ "rewards/accuracies": 0.7593749761581421,
+ "rewards/chosen": -0.043931327760219574,
+ "rewards/margins": 0.16487570106983185,
+ "rewards/rejected": -0.20880703628063202,
  "step": 320
  },
  {
  "epoch": 0.69,
  "learning_rate": 1.3139467229135999e-06,
- "logits/chosen": -2.4768006801605225,
- "logits/rejected": -2.4569873809814453,
- "logps/chosen": -263.0523681640625,
- "logps/rejected": -250.5469207763672,
- "loss": 2112.1141,
- "rewards/accuracies": 0.734375,
- "rewards/chosen": 0.044828929007053375,
- "rewards/margins": 0.13050048053264618,
- "rewards/rejected": -0.0856715738773346,
+ "logits/chosen": -2.362358570098877,
+ "logits/rejected": -2.3449196815490723,
+ "logps/chosen": -270.876220703125,
+ "logps/rejected": -259.251953125,
+ "loss": 1731.3877,
+ "rewards/accuracies": 0.753125011920929,
+ "rewards/chosen": -0.03340950980782509,
+ "rewards/margins": 0.13931182026863098,
+ "rewards/rejected": -0.17272132635116577,
  "step": 330
  },
  {
  "epoch": 0.71,
  "learning_rate": 1.1561076868822756e-06,
- "logits/chosen": -2.5158028602600098,
- "logits/rejected": -2.5096983909606934,
- "logps/chosen": -275.6848449707031,
- "logps/rejected": -246.7259979248047,
- "loss": 2151.2445,
- "rewards/accuracies": 0.746874988079071,
- "rewards/chosen": 0.052164845168590546,
- "rewards/margins": 0.15314052999019623,
- "rewards/rejected": -0.10097566992044449,
+ "logits/chosen": -2.3923397064208984,
+ "logits/rejected": -2.384582281112671,
+ "logps/chosen": -284.8360290527344,
+ "logps/rejected": -257.0713806152344,
+ "loss": 1778.2957,
+ "rewards/accuracies": 0.7718750238418579,
+ "rewards/chosen": -0.039347052574157715,
+ "rewards/margins": 0.1650826632976532,
+ "rewards/rejected": -0.20442970097064972,
  "step": 340
  },
  {
  "epoch": 0.73,
  "learning_rate": 1.0054723495346484e-06,
- "logits/chosen": -2.518799304962158,
- "logits/rejected": -2.4620516300201416,
- "logps/chosen": -249.27401733398438,
- "logps/rejected": -218.7183074951172,
- "loss": 2093.9803,
- "rewards/accuracies": 0.7593749761581421,
- "rewards/chosen": 0.0662151575088501,
- "rewards/margins": 0.14556364715099335,
- "rewards/rejected": -0.07934850454330444,
+ "logits/chosen": -2.3869528770446777,
+ "logits/rejected": -2.3370375633239746,
+ "logps/chosen": -259.75189208984375,
+ "logps/rejected": -231.13577270507812,
+ "loss": 1665.3461,
+ "rewards/accuracies": 0.7437499761581421,
+ "rewards/chosen": -0.03856384754180908,
+ "rewards/margins": 0.1649591028690338,
+ "rewards/rejected": -0.2035229504108429,
  "step": 350
  },
  {
  "epoch": 0.75,
  "learning_rate": 8.628481651367876e-07,
- "logits/chosen": -2.5340943336486816,
- "logits/rejected": -2.5006654262542725,
- "logps/chosen": -260.32464599609375,
- "logps/rejected": -237.3218536376953,
- "loss": 2094.1246,
- "rewards/accuracies": 0.765625,
- "rewards/chosen": 0.05396001785993576,
- "rewards/margins": 0.15317106246948242,
- "rewards/rejected": -0.09921105206012726,
+ "logits/chosen": -2.4105262756347656,
+ "logits/rejected": -2.352128744125366,
+ "logps/chosen": -269.03790283203125,
+ "logps/rejected": -247.90872192382812,
+ "loss": 1665.3982,
+ "rewards/accuracies": 0.7875000238418579,
+ "rewards/chosen": -0.03317265957593918,
+ "rewards/margins": 0.1719072014093399,
+ "rewards/rejected": -0.20507986843585968,
  "step": 360
  },
  {
  "epoch": 0.77,
  "learning_rate": 7.289996455765749e-07,
- "logits/chosen": -2.529265880584717,
- "logits/rejected": -2.515712261199951,
- "logps/chosen": -266.943115234375,
- "logps/rejected": -246.0579376220703,
- "loss": 2115.357,
- "rewards/accuracies": 0.71875,
- "rewards/chosen": 0.052078358829021454,
- "rewards/margins": 0.1462351232767105,
- "rewards/rejected": -0.09415675699710846,
+ "logits/chosen": -2.4054064750671387,
+ "logits/rejected": -2.38871431350708,
+ "logps/chosen": -279.2740173339844,
+ "logps/rejected": -259.63690185546875,
+ "loss": 1704.7645,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": -0.07123039662837982,
+ "rewards/margins": 0.15871620178222656,
+ "rewards/rejected": -0.22994661331176758,
  "step": 370
  },
  {
  "epoch": 0.8,
  "learning_rate": 6.046442623320145e-07,
- "logits/chosen": -2.4891440868377686,
- "logits/rejected": -2.499753952026367,
- "logps/chosen": -253.51632690429688,
- "logps/rejected": -245.4505615234375,
- "loss": 2082.182,
- "rewards/accuracies": 0.7406250238418579,
- "rewards/chosen": 0.051686953753232956,
- "rewards/margins": 0.1390691101551056,
- "rewards/rejected": -0.08738215267658234,
+ "logits/chosen": -2.3605639934539795,
+ "logits/rejected": -2.368460178375244,
+ "logps/chosen": -267.2261657714844,
+ "logps/rejected": -260.45550537109375,
+ "loss": 1647.7326,
+ "rewards/accuracies": 0.734375,
+ "rewards/chosen": -0.0854114517569542,
+ "rewards/margins": 0.15202030539512634,
+ "rewards/rejected": -0.23743176460266113,
  "step": 380
  },
  {
  "epoch": 0.82,
  "learning_rate": 4.904486005914027e-07,
- "logits/chosen": -2.532160997390747,
- "logits/rejected": -2.5001654624938965,
- "logps/chosen": -280.9754333496094,
- "logps/rejected": -279.0588684082031,
- "loss": 2114.3043,
- "rewards/accuracies": 0.7593749761581421,
- "rewards/chosen": 0.0547635443508625,
- "rewards/margins": 0.14076778292655945,
- "rewards/rejected": -0.08600424975156784,
+ "logits/chosen": -2.4076011180877686,
+ "logits/rejected": -2.3770554065704346,
+ "logps/chosen": -292.8500061035156,
+ "logps/rejected": -292.0636291503906,
+ "loss": 1739.5414,
+ "rewards/accuracies": 0.746874988079071,
+ "rewards/chosen": -0.06398223340511322,
+ "rewards/margins": 0.15206970274448395,
+ "rewards/rejected": -0.21605193614959717,
  "step": 390
  },
  {
  "epoch": 0.84,
  "learning_rate": 3.8702478614051353e-07,
- "logits/chosen": -2.4791765213012695,
- "logits/rejected": -2.4799935817718506,
- "logps/chosen": -246.14102172851562,
- "logps/rejected": -251.533447265625,
- "loss": 2099.8018,
- "rewards/accuracies": 0.703125,
- "rewards/chosen": 0.0392024889588356,
- "rewards/margins": 0.13221651315689087,
- "rewards/rejected": -0.09301402419805527,
+ "logits/chosen": -2.3384757041931152,
+ "logits/rejected": -2.3366100788116455,
+ "logps/chosen": -259.2252502441406,
+ "logps/rejected": -265.5692138671875,
+ "loss": 1717.4336,
+ "rewards/accuracies": 0.7124999761581421,
+ "rewards/chosen": -0.09164019674062729,
+ "rewards/margins": 0.14173154532909393,
+ "rewards/rejected": -0.23337173461914062,
  "step": 400
  },
  {
  "epoch": 0.84,
- "eval_logits/chosen": -2.254145860671997,
- "eval_logits/rejected": -2.2016360759735107,
- "eval_logps/chosen": -259.64398193359375,
- "eval_logps/rejected": -254.3590850830078,
- "eval_loss": 2121.667236328125,
- "eval_rewards/accuracies": 0.7539682388305664,
- "eval_rewards/chosen": 0.05380600318312645,
- "eval_rewards/margins": 0.14968746900558472,
- "eval_rewards/rejected": -0.09588146954774857,
- "eval_runtime": 547.9727,
- "eval_samples_per_second": 3.65,
+ "eval_logits/chosen": -2.088451385498047,
+ "eval_logits/rejected": -2.032222032546997,
+ "eval_logps/chosen": -271.9333190917969,
+ "eval_logps/rejected": -267.85687255859375,
+ "eval_loss": 1721.676513671875,
+ "eval_rewards/accuracies": 0.77182537317276,
+ "eval_rewards/chosen": -0.06908722221851349,
+ "eval_rewards/margins": 0.1617719829082489,
+ "eval_rewards/rejected": -0.23085922002792358,
+ "eval_runtime": 548.423,
+ "eval_samples_per_second": 3.647,
  "eval_steps_per_second": 0.115,
  "step": 400
  },
  {
  "epoch": 0.86,
  "learning_rate": 2.9492720416985004e-07,
- "logits/chosen": -2.4832329750061035,
- "logits/rejected": -2.463463306427002,
- "logps/chosen": -284.7741394042969,
- "logps/rejected": -252.4269561767578,
- "loss": 2145.448,
- "rewards/accuracies": 0.7250000238418579,
- "rewards/chosen": 0.05263269692659378,
- "rewards/margins": 0.15021036565303802,
- "rewards/rejected": -0.09757767617702484,
+ "logits/chosen": -2.3725836277008057,
+ "logits/rejected": -2.3304688930511475,
+ "logps/chosen": -296.72991943359375,
+ "logps/rejected": -266.0842590332031,
+ "loss": 1755.1898,
+ "rewards/accuracies": 0.7406250238418579,
+ "rewards/chosen": -0.0669253021478653,
+ "rewards/margins": 0.16722533106803894,
+ "rewards/rejected": -0.23415064811706543,
  "step": 410
  },
  {
  "epoch": 0.88,
  "learning_rate": 2.1464952759020857e-07,
- "logits/chosen": -2.4804348945617676,
- "logits/rejected": -2.457764148712158,
- "logps/chosen": -254.78604125976562,
- "logps/rejected": -278.61346435546875,
- "loss": 2123.6629,
- "rewards/accuracies": 0.6968749761581421,
- "rewards/chosen": 0.033899884670972824,
- "rewards/margins": 0.11116783320903778,
- "rewards/rejected": -0.07726795971393585,
+ "logits/chosen": -2.3592472076416016,
+ "logits/rejected": -2.331540107727051,
+ "logps/chosen": -266.99005126953125,
+ "logps/rejected": -292.03680419921875,
+ "loss": 1730.9672,
+ "rewards/accuracies": 0.706250011920929,
+ "rewards/chosen": -0.08814045041799545,
+ "rewards/margins": 0.12336041778326035,
+ "rewards/rejected": -0.2115008533000946,
  "step": 420
  },
  {
  "epoch": 0.9,
  "learning_rate": 1.4662207078575685e-07,
- "logits/chosen": -2.4848549365997314,
- "logits/rejected": -2.485640048980713,
- "logps/chosen": -268.3457336425781,
- "logps/rejected": -268.5885925292969,
- "loss": 2144.4309,
- "rewards/accuracies": 0.721875011920929,
- "rewards/chosen": 0.03841588646173477,
- "rewards/margins": 0.13024446368217468,
- "rewards/rejected": -0.09182857722043991,
+ "logits/chosen": -2.366381883621216,
+ "logits/rejected": -2.35951566696167,
+ "logps/chosen": -280.3116149902344,
+ "logps/rejected": -281.93939208984375,
+ "loss": 1760.3617,
+ "rewards/accuracies": 0.7281249761581421,
+ "rewards/chosen": -0.08124328404664993,
+ "rewards/margins": 0.14409320056438446,
+ "rewards/rejected": -0.2253364771604538,
  "step": 430
  },
  {
  "epoch": 0.92,
  "learning_rate": 9.120948298936422e-08,
- "logits/chosen": -2.457054615020752,
- "logits/rejected": -2.4329726696014404,
- "logps/chosen": -231.9584197998047,
- "logps/rejected": -234.6277313232422,
- "loss": 2118.3984,
- "rewards/accuracies": 0.737500011920929,
- "rewards/chosen": 0.038600482046604156,
- "rewards/margins": 0.13669805228710175,
- "rewards/rejected": -0.09809757024049759,
+ "logits/chosen": -2.3266444206237793,
+ "logits/rejected": -2.2898497581481934,
+ "logps/chosen": -243.9964141845703,
+ "logps/rejected": -248.1795196533203,
+ "loss": 1711.7143,
+ "rewards/accuracies": 0.75,
+ "rewards/chosen": -0.08177933841943741,
+ "rewards/margins": 0.15183614194393158,
+ "rewards/rejected": -0.23361548781394958,
  "step": 440
  },
  {
  "epoch": 0.94,
  "learning_rate": 4.870879364444109e-08,
- "logits/chosen": -2.5156655311584473,
- "logits/rejected": -2.563300848007202,
- "logps/chosen": -263.9936218261719,
- "logps/rejected": -265.6227722167969,
- "loss": 2123.5402,
- "rewards/accuracies": 0.7124999761581421,
- "rewards/chosen": 0.04902677983045578,
- "rewards/margins": 0.1260160207748413,
- "rewards/rejected": -0.07698923349380493,
+ "logits/chosen": -2.3814821243286133,
+ "logits/rejected": -2.4406635761260986,
+ "logps/chosen": -275.4070129394531,
+ "logps/rejected": -278.91082763671875,
+ "loss": 1743.9877,
+ "rewards/accuracies": 0.7406250238418579,
+ "rewards/chosen": -0.06510698050260544,
+ "rewards/margins": 0.14476314187049866,
+ "rewards/rejected": -0.2098701000213623,
  "step": 450
  },
  {
  "epoch": 0.96,
  "learning_rate": 1.93478202307823e-08,
- "logits/chosen": -2.470996379852295,
- "logits/rejected": -2.4720451831817627,
- "logps/chosen": -258.21734619140625,
- "logps/rejected": -262.04925537109375,
- "loss": 2078.5094,
- "rewards/accuracies": 0.7281249761581421,
- "rewards/chosen": 0.04391016811132431,
- "rewards/margins": 0.14817874133586884,
- "rewards/rejected": -0.10426857322454453,
+ "logits/chosen": -2.34289288520813,
+ "logits/rejected": -2.346625804901123,
+ "logps/chosen": -270.3787841796875,
+ "logps/rejected": -275.61651611328125,
+ "loss": 1676.5176,
+ "rewards/accuracies": 0.737500011920929,
+ "rewards/chosen": -0.07770398259162903,
+ "rewards/margins": 0.16223737597465515,
+ "rewards/rejected": -0.23994135856628418,
  "step": 460
  },
  {
  "epoch": 0.98,
  "learning_rate": 3.283947088983663e-09,
- "logits/chosen": -2.513140916824341,
- "logits/rejected": -2.535651206970215,
- "logps/chosen": -249.6727752685547,
- "logps/rejected": -248.2782745361328,
- "loss": 2093.2779,
- "rewards/accuracies": 0.75,
- "rewards/chosen": 0.04741714522242546,
- "rewards/margins": 0.143958181142807,
- "rewards/rejected": -0.09654103964567184,
+ "logits/chosen": -2.3888649940490723,
+ "logits/rejected": -2.4120144844055176,
+ "logps/chosen": -261.28997802734375,
+ "logps/rejected": -261.67755126953125,
+ "loss": 1663.4154,
+ "rewards/accuracies": 0.768750011920929,
+ "rewards/chosen": -0.0687546655535698,
+ "rewards/margins": 0.16177912056446075,
+ "rewards/rejected": -0.23053380846977234,
  "step": 470
  },
  {
  "epoch": 1.0,
  "step": 477,
  "total_flos": 0.0,
- "train_loss": 2164.5614415454666,
- "train_runtime": 32346.8016,
- "train_samples_per_second": 1.89,
+ "train_loss": 1826.8015694608227,
+ "train_runtime": 32379.7062,
+ "train_samples_per_second": 1.888,
  "train_steps_per_second": 0.015
  }
  ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89ae9269821a7a76bfccee733cea8c3af1d1b7b751ef31fb40915f4d080f4944
+oid sha256:e9de8831bf203a26c117251200a242a486dd5bc4f1aae373c17a996f39be3288
 size 4920