Model save

README.md

@@ -1,33 +1,29 @@
 ---
 license: apache-2.0
-base_model: plaguss/zephyr-7b-spin-iter1-v0
+base_model: plaguss/zephyr-7b-spin-iter2-v0
 tags:
-- alignment-handbook
 - generated_from_trainer
-datasets:
-- argilla/10k_prompts_SPIN_iter1_zephyr_top
-- argilla/10k_prompts_SPIN_iter2_zephyr_top
 model-index:
-- name: outputs
+- name: zephyr-7b-spin-iter3-v0
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# outputs
+# zephyr-7b-spin-iter3-v0
 
-This model is a fine-tuned version of [plaguss/zephyr-7b-spin-iter1-v0](https://huggingface.co/plaguss/zephyr-7b-spin-iter1-v0) on the argilla/10k_prompts_SPIN_iter1_zephyr_top and the argilla/10k_prompts_SPIN_iter2_zephyr_top datasets.
+This model is a fine-tuned version of [plaguss/zephyr-7b-spin-iter2-v0](https://huggingface.co/plaguss/zephyr-7b-spin-iter2-v0) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.1253
-- Rewards/real: -0.5683
-- Rewards/generated: -4.9538
-- Rewards/accuracies: 0.9479
-- Rewards/margins: 4.3854
-- Logps/generated: -739.3701
-- Logps/real: -278.2851
-- Logits/generated: -2.8430
-- Logits/real: -2.8375
+- Loss: 0.1099
+- Rewards/real: -2.9181
+- Rewards/generated: -29.6970
+- Rewards/accuracies: 0.9271
+- Rewards/margins: 26.7789
+- Logps/generated: -702.4378
+- Logps/real: -278.1470
+- Logits/generated: -2.8177
+- Logits/real: -2.8051
 
 ## Model description
 
@@ -64,10 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/real | Rewards/generated | Rewards/accuracies | Rewards/margins | Logps/generated | Logps/real | Logits/generated | Logits/real |
 |:-------------:|:-----:|:----:|:---------------:|:------------:|:-----------------:|:------------------:|:---------------:|:---------------:|:----------:|:----------------:|:-----------:|
-| 5.8769        | 0.49  | 25   | 0.1890          | -0.1680      | -2.9833           | 0.9375             | 2.8153          | -719.6649       | -274.2817  | -2.7940          | -2.8382     |
-| 0.1202        | 0.97  | 50   | 0.1440          | -0.4164      | -4.2256           | 0.9479             | 3.8092          | -732.0879       | -276.7652  | -2.8395          | -2.8439     |
-| 0.0754        | 1.46  | 75   | 0.1298          | -0.5468      | -4.7565           | 0.9583             | 4.2097          | -737.3973       | -278.0700  | -2.8411          | -2.8388     |
-| 0.0621        | 1.94  | 100  | 0.1253          | -0.5683      | -4.9538           | 0.9479             | 4.3854          | -739.3701       | -278.2851  | -2.8430          | -2.8375     |
+| 0.2928        | 0.49  | 25   | 0.3951          | -2.6212      | -20.3268          | 0.9062             | 17.7056         | -700.5638       | -278.0876  | -2.8098          | -2.8090     |
+| 0.1487        | 0.97  | 50   | 0.1319          | -2.9077      | -29.1459          | 0.9375             | 26.2382         | -702.3276       | -278.1449  | -2.8218          | -2.8066     |
+| 0.006         | 1.46  | 75   | 0.1269          | -2.6037      | -29.1519          | 0.9583             | 26.5482         | -702.3289       | -278.0841  | -2.8175          | -2.8037     |
+| 0.0086        | 1.94  | 100  | 0.1099          | -2.9181      | -29.6970          | 0.9271             | 26.7789         | -702.4378       | -278.1470  | -2.8177          | -2.8051     |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.98,
-    "train_loss": 0.7072318076503044,
-    "train_runtime": 2304.549,
+    "train_loss": 0.1742458962578573,
+    "train_runtime": 2590.1164,
     "train_samples": 3296,
-    "train_samples_per_second": 2.86,
-    "train_steps_per_second": 0.044
+    "train_samples_per_second": 2.545,
+    "train_steps_per_second": 0.039
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:49dc2ee422268337033e90b2ac39a226f2a72315d174e4163302f409152af4bb
+oid sha256:d7ff4ce11c67e15a74bdced2857f247219f6d717c7a745efdf77cb2c5e0b5bfd
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:55daecbb744efccf965b5bf66969995029a3a0cffa0e46f0c5947661e04eb026
+oid sha256:f1b2e6888d9714ab406b70141affcde42af62ef635d809e9c047ec04d2a1fb5b
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f40dd84b3355fe1d213a931d0dc794558b88fc24d6725e53ff4288630e52d0d5
+oid sha256:fb0d66e326b5f6c239ab28b5feb6fefc38262e43e830c75fbb97179b361d3360
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.98,
-    "train_loss": 0.7072318076503044,
-    "train_runtime": 2304.549,
+    "train_loss": 0.1742458962578573,
+    "train_runtime": 2590.1164,
     "train_samples": 3296,
-    "train_samples_per_second": 2.86,
-    "train_steps_per_second": 0.044
+    "train_samples_per_second": 2.545,
+    "train_steps_per_second": 0.039
 }

trainer_state.json

@@ -11,10 +11,10 @@
     {
       "epoch": 0.02,
       "learning_rate": 9.09090909090909e-09,
-      "logits/generated": -2.7517285346984863,
-      "logits/real": -2.7709789276123047,
-      "logps/generated": -844.9539794921875,
-      "logps/real": -335.89251708984375,
+      "logits/generated": -2.911376476287842,
+      "logits/real": -2.89278507232666,
+      "logps/generated": -668.016845703125,
+      "logps/real": -324.0830383300781,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/generated": 0.0,
@@ -25,215 +25,215 @@
     {
       "epoch": 0.19,
       "learning_rate": 9.09090909090909e-08,
-      "logits/generated": -2.6863832473754883,
-      "logits/real": -2.7313873767852783,
-      "logps/generated": -753.656005859375,
-      "logps/real": -260.1191711425781,
-      "loss": 0.5813,
-      "rewards/accuracies": 0.7430555820465088,
-      "rewards/generated": -0.3012603223323822,
-      "rewards/margins": 0.32114675641059875,
-      "rewards/real": 0.019886476919054985,
+      "logits/generated": -2.848540782928467,
+      "logits/real": -2.8350822925567627,
+      "logps/generated": -617.5262451171875,
+      "logps/real": -255.12559509277344,
+      "loss": 0.8027,
+      "rewards/accuracies": 0.6527777910232544,
+      "rewards/generated": -6.766346454620361,
+      "rewards/margins": 4.9929962158203125,
+      "rewards/real": -1.7733497619628906,
       "step": 10
     },
     {
       "epoch": 0.39,
       "learning_rate": 9.010989010989011e-08,
-      "logits/generated": -2.4782943725585938,
-      "logits/real": -2.5394017696380615,
-      "logps/generated": -1096.2259521484375,
-      "logps/real": -633.4000854492188,
-      "loss": 5.8769,
-      "rewards/accuracies": 0.8999999761581421,
-      "rewards/generated": -33.22383499145508,
-      "rewards/margins": -1.7550216913223267,
-      "rewards/real": -34.978858947753906,
+      "logits/generated": -2.8388895988464355,
+      "logits/real": -2.8330814838409424,
+      "logps/generated": -707.9961547851562,
+      "logps/real": -283.4620056152344,
+      "loss": 0.2928,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/generated": -20.150367736816406,
+      "rewards/margins": 17.525970458984375,
+      "rewards/real": -2.6243953704833984,
       "step": 20
     },
     {
       "epoch": 0.49,
-      "eval_logits/generated": -2.7939765453338623,
-      "eval_logits/real": -2.838216781616211,
-      "eval_logps/generated": -719.6648559570312,
-      "eval_logps/real": -274.2817077636719,
-      "eval_loss": 0.18901990354061127,
-      "eval_rewards/accuracies": 0.9375,
-      "eval_rewards/generated": -2.9832763671875,
-      "eval_rewards/margins": 2.815262794494629,
-      "eval_rewards/real": -0.16801361739635468,
-      "eval_runtime": 56.889,
-      "eval_samples_per_second": 6.469,
-      "eval_steps_per_second": 0.211,
+      "eval_logits/generated": -2.8098323345184326,
+      "eval_logits/real": -2.8089659214019775,
+      "eval_logps/generated": -700.5637817382812,
+      "eval_logps/real": -278.0876159667969,
+      "eval_loss": 0.3950892686843872,
+      "eval_rewards/accuracies": 0.90625,
+      "eval_rewards/generated": -20.326812744140625,
+      "eval_rewards/margins": 17.705642700195312,
+      "eval_rewards/real": -2.6211698055267334,
+      "eval_runtime": 75.909,
+      "eval_samples_per_second": 4.848,
+      "eval_steps_per_second": 0.158,
       "step": 25
     },
     {
       "epoch": 0.58,
       "learning_rate": 7.912087912087911e-08,
-      "logits/generated": -2.823836088180542,
-      "logits/real": -2.8936166763305664,
-      "logps/generated": -856.4396362304688,
-      "logps/real": -293.3896179199219,
-      "loss": 0.1538,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -3.060887336730957,
-      "rewards/margins": 2.894258975982666,
-      "rewards/real": -0.1666283905506134,
+      "logits/generated": -2.877678394317627,
+      "logits/real": -2.8707680702209473,
+      "logps/generated": -755.6437377929688,
+      "logps/real": -290.04058837890625,
+      "loss": 0.2615,
+      "rewards/accuracies": 0.96875,
+      "rewards/generated": -25.02899742126465,
+      "rewards/margins": 23.474206924438477,
+      "rewards/real": -1.5547949075698853,
       "step": 30
     },
     {
       "epoch": 0.78,
       "learning_rate": 6.813186813186813e-08,
-      "logits/generated": -2.8231825828552246,
-      "logits/real": -2.8379809856414795,
-      "logps/generated": -769.75048828125,
-      "logps/real": -256.47027587890625,
-      "loss": 0.1241,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/generated": -3.725442409515381,
-      "rewards/margins": 3.5178775787353516,
-      "rewards/real": -0.20756463706493378,
+      "logits/generated": -2.839968681335449,
+      "logits/real": -2.8121862411499023,
+      "logps/generated": -673.220703125,
+      "logps/real": -257.77001953125,
+      "loss": 0.1816,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/generated": -24.732364654541016,
+      "rewards/margins": 26.96199607849121,
+      "rewards/real": 2.2296276092529297,
       "step": 40
     },
     {
       "epoch": 0.97,
       "learning_rate": 5.714285714285714e-08,
-      "logits/generated": -2.8316774368286133,
-      "logits/real": -2.8536953926086426,
-      "logps/generated": -803.0462646484375,
-      "logps/real": -274.77337646484375,
-      "loss": 0.1202,
-      "rewards/accuracies": 0.949999988079071,
-      "rewards/generated": -3.983602523803711,
-      "rewards/margins": 3.5648624897003174,
-      "rewards/real": -0.4187401831150055,
+      "logits/generated": -2.8406929969787598,
+      "logits/real": -2.8268864154815674,
+      "logps/generated": -727.8557739257812,
+      "logps/real": -274.7761535644531,
+      "loss": 0.1487,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/generated": -29.384685516357422,
+      "rewards/margins": 30.1076717376709,
+      "rewards/real": 0.7229812145233154,
       "step": 50
     },
     {
       "epoch": 0.97,
-      "eval_logits/generated": -2.83947491645813,
-      "eval_logits/real": -2.8439435958862305,
-      "eval_logps/generated": -732.087890625,
-      "eval_logps/real": -276.7652282714844,
-      "eval_loss": 0.14404349029064178,
-      "eval_rewards/accuracies": 0.9479166865348816,
-      "eval_rewards/generated": -4.225581645965576,
-      "eval_rewards/margins": 3.809215784072876,
-      "eval_rewards/real": -0.41636598110198975,
-      "eval_runtime": 55.8686,
-      "eval_samples_per_second": 6.587,
-      "eval_steps_per_second": 0.215,
+      "eval_logits/generated": -2.821841239929199,
+      "eval_logits/real": -2.8065903186798096,
+      "eval_logps/generated": -702.32763671875,
+      "eval_logps/real": -278.1448974609375,
+      "eval_loss": 0.131888747215271,
+      "eval_rewards/accuracies": 0.9375,
+      "eval_rewards/generated": -29.145910263061523,
+      "eval_rewards/margins": 26.238248825073242,
+      "eval_rewards/real": -2.9076578617095947,
+      "eval_runtime": 74.4579,
+      "eval_samples_per_second": 4.942,
+      "eval_steps_per_second": 0.161,
       "step": 50
     },
     {
       "epoch": 1.17,
       "learning_rate": 4.615384615384615e-08,
-      "logits/generated": -2.8372151851654053,
-      "logits/real": -2.857466697692871,
-      "logps/generated": -920.998046875,
-      "logps/real": -288.32086181640625,
-      "loss": 0.0744,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -4.382575035095215,
-      "rewards/margins": 4.1766462326049805,
-      "rewards/real": -0.20592932403087616,
+      "logits/generated": -2.8634510040283203,
+      "logits/real": -2.8573107719421387,
+      "logps/generated": -768.2184448242188,
+      "logps/real": -281.481201171875,
+      "loss": 0.0249,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/generated": -33.38450241088867,
+      "rewards/margins": 33.606407165527344,
+      "rewards/real": 0.22189739346504211,
       "step": 60
     },
     {
       "epoch": 1.36,
       "learning_rate": 3.516483516483517e-08,
-      "logits/generated": -2.832484483718872,
-      "logits/real": -2.8351359367370605,
-      "logps/generated": -778.0480346679688,
-      "logps/real": -260.9442443847656,
-      "loss": 0.0754,
+      "logits/generated": -2.846198797225952,
+      "logits/real": -2.818643093109131,
+      "logps/generated": -667.6427001953125,
+      "logps/real": -262.6141052246094,
+      "loss": 0.006,
       "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -4.6376495361328125,
-      "rewards/margins": 4.451912879943848,
-      "rewards/real": -0.18573713302612305,
+      "rewards/generated": -29.605731964111328,
+      "rewards/margins": 33.2759895324707,
+      "rewards/real": 3.670259952545166,
       "step": 70
     },
     {
       "epoch": 1.46,
-      "eval_logits/generated": -2.841118574142456,
-      "eval_logits/real": -2.838819742202759,
-      "eval_logps/generated": -737.3972778320312,
-      "eval_logps/real": -278.0699768066406,
-      "eval_loss": 0.1297575831413269,
+      "eval_logits/generated": -2.8174571990966797,
+      "eval_logits/real": -2.8036582469940186,
+      "eval_logps/generated": -702.328857421875,
+      "eval_logps/real": -278.0840759277344,
+      "eval_loss": 0.12692251801490784,
       "eval_rewards/accuracies": 0.9583333134651184,
-      "eval_rewards/generated": -4.7565155029296875,
-      "eval_rewards/margins": 4.2096757888793945,
-      "eval_rewards/real": -0.546840488910675,
-      "eval_runtime": 57.5597,
-      "eval_samples_per_second": 6.393,
-      "eval_steps_per_second": 0.208,
+      "eval_rewards/generated": -29.151870727539062,
+      "eval_rewards/margins": 26.54817008972168,
+      "eval_rewards/real": -2.6037025451660156,
+      "eval_runtime": 74.2997,
+      "eval_samples_per_second": 4.953,
+      "eval_steps_per_second": 0.162,
       "step": 75
     },
     {
       "epoch": 1.55,
       "learning_rate": 2.4175824175824175e-08,
-      "logits/generated": -2.8195009231567383,
-      "logits/real": -2.84141206741333,
-      "logps/generated": -862.6012573242188,
-      "logps/real": -276.63531494140625,
-      "loss": 0.0666,
+      "logits/generated": -2.8320305347442627,
+      "logits/real": -2.810762882232666,
+      "logps/generated": -708.1215209960938,
+      "logps/real": -277.090576171875,
+      "loss": 0.0292,
       "rewards/accuracies": 0.9937499761581421,
-      "rewards/generated": -5.086130619049072,
-      "rewards/margins": 4.704850196838379,
-      "rewards/real": -0.3812801241874695,
+      "rewards/generated": -33.68733215332031,
+      "rewards/margins": 36.958492279052734,
+      "rewards/real": 3.271162748336792,
       "step": 80
     },
     {
       "epoch": 1.75,
       "learning_rate": 1.3186813186813187e-08,
-      "logits/generated": -2.8254876136779785,
-      "logits/real": -2.823962450027466,
-      "logps/generated": -835.1746215820312,
-      "logps/real": -272.4798278808594,
-      "loss": 0.0572,
-      "rewards/accuracies": 1.0,
-      "rewards/generated": -5.038660526275635,
-      "rewards/margins": 4.638853073120117,
-      "rewards/real": -0.39980727434158325,
+      "logits/generated": -2.8205840587615967,
+      "logits/real": -2.8130674362182617,
+      "logps/generated": -704.9281005859375,
+      "logps/real": -273.8406066894531,
+      "loss": 0.0322,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/generated": -31.48798179626465,
+      "rewards/margins": 33.98870849609375,
+      "rewards/real": 2.500725746154785,
       "step": 90
     },
     {
       "epoch": 1.94,
       "learning_rate": 2.197802197802198e-09,
-      "logits/generated": -2.8575081825256348,
-      "logits/real": -2.859795093536377,
-      "logps/generated": -838.92919921875,
-      "logps/real": -267.28704833984375,
-      "loss": 0.0621,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/generated": -5.040513038635254,
-      "rewards/margins": 4.70792818069458,
-      "rewards/real": -0.33258455991744995,
+      "logits/generated": -2.872133255004883,
+      "logits/real": -2.856153964996338,
+      "logps/generated": -711.3128662109375,
+      "logps/real": -259.7333068847656,
+      "loss": 0.0086,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -35.352970123291016,
+      "rewards/margins": 39.97649383544922,
+      "rewards/real": 4.623520851135254,
       "step": 100
     },
     {
       "epoch": 1.94,
-      "eval_logits/generated": -2.84299635887146,
-      "eval_logits/real": -2.8375051021575928,
-      "eval_logps/generated": -739.3701171875,
-      "eval_logps/real": -278.2850646972656,
-      "eval_loss": 0.12532015144824982,
-      "eval_rewards/accuracies": 0.9479166865348816,
-      "eval_rewards/generated": -4.953795433044434,
-      "eval_rewards/margins": 4.385446071624756,
-      "eval_rewards/real": -0.5683497786521912,
-      "eval_runtime": 56.3549,
-      "eval_samples_per_second": 6.53,
-      "eval_steps_per_second": 0.213,
+      "eval_logits/generated": -2.817664384841919,
+      "eval_logits/real": -2.805149793624878,
+      "eval_logps/generated": -702.4378051757812,
+      "eval_logps/real": -278.1470031738281,
+      "eval_loss": 0.10993197560310364,
+      "eval_rewards/accuracies": 0.9270833134651184,
+      "eval_rewards/generated": -29.69696617126465,
+      "eval_rewards/margins": 26.778860092163086,
+      "eval_rewards/real": -2.918107748031616,
+      "eval_runtime": 73.9005,
+      "eval_samples_per_second": 4.98,
+      "eval_steps_per_second": 0.162,
       "step": 100
     },
     {
       "epoch": 1.98,
       "step": 102,
       "total_flos": 0.0,
-      "train_loss": 0.7072318076503044,
-      "train_runtime": 2304.549,
-      "train_samples_per_second": 2.86,
-      "train_steps_per_second": 0.044
+      "train_loss": 0.1742458962578573,
+      "train_runtime": 2590.1164,
+      "train_samples_per_second": 2.545,
+      "train_steps_per_second": 0.039
     }
   ],
   "logging_steps": 10,