Model save

README.md

@@ -0,0 +1,68 @@
+---
+base_model: Qwen/Qwen2.5-VL-3B-Instruct
+library_name: transformers
+model_name: Qwen2.5-VL-3B-Instruct-Open-R1-Distill
+tags:
+- generated_from_trainer
+- trl
+- grpo
+licence: license
+---
+
+# Model Card for Qwen2.5-VL-3B-Instruct-Open-R1-Distill
+
+This model is a fine-tuned version of [Qwen/Qwen2.5-VL-3B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-3B-Instruct).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="zzhang1987/Qwen2.5-VL-3B-Instruct-Open-R1-Distill", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/causalai/huggingface/runs/wpz01iog) 
+
+
+This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).
+
+### Framework versions
+
+- TRL: 0.15.0.dev0
+- Transformers: 4.49.0.dev0
+- Pytorch: 2.5.1
+- Datasets: 3.2.0
+- Tokenizers: 0.21.0
+
+## Citations
+
+Cite GRPO as:
+
+```bibtex
+@article{zhihong2024deepseekmath,
+    title        = {{DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models}},
+    author       = {Zhihong Shao and Peiyi Wang and Qihao Zhu and Runxin Xu and Junxiao Song and Mingchuan Zhang and Y. K. Li and Y. Wu and Daya Guo},
+    year         = 2024,
+    eprint       = {arXiv:2402.03300},
+}
+
+```
+
+Cite TRL as:
+    
+```bibtex
+@misc{vonwerra2022trl,
+	title        = {{TRL: Transformer Reinforcement Learning}},
+	author       = {Leandro von Werra and Younes Belkada and Lewis Tunstall and Edward Beeching and Tristan Thrush and Nathan Lambert and Shengyi Huang and Kashif Rasul and Quentin Gallouédec},
+	year         = 2020,
+	journal      = {GitHub repository},
+	publisher    = {GitHub},
+	howpublished = {\url{https://github.com/huggingface/trl}}
+}
+```

all_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.8263391004140047,
+    "train_runtime": 308929.2472,
+    "train_samples": 17056,
+    "train_samples_per_second": 0.055,
+    "train_steps_per_second": 0.0
+}

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "attn_implementation": "flash_attention_2",
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "repetition_penalty": 1.05,
+  "temperature": 0.1,
+  "top_k": 1,
+  "top_p": 0.001,
+  "transformers_version": "4.49.0.dev0",
+  "use_cache": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "total_flos": 0.0,
+    "train_loss": 0.8263391004140047,
+    "train_runtime": 308929.2472,
+    "train_samples": 17056,
+    "train_samples_per_second": 0.055,
+    "train_steps_per_second": 0.0
+}

trainer_state.json

@@ -0,0 +1,428 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.99812382739212,
+  "eval_steps": 100,
+  "global_step": 133,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "completion_length": 108.8546875,
+      "epoch": 0.0375234521575985,
+      "grad_norm": 4.688192367553711,
+      "kl": 0.0056383728981018065,
+      "learning_rate": 7.1428571428571436e-06,
+      "loss": 0.0002,
+      "reward": 0.059375,
+      "reward_std": 0.10496962703764438,
+      "rewards/accuracy_reward": 0.01328125,
+      "rewards/format_reward": 0.03203125,
+      "rewards/relaxed_accuracy_reward": 0.0140625,
+      "step": 5
+    },
+    {
+      "completion_length": 86.25859375,
+      "epoch": 0.075046904315197,
+      "grad_norm": 4.734835147857666,
+      "kl": 0.27505035400390626,
+      "learning_rate": 1.4285714285714287e-05,
+      "loss": 0.011,
+      "reward": 0.79375,
+      "reward_std": 0.549490630812943,
+      "rewards/accuracy_reward": 0.093359375,
+      "rewards/format_reward": 0.59375,
+      "rewards/relaxed_accuracy_reward": 0.106640625,
+      "step": 10
+    },
+    {
+      "completion_length": 98.770703125,
+      "epoch": 0.1125703564727955,
+      "grad_norm": 1.7818490266799927,
+      "kl": 0.7628662109375,
+      "learning_rate": 1.9996515418688493e-05,
+      "loss": 0.0305,
+      "reward": 1.2640625,
+      "reward_std": 0.44707948826253413,
+      "rewards/accuracy_reward": 0.166015625,
+      "rewards/format_reward": 0.895703125,
+      "rewards/relaxed_accuracy_reward": 0.20234375,
+      "step": 15
+    },
+    {
+      "completion_length": 71.719140625,
+      "epoch": 0.150093808630394,
+      "grad_norm": 2.765423059463501,
+      "kl": 509.6938110351563,
+      "learning_rate": 1.9874809871741877e-05,
+      "loss": 20.421,
+      "reward": 1.01875,
+      "reward_std": 0.36167241632938385,
+      "rewards/accuracy_reward": 0.122265625,
+      "rewards/format_reward": 0.748046875,
+      "rewards/relaxed_accuracy_reward": 0.1484375,
+      "step": 20
+    },
+    {
+      "completion_length": 142.440625,
+      "epoch": 0.18761726078799248,
+      "grad_norm": 1.2772337198257446,
+      "kl": 0.4162109375,
+      "learning_rate": 1.9581296124106682e-05,
+      "loss": 0.0166,
+      "reward": 1.319921875,
+      "reward_std": 0.4763633420690894,
+      "rewards/accuracy_reward": 0.186328125,
+      "rewards/format_reward": 0.8890625,
+      "rewards/relaxed_accuracy_reward": 0.24453125,
+      "step": 25
+    },
+    {
+      "completion_length": 71.17890625,
+      "epoch": 0.225140712945591,
+      "grad_norm": 1.5595391988754272,
+      "kl": 0.44217529296875,
+      "learning_rate": 1.912108091398988e-05,
+      "loss": 0.0177,
+      "reward": 1.47109375,
+      "reward_std": 0.35589357279241085,
+      "rewards/accuracy_reward": 0.225,
+      "rewards/format_reward": 0.969140625,
+      "rewards/relaxed_accuracy_reward": 0.276953125,
+      "step": 30
+    },
+    {
+      "completion_length": 82.559375,
+      "epoch": 0.2626641651031895,
+      "grad_norm": 2.5247058868408203,
+      "kl": 7.60645751953125,
+      "learning_rate": 1.8502171357296144e-05,
+      "loss": 0.304,
+      "reward": 1.445703125,
+      "reward_std": 0.3477139575406909,
+      "rewards/accuracy_reward": 0.2125,
+      "rewards/format_reward": 0.95234375,
+      "rewards/relaxed_accuracy_reward": 0.280859375,
+      "step": 35
+    },
+    {
+      "completion_length": 153.034375,
+      "epoch": 0.300187617260788,
+      "grad_norm": 0.7359657287597656,
+      "kl": 0.399249267578125,
+      "learning_rate": 1.773533563475053e-05,
+      "loss": 0.016,
+      "reward": 1.30703125,
+      "reward_std": 0.4166026232764125,
+      "rewards/accuracy_reward": 0.17265625,
+      "rewards/format_reward": 0.919921875,
+      "rewards/relaxed_accuracy_reward": 0.214453125,
+      "step": 40
+    },
+    {
+      "completion_length": 75.854296875,
+      "epoch": 0.33771106941838647,
+      "grad_norm": 1.5962355136871338,
+      "kl": 0.36080322265625,
+      "learning_rate": 1.6833915640265485e-05,
+      "loss": 0.0144,
+      "reward": 1.375390625,
+      "reward_std": 0.3114532470703125,
+      "rewards/accuracy_reward": 0.178125,
+      "rewards/format_reward": 0.9796875,
+      "rewards/relaxed_accuracy_reward": 0.217578125,
+      "step": 45
+    },
+    {
+      "completion_length": 136.20546875,
+      "epoch": 0.37523452157598497,
+      "grad_norm": 0.9903507232666016,
+      "kl": 0.46458740234375,
+      "learning_rate": 1.58135948502146e-05,
+      "loss": 0.0186,
+      "reward": 1.396484375,
+      "reward_std": 0.36093369908630846,
+      "rewards/accuracy_reward": 0.205078125,
+      "rewards/format_reward": 0.9546875,
+      "rewards/relaxed_accuracy_reward": 0.23671875,
+      "step": 50
+    },
+    {
+      "completion_length": 233.16171875,
+      "epoch": 0.41275797373358347,
+      "grad_norm": 16.861698150634766,
+      "kl": 2.68153076171875,
+      "learning_rate": 1.4692125452370664e-05,
+      "loss": 0.1073,
+      "reward": 1.103125,
+      "reward_std": 0.5997822197154165,
+      "rewards/accuracy_reward": 0.1609375,
+      "rewards/format_reward": 0.74921875,
+      "rewards/relaxed_accuracy_reward": 0.19296875,
+      "step": 55
+    },
+    {
+      "completion_length": 195.894140625,
+      "epoch": 0.450281425891182,
+      "grad_norm": 1.7692217826843262,
+      "kl": 1.99765625,
+      "learning_rate": 1.348901948209167e-05,
+      "loss": 0.0799,
+      "reward": 1.21171875,
+      "reward_std": 0.6390306279063225,
+      "rewards/accuracy_reward": 0.188671875,
+      "rewards/format_reward": 0.782421875,
+      "rewards/relaxed_accuracy_reward": 0.240625,
+      "step": 60
+    },
+    {
+      "completion_length": 120.77265625,
+      "epoch": 0.4878048780487805,
+      "grad_norm": 24.82339859008789,
+      "kl": 2.957373046875,
+      "learning_rate": 1.2225209339563144e-05,
+      "loss": 0.1184,
+      "reward": 1.422265625,
+      "reward_std": 0.4825815072283149,
+      "rewards/accuracy_reward": 0.2375,
+      "rewards/format_reward": 0.893359375,
+      "rewards/relaxed_accuracy_reward": 0.29140625,
+      "step": 65
+    },
+    {
+      "completion_length": 133.466015625,
+      "epoch": 0.525328330206379,
+      "grad_norm": 10.857051849365234,
+      "kl": 2.94249267578125,
+      "learning_rate": 1.092268359463302e-05,
+      "loss": 0.1177,
+      "reward": 1.394921875,
+      "reward_std": 0.4191150635480881,
+      "rewards/accuracy_reward": 0.21953125,
+      "rewards/format_reward": 0.887109375,
+      "rewards/relaxed_accuracy_reward": 0.28828125,
+      "step": 70
+    },
+    {
+      "completion_length": 106.87109375,
+      "epoch": 0.5628517823639775,
+      "grad_norm": 4.481828689575195,
+      "kl": 1.958642578125,
+      "learning_rate": 9.604104415737309e-06,
+      "loss": 0.0783,
+      "reward": 1.502734375,
+      "reward_std": 0.4382351119071245,
+      "rewards/accuracy_reward": 0.255859375,
+      "rewards/format_reward": 0.918359375,
+      "rewards/relaxed_accuracy_reward": 0.328515625,
+      "step": 75
+    },
+    {
+      "completion_length": 54.36171875,
+      "epoch": 0.600375234521576,
+      "grad_norm": 9.72681999206543,
+      "kl": 1.2845703125,
+      "learning_rate": 8.292413279130625e-06,
+      "loss": 0.0514,
+      "reward": 1.551953125,
+      "reward_std": 0.2952204098924994,
+      "rewards/accuracy_reward": 0.25,
+      "rewards/format_reward": 0.973046875,
+      "rewards/relaxed_accuracy_reward": 0.32890625,
+      "step": 80
+    },
+    {
+      "completion_length": 61.505859375,
+      "epoch": 0.6378986866791745,
+      "grad_norm": 3.227721929550171,
+      "kl": 1.57088623046875,
+      "learning_rate": 7.010431818542298e-06,
+      "loss": 0.0629,
+      "reward": 1.4375,
+      "reward_std": 0.2820946782827377,
+      "rewards/accuracy_reward": 0.2046875,
+      "rewards/format_reward": 0.96640625,
+      "rewards/relaxed_accuracy_reward": 0.26640625,
+      "step": 85
+    },
+    {
+      "completion_length": 84.333984375,
+      "epoch": 0.6754221388367729,
+      "grad_norm": 1.625657558441162,
+      "kl": 1.31497802734375,
+      "learning_rate": 5.780464759928623e-06,
+      "loss": 0.0526,
+      "reward": 1.430078125,
+      "reward_std": 0.3262003194540739,
+      "rewards/accuracy_reward": 0.215625,
+      "rewards/format_reward": 0.948828125,
+      "rewards/relaxed_accuracy_reward": 0.265625,
+      "step": 90
+    },
+    {
+      "completion_length": 115.9625,
+      "epoch": 0.7129455909943715,
+      "grad_norm": 2.6035006046295166,
+      "kl": 1.67415771484375,
+      "learning_rate": 4.623911849714226e-06,
+      "loss": 0.067,
+      "reward": 1.365234375,
+      "reward_std": 0.3999515950679779,
+      "rewards/accuracy_reward": 0.196484375,
+      "rewards/format_reward": 0.930078125,
+      "rewards/relaxed_accuracy_reward": 0.238671875,
+      "step": 95
+    },
+    {
+      "completion_length": 138.86328125,
+      "epoch": 0.7504690431519699,
+      "grad_norm": 5.155971527099609,
+      "kl": 1.94195556640625,
+      "learning_rate": 3.560895528440844e-06,
+      "loss": 0.0777,
+      "reward": 1.417578125,
+      "reward_std": 0.38890475425869225,
+      "rewards/accuracy_reward": 0.22578125,
+      "rewards/format_reward": 0.91796875,
+      "rewards/relaxed_accuracy_reward": 0.273828125,
+      "step": 100
+    },
+    {
+      "epoch": 0.7504690431519699,
+      "eval_completion_length": 102.29363066825776,
+      "eval_kl": 1.1877522980683175,
+      "eval_loss": 0.04750162735581398,
+      "eval_reward": 2.961142601431981,
+      "eval_reward_std": 0.07158712283452541,
+      "eval_rewards/accuracy_reward": 1.0,
+      "eval_rewards/format_reward": 0.9611426014319809,
+      "eval_rewards/relaxed_accuracy_reward": 1.0,
+      "eval_runtime": 25687.3005,
+      "eval_samples_per_second": 0.261,
+      "eval_steps_per_second": 0.065,
+      "step": 100
+    },
+    {
+      "completion_length": 100.34609375,
+      "epoch": 0.7879924953095685,
+      "grad_norm": 1.6533355712890625,
+      "kl": 0.996075439453125,
+      "learning_rate": 2.6099108277934105e-06,
+      "loss": 0.0399,
+      "reward": 1.522265625,
+      "reward_std": 0.3057616015896201,
+      "rewards/accuracy_reward": 0.2515625,
+      "rewards/format_reward": 0.96875,
+      "rewards/relaxed_accuracy_reward": 0.301953125,
+      "step": 105
+    },
+    {
+      "completion_length": 112.546484375,
+      "epoch": 0.8255159474671669,
+      "grad_norm": 3.2183730602264404,
+      "kl": 1.140087890625,
+      "learning_rate": 1.7875035823168641e-06,
+      "loss": 0.0456,
+      "reward": 1.494140625,
+      "reward_std": 0.3347205650061369,
+      "rewards/accuracy_reward": 0.250390625,
+      "rewards/format_reward": 0.95234375,
+      "rewards/relaxed_accuracy_reward": 0.29140625,
+      "step": 110
+    },
+    {
+      "completion_length": 126.648828125,
+      "epoch": 0.8630393996247655,
+      "grad_norm": 1.3182932138442993,
+      "kl": 1.543743896484375,
+      "learning_rate": 1.1079825545001887e-06,
+      "loss": 0.0617,
+      "reward": 1.470703125,
+      "reward_std": 0.3658630719408393,
+      "rewards/accuracy_reward": 0.246875,
+      "rewards/format_reward": 0.93125,
+      "rewards/relaxed_accuracy_reward": 0.292578125,
+      "step": 115
+    },
+    {
+      "completion_length": 119.1421875,
+      "epoch": 0.900562851782364,
+      "grad_norm": 0.7645585536956787,
+      "kl": 1.223272705078125,
+      "learning_rate": 5.831704818578842e-07,
+      "loss": 0.0489,
+      "reward": 1.36796875,
+      "reward_std": 0.33250761572271587,
+      "rewards/accuracy_reward": 0.187109375,
+      "rewards/format_reward": 0.941796875,
+      "rewards/relaxed_accuracy_reward": 0.2390625,
+      "step": 120
+    },
+    {
+      "completion_length": 116.8734375,
+      "epoch": 0.9380863039399625,
+      "grad_norm": 2.7149899005889893,
+      "kl": 1.196661376953125,
+      "learning_rate": 2.2219837744959284e-07,
+      "loss": 0.0479,
+      "reward": 1.36953125,
+      "reward_std": 0.33634590823203325,
+      "rewards/accuracy_reward": 0.184765625,
+      "rewards/format_reward": 0.946875,
+      "rewards/relaxed_accuracy_reward": 0.237890625,
+      "step": 125
+    },
+    {
+      "completion_length": 111.841015625,
+      "epoch": 0.975609756097561,
+      "grad_norm": 1.0174047946929932,
+      "kl": 1.1353759765625,
+      "learning_rate": 3.134666272774034e-08,
+      "loss": 0.0454,
+      "reward": 1.441796875,
+      "reward_std": 0.32110827695578337,
+      "rewards/accuracy_reward": 0.21953125,
+      "rewards/format_reward": 0.951953125,
+      "rewards/relaxed_accuracy_reward": 0.2703125,
+      "step": 130
+    },
+    {
+      "completion_length": 112.2109375,
+      "epoch": 0.99812382739212,
+      "kl": 1.1811625162760417,
+      "reward": 1.55078125,
+      "reward_std": 0.3744206676880519,
+      "rewards/accuracy_reward": 0.2643229166666667,
+      "rewards/format_reward": 0.953125,
+      "rewards/relaxed_accuracy_reward": 0.3333333333333333,
+      "step": 133,
+      "total_flos": 0.0,
+      "train_loss": 0.8263391004140047,
+      "train_runtime": 308929.2472,
+      "train_samples_per_second": 0.055,
+      "train_steps_per_second": 0.0
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 133,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 30,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}