Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,3 @@
 ---
 license: apache-2.0
----
+---

adapter_config.json

@@ -0,0 +1,124 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "ckpt/VQR1-7B-YouTubeUGC",
+  "bias": "none",
+  "corda_config": null,
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 128,
+  "lora_bias": false,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "qalora_group_size": 16,
+  "r": 64,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "layers.18.mlp.down_proj",
+    "layers.3.mlp.down_proj",
+    "layers.25.mlp.gate_proj",
+    "layers.8.mlp.gate_proj",
+    "layers.13.mlp.up_proj",
+    "layers.12.mlp.gate_proj",
+    "layers.14.mlp.down_proj",
+    "layers.17.mlp.gate_proj",
+    "layers.20.mlp.gate_proj",
+    "layers.4.mlp.gate_proj",
+    "layers.13.mlp.down_proj",
+    "layers.20.mlp.up_proj",
+    "layers.25.mlp.up_proj",
+    "layers.15.mlp.gate_proj",
+    "layers.23.mlp.up_proj",
+    "layers.27.mlp.up_proj",
+    "layers.8.mlp.up_proj",
+    "layers.11.mlp.up_proj",
+    "layers.13.mlp.gate_proj",
+    "layers.0.mlp.up_proj",
+    "layers.5.mlp.gate_proj",
+    "layers.7.mlp.up_proj",
+    "layers.0.mlp.gate_proj",
+    "lm_head",
+    "layers.26.mlp.down_proj",
+    "layers.2.mlp.gate_proj",
+    "layers.11.mlp.down_proj",
+    "layers.17.mlp.down_proj",
+    "layers.4.mlp.up_proj",
+    "layers.24.mlp.gate_proj",
+    "layers.15.mlp.up_proj",
+    "v_proj",
+    "layers.19.mlp.gate_proj",
+    "layers.6.mlp.gate_proj",
+    "layers.6.mlp.up_proj",
+    "layers.21.mlp.down_proj",
+    "layers.19.mlp.up_proj",
+    "layers.22.mlp.down_proj",
+    "layers.22.mlp.gate_proj",
+    "layers.25.mlp.down_proj",
+    "layers.18.mlp.gate_proj",
+    "layers.4.mlp.down_proj",
+    "layers.12.mlp.down_proj",
+    "layers.15.mlp.down_proj",
+    "layers.5.mlp.down_proj",
+    "layers.19.mlp.down_proj",
+    "layers.7.mlp.down_proj",
+    "layers.6.mlp.down_proj",
+    "layers.16.mlp.up_proj",
+    "layers.5.mlp.up_proj",
+    "layers.24.mlp.up_proj",
+    "layers.2.mlp.down_proj",
+    "layers.9.mlp.down_proj",
+    "layers.23.mlp.gate_proj",
+    "o_proj",
+    "layers.9.mlp.up_proj",
+    "layers.14.mlp.gate_proj",
+    "layers.11.mlp.gate_proj",
+    "layers.24.mlp.down_proj",
+    "layers.10.mlp.gate_proj",
+    "q_proj",
+    "layers.1.mlp.up_proj",
+    "layers.9.mlp.gate_proj",
+    "layers.21.mlp.gate_proj",
+    "layers.22.mlp.up_proj",
+    "layers.3.mlp.up_proj",
+    "layers.7.mlp.gate_proj",
+    "layers.2.mlp.up_proj",
+    "layers.23.mlp.down_proj",
+    "k_proj",
+    "layers.26.mlp.up_proj",
+    "layers.1.mlp.down_proj",
+    "layers.14.mlp.up_proj",
+    "layers.16.mlp.gate_proj",
+    "layers.27.mlp.down_proj",
+    "layers.18.mlp.up_proj",
+    "layers.27.mlp.gate_proj",
+    "layers.8.mlp.down_proj",
+    "layers.12.mlp.up_proj",
+    "layers.17.mlp.up_proj",
+    "layers.20.mlp.down_proj",
+    "layers.0.mlp.down_proj",
+    "layers.21.mlp.up_proj",
+    "layers.10.mlp.down_proj",
+    "layers.1.mlp.gate_proj",
+    "layers.10.mlp.up_proj",
+    "layers.26.mlp.gate_proj",
+    "layers.3.mlp.gate_proj",
+    "layers.16.mlp.down_proj"
+  ],
+  "target_parameters": null,
+  "task_type": "CAUSAL_LM",
+  "trainable_token_indices": null,
+  "use_dora": false,
+  "use_qalora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:18d3b56eb05151afbe7c8e1af8427f1204a3184bc91a9de14c1ba61fc6398318
+size 1432933648

added_tokens.json

@@ -0,0 +1,24 @@
+{
+  "</tool_call>": 151658,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "{% set image_count = namespace(value=0) %}{% set video_count = namespace(value=0) %}{% for message in messages %}{% if loop.first and message['role'] != 'system' %}<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n{% endif %}<|im_start|>{{ message['role'] }}\n{% if message['content'] is string %}{{ message['content'] }}<|im_end|>\n{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{% set image_count.value = image_count.value + 1 %}{% if add_vision_id %}Picture {{ image_count.value }}: {% endif %}<|vision_start|><|image_pad|><|vision_end|>{% elif content['type'] == 'video' or 'video' in content %}{% set video_count.value = video_count.value + 1 %}{% if add_vision_id %}Video {{ video_count.value }}: {% endif %}<|vision_start|><|video_pad|><|vision_end|>{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}<|im_end|>\n{% endif %}{% endfor %}{% if add_generation_prompt %}<|im_start|>assistant\n{% endif %}"
+}

preprocessor_config.json

@@ -0,0 +1,32 @@
+{
+  "crop_size": null,
+  "default_to_square": true,
+  "do_center_crop": null,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "Qwen2VLImageProcessorFast",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "max_pixels": 12845056,
+  "merge_size": 2,
+  "min_pixels": 3136,
+  "patch_size": 14,
+  "processor_class": "Qwen2_5_VLProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 12845056,
+    "shortest_edge": 3136
+  },
+  "temporal_patch_size": 2
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5eee858c5123a4279c3e1f7b81247343f356ac767940b2692a928ad929543214
+size 11422063

tokenizer_config.json

@@ -0,0 +1,213 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "max_length": null,
+  "model_max_length": 131072,
+  "pad_to_multiple_of": null,
+  "pad_token": "<|endoftext|>",
+  "pad_token_type_id": 0,
+  "padding_side": "left",
+  "processor_class": "Qwen2_5_VLProcessor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,1853 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 10.0,
+  "eval_steps": 500,
+  "global_step": 130,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.359375,
+      "epoch": 0.07692307692307693,
+      "grad_norm": 3.067964792251587,
+      "kl": 0.0,
+      "learning_rate": 9.923076923076923e-06,
+      "loss": 0.3883,
+      "reward": 0.3882647454738617,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2788897454738617,
+      "rewards/format_reward_difficult": 0.109375,
+      "step": 1
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.71875,
+      "epoch": 0.15384615384615385,
+      "grad_norm": 1.5353840589523315,
+      "kl": 0.0025787353515625,
+      "learning_rate": 9.846153846153848e-06,
+      "loss": 0.3236,
+      "reward": 0.32353800535202026,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27666300535202026,
+      "rewards/format_reward_difficult": 0.046875,
+      "step": 2
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 0.23076923076923078,
+      "grad_norm": 0.16299119591712952,
+      "kl": 0.000873565673828125,
+      "learning_rate": 9.76923076923077e-06,
+      "loss": 0.2636,
+      "reward": 0.263530969619751,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.263530969619751,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 3
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 0.3076923076923077,
+      "grad_norm": 0.23624305427074432,
+      "kl": 0.00125885009765625,
+      "learning_rate": 9.692307692307693e-06,
+      "loss": 0.3049,
+      "reward": 0.3048969805240631,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3048969507217407,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 4
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 0.38461538461538464,
+      "grad_norm": 0.2085833102464676,
+      "kl": 0.002227783203125,
+      "learning_rate": 9.615384615384616e-06,
+      "loss": 0.2785,
+      "reward": 0.27844351530075073,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27844351530075073,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 5
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 0.46153846153846156,
+      "grad_norm": 0.19644798338413239,
+      "kl": 0.00408935546875,
+      "learning_rate": 9.53846153846154e-06,
+      "loss": 0.2716,
+      "reward": 0.2713888883590698,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2713889181613922,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 6
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 0.5384615384615384,
+      "grad_norm": 0.3711218237876892,
+      "kl": 0.008056640625,
+      "learning_rate": 9.461538461538463e-06,
+      "loss": 0.3079,
+      "reward": 0.3075915575027466,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3075915575027466,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 7
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 0.6153846153846154,
+      "grad_norm": 0.14840316772460938,
+      "kl": 0.01513671875,
+      "learning_rate": 9.384615384615385e-06,
+      "loss": 0.2807,
+      "reward": 0.2801111340522766,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2801111340522766,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 8
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 0.6923076923076923,
+      "grad_norm": 0.2131115049123764,
+      "kl": 0.0194091796875,
+      "learning_rate": 9.307692307692308e-06,
+      "loss": 0.271,
+      "reward": 0.270263135433197,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.270263135433197,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 9
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 0.7692307692307693,
+      "grad_norm": 0.29820719361305237,
+      "kl": 0.04736328125,
+      "learning_rate": 9.230769230769232e-06,
+      "loss": 0.279,
+      "reward": 0.2771317660808563,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2771317660808563,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 10
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 0.8461538461538461,
+      "grad_norm": 0.24093545973300934,
+      "kl": 0.03271484375,
+      "learning_rate": 9.153846153846155e-06,
+      "loss": 0.2688,
+      "reward": 0.2675073444843292,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2675073444843292,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 11
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.046875,
+      "epoch": 0.9230769230769231,
+      "grad_norm": 0.23784278333187103,
+      "kl": 0.04150390625,
+      "learning_rate": 9.076923076923078e-06,
+      "loss": 0.2827,
+      "reward": 0.2810213565826416,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2810213565826416,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 12
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.066666603088379,
+      "epoch": 1.0,
+      "grad_norm": 0.30485284328460693,
+      "kl": 0.03857421875,
+      "learning_rate": 9e-06,
+      "loss": 0.2886,
+      "reward": 0.3031400442123413,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3031400442123413,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 13
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 1.0769230769230769,
+      "grad_norm": 0.20045126974582672,
+      "kl": 0.037353515625,
+      "learning_rate": 8.923076923076925e-06,
+      "loss": 0.2489,
+      "reward": 0.24742111563682556,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.24742111563682556,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 14
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.078125,
+      "epoch": 1.1538461538461537,
+      "grad_norm": 0.228653684258461,
+      "kl": 0.03857421875,
+      "learning_rate": 8.846153846153847e-06,
+      "loss": 0.2638,
+      "reward": 0.26221930980682373,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26221930980682373,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 15
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 1.2307692307692308,
+      "grad_norm": 0.20209859311580658,
+      "kl": 0.024169921875,
+      "learning_rate": 8.76923076923077e-06,
+      "loss": 0.267,
+      "reward": 0.26602065563201904,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26602068543434143,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 16
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 1.3076923076923077,
+      "grad_norm": 0.21561630070209503,
+      "kl": 0.0150146484375,
+      "learning_rate": 8.692307692307692e-06,
+      "loss": 0.2921,
+      "reward": 0.2915390133857727,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2915390133857727,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 17
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.046875,
+      "epoch": 1.3846153846153846,
+      "grad_norm": 0.21723279356956482,
+      "kl": 0.0086669921875,
+      "learning_rate": 8.615384615384617e-06,
+      "loss": 0.2818,
+      "reward": 0.2814556062221527,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2814556360244751,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 18
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 1.4615384615384617,
+      "grad_norm": 0.15080967545509338,
+      "kl": 0.01300048828125,
+      "learning_rate": 8.53846153846154e-06,
+      "loss": 0.2719,
+      "reward": 0.27134111523628235,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27134114503860474,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 19
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 1.5384615384615383,
+      "grad_norm": 0.19102635979652405,
+      "kl": 0.0096435546875,
+      "learning_rate": 8.461538461538462e-06,
+      "loss": 0.2962,
+      "reward": 0.29586321115493774,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29586321115493774,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 20
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 1.6153846153846154,
+      "grad_norm": 0.24827036261558533,
+      "kl": 0.010498046875,
+      "learning_rate": 8.384615384615385e-06,
+      "loss": 0.2998,
+      "reward": 0.29938018321990967,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29938018321990967,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 21
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 1.6923076923076923,
+      "grad_norm": 0.19604089856147766,
+      "kl": 0.012939453125,
+      "learning_rate": 8.307692307692309e-06,
+      "loss": 0.2509,
+      "reward": 0.2504294514656067,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2504294812679291,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 22
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0625,
+      "epoch": 1.7692307692307692,
+      "grad_norm": 0.3444845676422119,
+      "kl": 0.0250244140625,
+      "learning_rate": 8.230769230769232e-06,
+      "loss": 0.2568,
+      "reward": 0.2558209300041199,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2558209300041199,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 23
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 1.8461538461538463,
+      "grad_norm": 0.20756566524505615,
+      "kl": 0.0184326171875,
+      "learning_rate": 8.153846153846154e-06,
+      "loss": 0.3009,
+      "reward": 0.30018797516822815,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.30018800497055054,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 24
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 1.9230769230769231,
+      "grad_norm": 0.19010891020298004,
+      "kl": 0.020751953125,
+      "learning_rate": 8.076923076923077e-06,
+      "loss": 0.3023,
+      "reward": 0.30143213272094727,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.30143213272094727,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 25
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 2.0,
+      "grad_norm": 0.41885465383529663,
+      "kl": 0.0308837890625,
+      "learning_rate": 8.000000000000001e-06,
+      "loss": 0.3111,
+      "reward": 0.3117591440677643,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3117591440677643,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 26
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 2.076923076923077,
+      "grad_norm": 0.1285719871520996,
+      "kl": 0.0260009765625,
+      "learning_rate": 7.923076923076924e-06,
+      "loss": 0.2447,
+      "reward": 0.24369044601917267,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.24369046092033386,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 27
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.046875,
+      "epoch": 2.1538461538461537,
+      "grad_norm": 0.16132007539272308,
+      "kl": 0.015869140625,
+      "learning_rate": 7.846153846153847e-06,
+      "loss": 0.2806,
+      "reward": 0.28000369668006897,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28000369668006897,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 28
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.078125,
+      "epoch": 2.230769230769231,
+      "grad_norm": 0.17503224313259125,
+      "kl": 0.0341796875,
+      "learning_rate": 7.76923076923077e-06,
+      "loss": 0.2815,
+      "reward": 0.2801356017589569,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2801356017589569,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 29
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 2.3076923076923075,
+      "grad_norm": 0.17042557895183563,
+      "kl": 0.026123046875,
+      "learning_rate": 7.692307692307694e-06,
+      "loss": 0.2729,
+      "reward": 0.27183669805526733,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27183669805526733,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 30
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 2.3846153846153846,
+      "grad_norm": 0.18556250631809235,
+      "kl": 0.0181884765625,
+      "learning_rate": 7.615384615384615e-06,
+      "loss": 0.2647,
+      "reward": 0.2639944553375244,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2639944553375244,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 31
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 2.4615384615384617,
+      "grad_norm": 0.23845872282981873,
+      "kl": 0.013427734375,
+      "learning_rate": 7.538461538461539e-06,
+      "loss": 0.3313,
+      "reward": 0.33079713582992554,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.33079713582992554,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 32
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.046875,
+      "epoch": 2.5384615384615383,
+      "grad_norm": 0.17577950656414032,
+      "kl": 0.0172119140625,
+      "learning_rate": 7.461538461538462e-06,
+      "loss": 0.2814,
+      "reward": 0.28066563606262207,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2806656062602997,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 33
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 2.6153846153846154,
+      "grad_norm": 0.19558458030223846,
+      "kl": 0.01116943359375,
+      "learning_rate": 7.384615384615386e-06,
+      "loss": 0.2447,
+      "reward": 0.24424949288368225,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.24424949288368225,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 34
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.03125,
+      "epoch": 2.6923076923076925,
+      "grad_norm": 0.13503773510456085,
+      "kl": 0.0257568359375,
+      "learning_rate": 7.307692307692308e-06,
+      "loss": 0.3036,
+      "reward": 0.3026180565357208,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3026180565357208,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 35
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 2.769230769230769,
+      "grad_norm": 0.28068050742149353,
+      "kl": 0.0194091796875,
+      "learning_rate": 7.230769230769231e-06,
+      "loss": 0.2955,
+      "reward": 0.29469728469848633,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29469728469848633,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 36
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 2.8461538461538463,
+      "grad_norm": 0.20307907462120056,
+      "kl": 0.0257568359375,
+      "learning_rate": 7.153846153846155e-06,
+      "loss": 0.2911,
+      "reward": 0.29009735584259033,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29009732604026794,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 37
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 2.9230769230769234,
+      "grad_norm": 0.16755792498588562,
+      "kl": 0.0306396484375,
+      "learning_rate": 7.076923076923078e-06,
+      "loss": 0.2883,
+      "reward": 0.28702911734580994,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28702911734580994,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 38
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.022222518920898,
+      "epoch": 3.0,
+      "grad_norm": 0.22273053228855133,
+      "kl": 0.032470703125,
+      "learning_rate": 7e-06,
+      "loss": 0.2886,
+      "reward": 0.29712945222854614,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29712942242622375,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 39
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.076923076923077,
+      "grad_norm": 0.19987061619758606,
+      "kl": 0.036376953125,
+      "learning_rate": 6.923076923076923e-06,
+      "loss": 0.2468,
+      "reward": 0.2453417181968689,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2453417181968689,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 40
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.1538461538461537,
+      "grad_norm": 0.20987683534622192,
+      "kl": 0.0341796875,
+      "learning_rate": 6.846153846153847e-06,
+      "loss": 0.2965,
+      "reward": 0.2951684594154358,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2951684594154358,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 41
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 3.230769230769231,
+      "grad_norm": 0.2177601307630539,
+      "kl": 0.04150390625,
+      "learning_rate": 6.76923076923077e-06,
+      "loss": 0.2819,
+      "reward": 0.28020694851875305,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28020694851875305,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 42
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.3076923076923075,
+      "grad_norm": 0.2715127766132355,
+      "kl": 0.03271484375,
+      "learning_rate": 6.692307692307692e-06,
+      "loss": 0.2727,
+      "reward": 0.2714316248893738,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2714316248893738,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 43
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 3.3846153846153846,
+      "grad_norm": 0.23115268349647522,
+      "kl": 0.033203125,
+      "learning_rate": 6.615384615384616e-06,
+      "loss": 0.3309,
+      "reward": 0.3295760750770569,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3295760750770569,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 44
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.4615384615384617,
+      "grad_norm": 0.1507553905248642,
+      "kl": 0.0262451171875,
+      "learning_rate": 6.538461538461539e-06,
+      "loss": 0.2814,
+      "reward": 0.2803950905799866,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2803950607776642,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 45
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.5384615384615383,
+      "grad_norm": 0.26286324858665466,
+      "kl": 0.0257568359375,
+      "learning_rate": 6.461538461538463e-06,
+      "loss": 0.2699,
+      "reward": 0.2688286006450653,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2688285708427429,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 46
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.6153846153846154,
+      "grad_norm": 0.2555187940597534,
+      "kl": 0.0263671875,
+      "learning_rate": 6.384615384615384e-06,
+      "loss": 0.246,
+      "reward": 0.24495477974414825,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.24495480954647064,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 47
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.046875,
+      "epoch": 3.6923076923076925,
+      "grad_norm": 0.22587725520133972,
+      "kl": 0.0269775390625,
+      "learning_rate": 6.307692307692308e-06,
+      "loss": 0.2992,
+      "reward": 0.2980806231498718,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29808059334754944,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 48
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 3.769230769230769,
+      "grad_norm": 0.16825157403945923,
+      "kl": 0.0223388671875,
+      "learning_rate": 6.230769230769231e-06,
+      "loss": 0.2983,
+      "reward": 0.29743459820747375,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29743462800979614,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 49
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.8461538461538463,
+      "grad_norm": 0.207729771733284,
+      "kl": 0.02001953125,
+      "learning_rate": 6.153846153846155e-06,
+      "loss": 0.2738,
+      "reward": 0.27299824357032776,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27299824357032776,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 50
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 3.9230769230769234,
+      "grad_norm": 0.3381814956665039,
+      "kl": 0.0196533203125,
+      "learning_rate": 6.076923076923077e-06,
+      "loss": 0.3251,
+      "reward": 0.3243548274040222,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3243548274040222,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 51
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.0,
+      "grad_norm": 0.20933787524700165,
+      "kl": 0.0179443359375,
+      "learning_rate": 6e-06,
+      "loss": 0.2583,
+      "reward": 0.2686362564563751,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26863622665405273,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 52
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.076923076923077,
+      "grad_norm": 0.20907750725746155,
+      "kl": 0.0108642578125,
+      "learning_rate": 5.923076923076924e-06,
+      "loss": 0.2991,
+      "reward": 0.298641175031662,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2986412048339844,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 53
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.153846153846154,
+      "grad_norm": 0.2966057062149048,
+      "kl": 0.0162353515625,
+      "learning_rate": 5.846153846153847e-06,
+      "loss": 0.276,
+      "reward": 0.2753836512565613,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2753836512565613,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 54
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 4.230769230769231,
+      "grad_norm": 0.19785088300704956,
+      "kl": 0.00958251953125,
+      "learning_rate": 5.769230769230769e-06,
+      "loss": 0.2787,
+      "reward": 0.2783137559890747,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2783137559890747,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 55
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.3076923076923075,
+      "grad_norm": 0.12805107235908508,
+      "kl": 0.009033203125,
+      "learning_rate": 5.692307692307692e-06,
+      "loss": 0.2793,
+      "reward": 0.27893954515457153,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27893954515457153,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 56
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 4.384615384615385,
+      "grad_norm": 0.18562079966068268,
+      "kl": 0.00592041015625,
+      "learning_rate": 5.615384615384616e-06,
+      "loss": 0.2708,
+      "reward": 0.2705666422843933,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2705666422843933,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 57
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 4.461538461538462,
+      "grad_norm": 0.1820378303527832,
+      "kl": 0.0069580078125,
+      "learning_rate": 5.538461538461539e-06,
+      "loss": 0.2829,
+      "reward": 0.282640665769577,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28264063596725464,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 58
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.538461538461538,
+      "grad_norm": 0.15690194070339203,
+      "kl": 0.007659912109375,
+      "learning_rate": 5.461538461538461e-06,
+      "loss": 0.2871,
+      "reward": 0.2867472767829895,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2867472767829895,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 59
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 4.615384615384615,
+      "grad_norm": 0.15542441606521606,
+      "kl": 0.007659912109375,
+      "learning_rate": 5.384615384615385e-06,
+      "loss": 0.2822,
+      "reward": 0.28193145990371704,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28193148970603943,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 60
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.6923076923076925,
+      "grad_norm": 0.23112843930721283,
+      "kl": 0.006805419921875,
+      "learning_rate": 5.307692307692308e-06,
+      "loss": 0.2989,
+      "reward": 0.29866164922714233,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29866164922714233,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 61
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.769230769230769,
+      "grad_norm": 0.1283213198184967,
+      "kl": 0.006072998046875,
+      "learning_rate": 5.230769230769232e-06,
+      "loss": 0.2429,
+      "reward": 0.24262070655822754,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.24262070655822754,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 62
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.846153846153846,
+      "grad_norm": 0.1647702008485794,
+      "kl": 0.00494384765625,
+      "learning_rate": 5.1538461538461534e-06,
+      "loss": 0.2855,
+      "reward": 0.2852955758571625,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28529560565948486,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 63
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 4.923076923076923,
+      "grad_norm": 0.13007065653800964,
+      "kl": 0.0067138671875,
+      "learning_rate": 5.076923076923077e-06,
+      "loss": 0.2591,
+      "reward": 0.2587839961051941,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2587839961051941,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 64
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.0,
+      "grad_norm": 0.14555510878562927,
+      "kl": 0.005767822265625,
+      "learning_rate": 5e-06,
+      "loss": 0.3081,
+      "reward": 0.31418463587760925,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.31418463587760925,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 65
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 5.076923076923077,
+      "grad_norm": 0.14895249903202057,
+      "kl": 0.007537841796875,
+      "learning_rate": 4.923076923076924e-06,
+      "loss": 0.3003,
+      "reward": 0.3000049889087677,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3000049889087677,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 66
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.153846153846154,
+      "grad_norm": 0.19647394120693207,
+      "kl": 0.0037994384765625,
+      "learning_rate": 4.8461538461538465e-06,
+      "loss": 0.2613,
+      "reward": 0.2611357271671295,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26113569736480713,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 67
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.230769230769231,
+      "grad_norm": 0.30039992928504944,
+      "kl": 0.01031494140625,
+      "learning_rate": 4.76923076923077e-06,
+      "loss": 0.2934,
+      "reward": 0.29294028878211975,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27731528878211975,
+      "rewards/format_reward_difficult": 0.015625,
+      "step": 68
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.3076923076923075,
+      "grad_norm": 0.1374543011188507,
+      "kl": 0.0054931640625,
+      "learning_rate": 4.692307692307693e-06,
+      "loss": 0.253,
+      "reward": 0.25275111198425293,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.25275111198425293,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 69
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.384615384615385,
+      "grad_norm": 0.19475702941417694,
+      "kl": 0.00567626953125,
+      "learning_rate": 4.615384615384616e-06,
+      "loss": 0.2626,
+      "reward": 0.26239192485809326,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26239195466041565,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 70
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.461538461538462,
+      "grad_norm": 0.16289253532886505,
+      "kl": 0.00701904296875,
+      "learning_rate": 4.538461538461539e-06,
+      "loss": 0.2545,
+      "reward": 0.2542119026184082,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2542119026184082,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 71
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.538461538461538,
+      "grad_norm": 0.1631956696510315,
+      "kl": 0.005950927734375,
+      "learning_rate": 4.461538461538462e-06,
+      "loss": 0.295,
+      "reward": 0.2947132885456085,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2947132885456085,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 72
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.615384615384615,
+      "grad_norm": 0.12790323793888092,
+      "kl": 0.00811767578125,
+      "learning_rate": 4.384615384615385e-06,
+      "loss": 0.2917,
+      "reward": 0.2913294732570648,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2913294732570648,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 73
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.6923076923076925,
+      "grad_norm": 0.19049568474292755,
+      "kl": 0.00665283203125,
+      "learning_rate": 4.307692307692308e-06,
+      "loss": 0.2816,
+      "reward": 0.2812873423099518,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2812873423099518,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 74
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.769230769230769,
+      "grad_norm": 0.13436998426914215,
+      "kl": 0.00653076171875,
+      "learning_rate": 4.230769230769231e-06,
+      "loss": 0.2647,
+      "reward": 0.2644004225730896,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2644004225730896,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 75
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.846153846153846,
+      "grad_norm": 0.18125970661640167,
+      "kl": 0.01007080078125,
+      "learning_rate": 4.1538461538461545e-06,
+      "loss": 0.2945,
+      "reward": 0.29406315088272095,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29406312108039856,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 76
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 5.923076923076923,
+      "grad_norm": 0.19487197697162628,
+      "kl": 0.006988525390625,
+      "learning_rate": 4.076923076923077e-06,
+      "loss": 0.2729,
+      "reward": 0.2726302444934845,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2726302444934845,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 77
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.977778434753418,
+      "epoch": 6.0,
+      "grad_norm": 0.29056689143180847,
+      "kl": 0.0108642578125,
+      "learning_rate": 4.000000000000001e-06,
+      "loss": 0.2648,
+      "reward": 0.23201464116573334,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.23201464116573334,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 78
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 6.076923076923077,
+      "grad_norm": 0.21042796969413757,
+      "kl": 0.010986328125,
+      "learning_rate": 3.923076923076923e-06,
+      "loss": 0.2675,
+      "reward": 0.2670348882675171,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2670348882675171,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 79
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 6.153846153846154,
+      "grad_norm": 0.12629854679107666,
+      "kl": 0.006927490234375,
+      "learning_rate": 3.846153846153847e-06,
+      "loss": 0.2532,
+      "reward": 0.25287747383117676,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.25287747383117676,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 80
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.96875,
+      "epoch": 6.230769230769231,
+      "grad_norm": 0.19483327865600586,
+      "kl": 0.008056640625,
+      "learning_rate": 3.7692307692307694e-06,
+      "loss": 0.2691,
+      "reward": 0.26877695322036743,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26877695322036743,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 81
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 6.3076923076923075,
+      "grad_norm": 0.2610100507736206,
+      "kl": 0.00836181640625,
+      "learning_rate": 3.692307692307693e-06,
+      "loss": 0.2651,
+      "reward": 0.26472797989845276,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26472797989845276,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 82
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 6.384615384615385,
+      "grad_norm": 0.2587060332298279,
+      "kl": 0.005859375,
+      "learning_rate": 3.6153846153846156e-06,
+      "loss": 0.2792,
+      "reward": 0.2789880037307739,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27898797392845154,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 83
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 6.461538461538462,
+      "grad_norm": 0.23702670633792877,
+      "kl": 0.01007080078125,
+      "learning_rate": 3.538461538461539e-06,
+      "loss": 0.313,
+      "reward": 0.31260108947753906,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.31260108947753906,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 84
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 6.538461538461538,
+      "grad_norm": 0.1691543310880661,
+      "kl": 0.01068115234375,
+      "learning_rate": 3.4615384615384617e-06,
+      "loss": 0.2804,
+      "reward": 0.28002238273620605,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28002238273620605,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 85
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 6.615384615384615,
+      "grad_norm": 0.20837225019931793,
+      "kl": 0.0052490234375,
+      "learning_rate": 3.384615384615385e-06,
+      "loss": 0.2682,
+      "reward": 0.26800811290740967,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26800811290740967,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 86
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 6.6923076923076925,
+      "grad_norm": 0.2787313163280487,
+      "kl": 0.0118408203125,
+      "learning_rate": 3.307692307692308e-06,
+      "loss": 0.3107,
+      "reward": 0.3102538287639618,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.3102538287639618,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 87
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.9375,
+      "epoch": 6.769230769230769,
+      "grad_norm": 0.1248946487903595,
+      "kl": 0.00909423828125,
+      "learning_rate": 3.2307692307692313e-06,
+      "loss": 0.2667,
+      "reward": 0.26630812883377075,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26630812883377075,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 88
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 6.846153846153846,
+      "grad_norm": 0.27785858511924744,
+      "kl": 0.0062255859375,
+      "learning_rate": 3.153846153846154e-06,
+      "loss": 0.2946,
+      "reward": 0.29435861110687256,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.29435861110687256,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 89
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 6.923076923076923,
+      "grad_norm": 0.16021189093589783,
+      "kl": 0.00848388671875,
+      "learning_rate": 3.0769230769230774e-06,
+      "loss": 0.2792,
+      "reward": 0.278828501701355,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.278828501701355,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 90
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.0,
+      "grad_norm": 0.21517950296401978,
+      "kl": 0.00909423828125,
+      "learning_rate": 3e-06,
+      "loss": 0.2556,
+      "reward": 0.24929477274417877,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.24929475784301758,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 91
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.076923076923077,
+      "grad_norm": 0.19870223104953766,
+      "kl": 0.0091552734375,
+      "learning_rate": 2.9230769230769236e-06,
+      "loss": 0.2609,
+      "reward": 0.2605642080307007,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2605642080307007,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 92
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.153846153846154,
+      "grad_norm": 0.18426446616649628,
+      "kl": 0.0079345703125,
+      "learning_rate": 2.846153846153846e-06,
+      "loss": 0.2854,
+      "reward": 0.2851170301437378,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2851170599460602,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 93
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 7.230769230769231,
+      "grad_norm": 0.19061654806137085,
+      "kl": 0.00970458984375,
+      "learning_rate": 2.7692307692307697e-06,
+      "loss": 0.2844,
+      "reward": 0.2839883863925934,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.283988356590271,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 94
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.3076923076923075,
+      "grad_norm": 0.12485190480947495,
+      "kl": 0.0050048828125,
+      "learning_rate": 2.6923076923076923e-06,
+      "loss": 0.2701,
+      "reward": 0.2698749303817749,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2698749303817749,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 95
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 7.384615384615385,
+      "grad_norm": 0.18052718043327332,
+      "kl": 0.004364013671875,
+      "learning_rate": 2.615384615384616e-06,
+      "loss": 0.2638,
+      "reward": 0.2636146545410156,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2636146545410156,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 96
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.461538461538462,
+      "grad_norm": 0.1336546540260315,
+      "kl": 0.0068359375,
+      "learning_rate": 2.5384615384615385e-06,
+      "loss": 0.2524,
+      "reward": 0.2521340250968933,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2521340548992157,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 97
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.538461538461538,
+      "grad_norm": 0.18070746958255768,
+      "kl": 0.005767822265625,
+      "learning_rate": 2.461538461538462e-06,
+      "loss": 0.2931,
+      "reward": 0.2928406596183777,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2928406596183777,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 98
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.615384615384615,
+      "grad_norm": 0.24408181011676788,
+      "kl": 0.005401611328125,
+      "learning_rate": 2.384615384615385e-06,
+      "loss": 0.286,
+      "reward": 0.28575190901756287,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28575190901756287,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 99
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 7.6923076923076925,
+      "grad_norm": 0.23248958587646484,
+      "kl": 0.00970458984375,
+      "learning_rate": 2.307692307692308e-06,
+      "loss": 0.2668,
+      "reward": 0.2664176821708679,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2664176821708679,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 100
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 7.769230769230769,
+      "grad_norm": 0.17263062298297882,
+      "kl": 0.007415771484375,
+      "learning_rate": 2.230769230769231e-06,
+      "loss": 0.2528,
+      "reward": 0.25248458981513977,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.25248458981513977,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 101
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.846153846153846,
+      "grad_norm": 0.22815687954425812,
+      "kl": 0.008544921875,
+      "learning_rate": 2.153846153846154e-06,
+      "loss": 0.2992,
+      "reward": 0.29885515570640564,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.298855185508728,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 102
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 7.923076923076923,
+      "grad_norm": 0.23697972297668457,
+      "kl": 0.0084228515625,
+      "learning_rate": 2.0769230769230773e-06,
+      "loss": 0.2687,
+      "reward": 0.26833266019821167,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26833266019821167,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 103
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.0,
+      "grad_norm": 0.11839210242033005,
+      "kl": 0.006011962890625,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.2845,
+      "reward": 0.27279049158096313,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27279049158096313,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 104
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.076923076923077,
+      "grad_norm": 0.21847942471504211,
+      "kl": 0.007110595703125,
+      "learning_rate": 1.9230769230769234e-06,
+      "loss": 0.2999,
+      "reward": 0.2996511459350586,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.299651175737381,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 105
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 8.153846153846153,
+      "grad_norm": 0.18120527267456055,
+      "kl": 0.00433349609375,
+      "learning_rate": 1.8461538461538465e-06,
+      "loss": 0.29,
+      "reward": 0.28983789682388306,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28983789682388306,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 106
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.23076923076923,
+      "grad_norm": 0.2113288938999176,
+      "kl": 0.00897216796875,
+      "learning_rate": 1.7692307692307695e-06,
+      "loss": 0.2978,
+      "reward": 0.2974831759929657,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2974831759929657,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 107
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.307692307692308,
+      "grad_norm": 0.3209337294101715,
+      "kl": 0.00860595703125,
+      "learning_rate": 1.6923076923076926e-06,
+      "loss": 0.2395,
+      "reward": 0.23912940919399261,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2391294240951538,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 108
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.384615384615385,
+      "grad_norm": 0.17968839406967163,
+      "kl": 0.00726318359375,
+      "learning_rate": 1.6153846153846157e-06,
+      "loss": 0.2959,
+      "reward": 0.2956576347351074,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2956576347351074,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 109
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.461538461538462,
+      "grad_norm": 0.14308209717273712,
+      "kl": 0.0068359375,
+      "learning_rate": 1.5384615384615387e-06,
+      "loss": 0.2538,
+      "reward": 0.25351810455322266,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.25351810455322266,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 110
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.015625,
+      "epoch": 8.538461538461538,
+      "grad_norm": 0.2102474868297577,
+      "kl": 0.00823974609375,
+      "learning_rate": 1.4615384615384618e-06,
+      "loss": 0.2943,
+      "reward": 0.2939666509628296,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2939666509628296,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 111
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.615384615384615,
+      "grad_norm": 0.33672747015953064,
+      "kl": 0.0089111328125,
+      "learning_rate": 1.3846153846153848e-06,
+      "loss": 0.2506,
+      "reward": 0.25019484758377075,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.25019487738609314,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 112
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.692307692307692,
+      "grad_norm": 0.16569578647613525,
+      "kl": 0.00811767578125,
+      "learning_rate": 1.307692307692308e-06,
+      "loss": 0.2652,
+      "reward": 0.2648388147354126,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2648388147354126,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 113
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.76923076923077,
+      "grad_norm": 0.15254569053649902,
+      "kl": 0.005615234375,
+      "learning_rate": 1.230769230769231e-06,
+      "loss": 0.2409,
+      "reward": 0.24063026905059814,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.24063026905059814,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 114
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 8.846153846153847,
+      "grad_norm": 0.1767226904630661,
+      "kl": 0.00634765625,
+      "learning_rate": 1.153846153846154e-06,
+      "loss": 0.2646,
+      "reward": 0.26436787843704224,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26436787843704224,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 115
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 8.923076923076923,
+      "grad_norm": 0.2110082060098648,
+      "kl": 0.006744384765625,
+      "learning_rate": 1.076923076923077e-06,
+      "loss": 0.278,
+      "reward": 0.2777659595012665,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2777659595012665,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 116
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.0,
+      "grad_norm": 0.19476693868637085,
+      "kl": 0.0059814453125,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.2837,
+      "reward": 0.27837127447128296,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27837127447128296,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 117
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.076923076923077,
+      "grad_norm": 0.1834818571805954,
+      "kl": 0.0096435546875,
+      "learning_rate": 9.230769230769232e-07,
+      "loss": 0.2828,
+      "reward": 0.282396525144577,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.282396525144577,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 118
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 9.153846153846153,
+      "grad_norm": 0.2241596281528473,
+      "kl": 0.005767822265625,
+      "learning_rate": 8.461538461538463e-07,
+      "loss": 0.2372,
+      "reward": 0.2369421124458313,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2369421124458313,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 119
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 9.23076923076923,
+      "grad_norm": 0.16005802154541016,
+      "kl": 0.010498046875,
+      "learning_rate": 7.692307692307694e-07,
+      "loss": 0.2721,
+      "reward": 0.2716958522796631,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2716958522796631,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 120
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.307692307692308,
+      "grad_norm": 0.18658731877803802,
+      "kl": 0.003173828125,
+      "learning_rate": 6.923076923076924e-07,
+      "loss": 0.2627,
+      "reward": 0.26256653666496277,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2625665068626404,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 121
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 9.384615384615385,
+      "grad_norm": 0.20045258104801178,
+      "kl": 0.00506591796875,
+      "learning_rate": 6.153846153846155e-07,
+      "loss": 0.292,
+      "reward": 0.2917952835559845,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2917952835559845,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 122
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.461538461538462,
+      "grad_norm": 0.16684666275978088,
+      "kl": 0.006591796875,
+      "learning_rate": 5.384615384615386e-07,
+      "loss": 0.2954,
+      "reward": 0.2951161861419678,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2951161563396454,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 123
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 9.538461538461538,
+      "grad_norm": 0.1514033079147339,
+      "kl": 0.0069580078125,
+      "learning_rate": 4.615384615384616e-07,
+      "loss": 0.2546,
+      "reward": 0.2542846202850342,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.25428465008735657,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 124
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.615384615384615,
+      "grad_norm": 0.19439052045345306,
+      "kl": 0.00689697265625,
+      "learning_rate": 3.846153846153847e-07,
+      "loss": 0.2646,
+      "reward": 0.26434749364852905,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.26434749364852905,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 125
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 10.984375,
+      "epoch": 9.692307692307692,
+      "grad_norm": 0.1586785763502121,
+      "kl": 0.00848388671875,
+      "learning_rate": 3.0769230769230774e-07,
+      "loss": 0.2862,
+      "reward": 0.2858174741268158,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2858174741268158,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 126
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.76923076923077,
+      "grad_norm": 0.1626785844564438,
+      "kl": 0.00408935546875,
+      "learning_rate": 2.307692307692308e-07,
+      "loss": 0.2757,
+      "reward": 0.27550429105758667,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.27550429105758667,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 127
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.846153846153847,
+      "grad_norm": 0.24578723311424255,
+      "kl": 0.00665283203125,
+      "learning_rate": 1.5384615384615387e-07,
+      "loss": 0.2872,
+      "reward": 0.2869015038013458,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2869015038013458,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 128
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 9.923076923076923,
+      "grad_norm": 0.17852193117141724,
+      "kl": 0.0048828125,
+      "learning_rate": 7.692307692307694e-08,
+      "loss": 0.272,
+      "reward": 0.27175620198249817,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.2717561721801758,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 129
+    },
+    {
+      "clip_ratio": 0.0,
+      "completion_length": 11.0,
+      "epoch": 10.0,
+      "grad_norm": 0.1058979406952858,
+      "kl": 0.00396728515625,
+      "learning_rate": 0.0,
+      "loss": 0.2825,
+      "reward": 0.28774890303611755,
+      "reward_std": 1.0,
+      "rewards/accuracy_loss_difficult": 0.28774890303611755,
+      "rewards/format_reward_difficult": 0.0,
+      "step": 130
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 130,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 10,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:684cd66e3220a809a1a44636e536d2e31c746679548c60a36559317d43772c05
+size 8184