Training in progress, epoch 1

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "/data/horse/ws/ryma833h-DCFT_Shared/huggingface/hub/Qwen/Qwen2.5-7B-Instruct",
+  "_name_or_path": "Qwen/Qwen2.5-7B-Instruct",
   "architectures": [
     "Qwen2ForCausalLM"
   ],

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:561b72ae3bd9daa58dbd310a6f0a024e6243b2545174f14cedd14107ad65524c
+oid sha256:0f2ad04589b6dc72ec76b95b02db4661b9d5e834313eaaa4cf9463e5f43b36d8
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5fc5d5f4503d877e957d0e1f80a4db682d8bfd00ab5134b100ff9b2ff05f4851
+oid sha256:b260d59e122a42972315dc3416b98012d73031a51a39246953bd0cc2927dc43f
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b59b8b9793b1f91c1c59d296bfb5249f4f179f5db8331b3a668fd96cb8411646
+oid sha256:f2eb44b4c5053339b770301f7d87fbc4d7263f68083fca77ec9183e972b49d09
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:53bc45428cbda968ebaddc622f57110e74a6a6203a91b475e3a1ad93b580da57
+oid sha256:8100a42157599910b6ec01fec7ad69d8f9fbdcef534c419fae525b1f853f650a
 size 1089994880

trainer_log.jsonl

@@ -1,3 +1,3 @@
-{"current_steps": 1, "total_steps": 2, "loss": 1.2459, "lr": 2e-05, "epoch": 0.5, "percentage": 50.0, "elapsed_time": "0:01:52", "remaining_time": "0:01:52"}
-{"current_steps": 2, "total_steps": 2, "loss": 1.2028, "lr": 0.0, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:03:34", "remaining_time": "0:00:00"}
-{"current_steps": 2, "total_steps": 2, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:06:35", "remaining_time": "0:00:00"}
+{"current_steps": 1, "total_steps": 2, "loss": 1.2459, "lr": 2e-05, "epoch": 0.5, "percentage": 50.0, "elapsed_time": "0:02:03", "remaining_time": "0:02:03"}
+{"current_steps": 2, "total_steps": 2, "loss": 1.2028, "lr": 0.0, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:03:44", "remaining_time": "0:00:00"}
+{"current_steps": 2, "total_steps": 2, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "0:06:21", "remaining_time": "0:00:00"}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17fb8d58f0092a0ab77418a0643622015fc3f3a34cf3d26034253f51f4af6424
+oid sha256:a090da2170c8c3fe4545affa6dd128b771479f14fa1ee3dede9a1a44101e360e
 size 7288

wandb/debug-internal.log

@@ -1,7 +1,7 @@
-{"time":"2025-04-12T23:40:30.119603299+02:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250412_234030-6n803w1v/logs/debug-core.log"}
-{"time":"2025-04-12T23:40:30.400735107+02:00","level":"INFO","msg":"created new stream","id":"6n803w1v"}
-{"time":"2025-04-12T23:40:30.400829687+02:00","level":"INFO","msg":"stream: started","id":"6n803w1v"}
-{"time":"2025-04-12T23:40:30.400876007+02:00","level":"INFO","msg":"writer: Do: started","stream_id":"6n803w1v"}
-{"time":"2025-04-12T23:40:30.400943917+02:00","level":"INFO","msg":"sender: started","stream_id":"6n803w1v"}
-{"time":"2025-04-12T23:40:30.400957198+02:00","level":"INFO","msg":"handler: started","stream_id":"6n803w1v"}
-{"time":"2025-04-12T23:40:30.678619044+02:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-04-13T00:22:04.825520919+02:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250413_002204-33xvut2k/logs/debug-core.log"}
+{"time":"2025-04-13T00:22:05.128494964+02:00","level":"INFO","msg":"created new stream","id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128573564+02:00","level":"INFO","msg":"stream: started","id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128604494+02:00","level":"INFO","msg":"writer: Do: started","stream_id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128621084+02:00","level":"INFO","msg":"sender: started","stream_id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128629264+02:00","level":"INFO","msg":"handler: started","stream_id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.425669006+02:00","level":"INFO","msg":"Starting system monitor"}

wandb/debug.log

@@ -1,25 +1,25 @@
-2025-04-12 23:40:30,108 INFO    MainThread:400396 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
-2025-04-12 23:40:30,109 INFO    MainThread:400396 [wandb_setup.py:_flush():67] Configure stats pid to 400396
-2025-04-12 23:40:30,109 INFO    MainThread:400396 [wandb_setup.py:_flush():67] Loading settings from /home/ryma833h/.config/wandb/settings
-2025-04-12 23:40:30,109 INFO    MainThread:400396 [wandb_setup.py:_flush():67] Loading settings from /data/horse/ws/ryma833h-DCFT_Shared/dcft_private/wandb/settings
-2025-04-12 23:40:30,109 INFO    MainThread:400396 [wandb_setup.py:_flush():67] Loading settings from environment variables
-2025-04-12 23:40:30,110 INFO    MainThread:400396 [wandb_init.py:setup_run_log_directory():647] Logging user logs to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250412_234030-6n803w1v/logs/debug.log
-2025-04-12 23:40:30,110 INFO    MainThread:400396 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250412_234030-6n803w1v/logs/debug-internal.log
-2025-04-12 23:40:30,110 INFO    MainThread:400396 [wandb_init.py:init():761] calling init triggers
-2025-04-12 23:40:30,110 INFO    MainThread:400396 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+2025-04-13 00:22:04,770 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Configure stats pid to 402765
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Loading settings from /home/ryma833h/.config/wandb/settings
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Loading settings from /data/horse/ws/ryma833h-DCFT_Shared/dcft_private/wandb/settings
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_init.py:setup_run_log_directory():647] Logging user logs to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250413_002204-33xvut2k/logs/debug.log
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250413_002204-33xvut2k/logs/debug-internal.log
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():761] calling init triggers
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
 config: {'_wandb': {}}
-2025-04-12 23:40:30,110 INFO    MainThread:400396 [wandb_init.py:init():784] starting backend
-2025-04-12 23:40:30,110 INFO    MainThread:400396 [wandb_init.py:init():788] sending inform_init request
-2025-04-12 23:40:30,116 INFO    MainThread:400396 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
-2025-04-12 23:40:30,116 INFO    MainThread:400396 [wandb_init.py:init():798] backend started and connected
-2025-04-12 23:40:30,118 INFO    MainThread:400396 [wandb_init.py:init():891] updated telemetry
-2025-04-12 23:40:30,152 INFO    MainThread:400396 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
-2025-04-12 23:40:30,674 INFO    MainThread:400396 [wandb_init.py:init():990] starting run threads in backend
-2025-04-12 23:40:31,056 INFO    MainThread:400396 [wandb_run.py:_console_start():2375] atexit reg
-2025-04-12 23:40:31,057 INFO    MainThread:400396 [wandb_run.py:_redirect():2227] redirect: wrap_raw
-2025-04-12 23:40:31,057 INFO    MainThread:400396 [wandb_run.py:_redirect():2292] Wrapping output streams.
-2025-04-12 23:40:31,057 INFO    MainThread:400396 [wandb_run.py:_redirect():2315] Redirects installed.
-2025-04-12 23:40:31,063 INFO    MainThread:400396 [wandb_init.py:init():1032] run started, returning control to user process
-2025-04-12 23:40:31,064 INFO    MainThread:400396 [wandb_run.py:_config_callback():1261] config_cb None None {'vocab_size': 152064, 'max_position_embeddings': 32768, 'hidden_size': 3584, 'intermediate_size': 18944, 'num_hidden_layers': 28, 'num_attention_heads': 28, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 28, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'rope_scaling': None, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 151643, 'pad_token_id': None, 'eos_token_id': 151645, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '/data/horse/ws/ryma833h-DCFT_Shared/huggingface/hub/Qwen/Qwen2.5-7B-Instruct', '_attn_implementation_autoset': True, 'transformers_version': '4.46.1', 'model_type': 'qwen2', 'output_dir': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192', 'overwrite_output_dir': False, 'do_train': True, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 24, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/runs/Apr12_23-40-04_c126', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'dcft/train/zero3.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 's1k-11-test-192', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 180000000, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False, 'sortish_sampler': False, 'predict_with_generate': False, 'generation_max_length': 16384, 'generation_num_beams': None, 'generation_config': None}
-2025-04-12 23:40:31,066 INFO    MainThread:400396 [wandb_config.py:__setitem__():154] config set model/num_parameters = 0 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7efb74038d00>>
-2025-04-12 23:40:31,066 INFO    MainThread:400396 [wandb_run.py:_config_callback():1261] config_cb model/num_parameters 0 None
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():784] starting backend
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():788] sending inform_init request
+2025-04-13 00:22:04,822 INFO    MainThread:402765 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-04-13 00:22:04,823 INFO    MainThread:402765 [wandb_init.py:init():798] backend started and connected
+2025-04-13 00:22:04,824 INFO    MainThread:402765 [wandb_init.py:init():891] updated telemetry
+2025-04-13 00:22:04,991 INFO    MainThread:402765 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-04-13 00:22:05,420 INFO    MainThread:402765 [wandb_init.py:init():990] starting run threads in backend
+2025-04-13 00:22:07,461 INFO    MainThread:402765 [wandb_run.py:_console_start():2375] atexit reg
+2025-04-13 00:22:07,462 INFO    MainThread:402765 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-04-13 00:22:07,462 INFO    MainThread:402765 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-04-13 00:22:07,462 INFO    MainThread:402765 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-04-13 00:22:07,505 INFO    MainThread:402765 [wandb_init.py:init():1032] run started, returning control to user process
+2025-04-13 00:22:07,506 INFO    MainThread:402765 [wandb_run.py:_config_callback():1261] config_cb None None {'vocab_size': 152064, 'max_position_embeddings': 32768, 'hidden_size': 3584, 'intermediate_size': 18944, 'num_hidden_layers': 28, 'num_attention_heads': 28, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 28, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'rope_scaling': None, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 151643, 'pad_token_id': None, 'eos_token_id': 151645, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'Qwen/Qwen2.5-7B-Instruct', '_attn_implementation_autoset': True, 'transformers_version': '4.46.1', 'model_type': 'qwen2', 'output_dir': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192', 'overwrite_output_dir': False, 'do_train': True, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 24, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/runs/Apr13_00-20-44_c126', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'dcft/train/zero3.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 's1k-11-test-192', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 180000000, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False, 'sortish_sampler': False, 'predict_with_generate': False, 'generation_max_length': 16384, 'generation_num_beams': None, 'generation_config': None}
+2025-04-13 00:22:07,508 INFO    MainThread:402765 [wandb_config.py:__setitem__():154] config set model/num_parameters = 0 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7faa58768340>>
+2025-04-13 00:22:07,508 INFO    MainThread:402765 [wandb_run.py:_config_callback():1261] config_cb model/num_parameters 0 None

wandb/run-20250413_002204-33xvut2k/files/output.log

@@ -0,0 +1,40 @@
+100%|██████████| 2/2 [03:44<00:00, 110.11s/it][INFO|trainer.py:3801] 2025-04-13 00:25:53,959 >> Saving model checkpoint to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2
+{'loss': 1.2459, 'grad_norm': 6.925258567695526, 'learning_rate': 2e-05, 'epoch': 0.5}
+{'loss': 1.2028, 'grad_norm': 7.060579782684043, 'learning_rate': 0.0, 'epoch': 1.0}
+[INFO|configuration_utils.py:414] 2025-04-13 00:25:53,969 >> Configuration saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/config.json
+[INFO|configuration_utils.py:865] 2025-04-13 00:25:53,972 >> Configuration saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/generation_config.json
+[INFO|modeling_utils.py:3043] 2025-04-13 00:26:09,813 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/model.safetensors.index.json.
+[INFO|tokenization_utils_base.py:2646] 2025-04-13 00:26:09,816 >> tokenizer config file saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2655] 2025-04-13 00:26:09,817 >> Special tokens file saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/special_tokens_map.json
+[2025-04-13 00:26:10,062] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step2 is about to be saved!
+[2025-04-13 00:26:10,107] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2025-04-13 00:26:10,107] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2025-04-13 00:26:10,187] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2025-04-13 00:26:10,189] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2025-04-13 00:26:46,139] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2025-04-13 00:26:46,161] [INFO] [engine.py:3536:_save_zero_checkpoint] zero checkpoint saved /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2025-04-13 00:26:47,801] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step2 is ready now!
+[INFO|tokenization_utils_base.py:2646] 2025-04-13 00:27:40,232 >> tokenizer config file saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2655] 2025-04-13 00:27:40,234 >> Special tokens file saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/special_tokens_map.json
+[INFO|trainer.py:3801] 2025-04-13 00:27:42,542 >> Saving model checkpoint to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2
+[INFO|configuration_utils.py:414] 2025-04-13 00:27:42,547 >> Configuration saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/config.json
+[INFO|configuration_utils.py:865] 2025-04-13 00:27:42,549 >> Configuration saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/generation_config.json
+[INFO|modeling_utils.py:3043] 2025-04-13 00:27:57,600 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 4 checkpoint shards. You can find where each parameters has been saved in the index located at /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/model.safetensors.index.json.
+[INFO|tokenization_utils_base.py:2646] 2025-04-13 00:27:57,602 >> tokenizer config file saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2655] 2025-04-13 00:27:57,603 >> Special tokens file saved in /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/special_tokens_map.json
+[2025-04-13 00:27:57,807] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step2 is about to be saved!
+[2025-04-13 00:27:57,815] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/zero_pp_rank_0_mp_rank_00_model_states.pt
+[2025-04-13 00:27:57,815] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/zero_pp_rank_0_mp_rank_00_model_states.pt...
+[2025-04-13 00:27:57,827] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/zero_pp_rank_0_mp_rank_00_model_states.pt.
+[2025-04-13 00:27:57,844] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2025-04-13 00:28:28,367] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2025-04-13 00:28:28,370] [INFO] [engine.py:3536:_save_zero_checkpoint] zero checkpoint saved /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/checkpoint-2/global_step2/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2025-04-13 00:28:28,921] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step2 is ready now!
+[INFO|trainer.py:2584] 2025-04-13 00:28:28,929 >>
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+100%|██████████| 2/2 [06:21<00:00, 190.72s/it]
+{'train_runtime': 384.9677, 'train_samples_per_second': 0.499, 'train_steps_per_second': 0.005, 'train_loss': 1.2243931889533997, 'epoch': 1.0}
+[INFO|trainer.py:4582] 2025-04-13 00:28:28,948 >> Waiting for the current checkpoint push to be finished, this might take a couple of minutes.

wandb/run-20250413_002204-33xvut2k/files/requirements.txt

@@ -0,0 +1,169 @@
+nvidia-cusolver-cu12==11.6.1.9
+greenlet==3.1.1
+Jinja2==3.1.6
+GitPython==3.1.44
+lm_eval==0.4.8
+semantic-version==2.10.0
+Pygments==2.19.1
+nvidia-cufft-cu12==11.2.1.3
+sympy==1.13.1
+charset-normalizer==3.4.1
+pillow==10.4.0
+wandb==0.19.8
+h11==0.14.0
+aiohttp==3.11.14
+datasets==3.1.0
+fonttools==4.56.0
+huggingface-hub==0.29.3
+chardet==5.2.0
+colorama==0.4.6
+sse-starlette==2.2.1
+trl==0.9.6
+tzdata==2025.2
+aiosignal==1.3.2
+Markdown==3.7
+zstandard==0.23.0
+nvidia-nccl-cu12==2.21.5
+tensorboard-data-server==0.7.2
+filelock==3.18.0
+liger_kernel==0.3.1
+msgpack==1.1.0
+gitdb==4.0.12
+wheel==0.45.1
+peft==0.12.0
+mbstrdecoder==1.1.4
+cycler==0.12.1
+tyro==0.9.17
+av==14.2.0
+httpx==0.28.1
+typepy==1.3.4
+pytz==2025.2
+py-cpuinfo==9.0.0
+pydantic==2.10.6
+requests==2.32.3
+typeguard==4.4.2
+dcft==0.1.0
+exceptiongroup==1.2.2
+fsspec==2024.9.0
+nvidia-nvjitlink-cu12==12.4.127
+tensorboard==2.19.0
+tabulate==0.9.0
+tokenizers==0.20.3
+multidict==6.2.0
+python-multipart==0.0.20
+multiprocess==0.70.16
+packaging==24.2
+propcache==0.3.1
+rich==13.9.4
+nltk==3.9.1
+rouge_score==0.1.2
+psutil==7.0.0
+deepspeed==0.15.2
+nvidia-cuda-runtime-cu12==12.4.127
+contourpy==1.3.1
+yarl==1.18.3
+tcolorpy==0.1.7
+mpmath==1.3.0
+Werkzeug==3.1.3
+triton==3.2.0
+xxhash==3.5.0
+pydub==0.25.1
+nvidia-cuda-cupti-cu12==12.4.127
+typer==0.15.2
+joblib==1.4.2
+threadpoolctl==3.6.0
+fire==0.7.0
+kiwisolver==1.4.8
+mdurl==0.1.2
+SQLAlchemy==2.0.39
+PyYAML==6.0.2
+torch==2.6.0
+attrs==25.3.0
+sqlitedict==2.1.0
+portalocker==3.1.1
+setproctitle==1.3.5
+tabledata==1.3.4
+click==8.1.8
+scipy==1.15.2
+tiktoken==0.9.0
+scikit-learn==1.6.1
+pathvalidate==3.2.3
+grpcio==1.71.0
+nvidia-cublas-cu12==12.4.5.8
+lxml==5.3.1
+six==1.17.0
+smmap==5.0.2
+pytablewriter==1.2.1
+nvidia-cudnn-cu12==9.1.0.70
+numexpr==2.10.2
+python-dateutil==2.9.0.post0
+more-itertools==10.6.0
+setuptools==75.8.0
+nvidia-cusparse-cu12==12.3.1.170
+nvidia-cuda-nvrtc-cu12==12.4.127
+certifi==2025.1.31
+protobuf==5.29.4
+importlib_resources==6.5.2
+gradio_client==1.3.0
+starlette==0.46.1
+gradio==4.44.1
+uvicorn==0.34.0
+pandas==2.2.3
+numpy==1.26.4
+markdown-it-py==3.0.0
+torchvision==0.21.0
+ruff==0.11.2
+hjson==3.1.0
+pyarrow==19.0.1
+websockets==12.0
+absl-py==2.2.1
+ffmpy==0.5.0
+termcolor==2.5.0
+sentry-sdk==2.24.1
+tomlkit==0.12.0
+frozenlist==1.5.0
+tqdm-multiprocess==0.0.11
+urllib3==2.3.0
+sentencepiece==0.2.0
+tqdm==4.67.1
+dill==0.3.8
+nvidia-nvtx-cu12==12.4.127
+pyparsing==3.2.3
+fastapi==0.115.12
+shellingham==1.5.4
+annotated-types==0.7.0
+psycopg2-binary==2.9.10
+pybind11==2.13.6
+safetensors==0.5.3
+bitsandbytes==0.45.4
+aiofiles==23.2.1
+matplotlib==3.10.1
+einops==0.8.1
+pip==25.0
+orjson==3.10.16
+idna==3.10
+typing_extensions==4.13.0
+docstring_parser==0.16
+nvidia-cusparselt-cu12==0.6.2
+platformdirs==4.3.7
+pydantic_core==2.27.2
+MarkupSafe==2.1.5
+async-timeout==5.0.1
+word2number==1.1
+accelerate==1.0.1
+anyio==4.9.0
+docker-pycreds==0.4.0
+nvidia-curand-cu12==10.3.5.147
+httpcore==1.0.7
+shtab==1.7.1
+transformers==4.46.1
+DataProperty==1.1.0
+sniffio==1.3.1
+regex==2024.11.6
+jsonlines==4.0.0
+ninja==1.11.1.4
+aiohappyeyeballs==2.6.1
+python-dotenv==1.1.0
+networkx==3.4.2
+evaluate==0.4.3
+sacrebleu==2.5.1

wandb/run-20250413_002204-33xvut2k/files/wandb-metadata.json

@@ -0,0 +1,124 @@
+{
+  "os": "Linux-5.14.0-427.33.1.el9_4.x86_64-x86_64-with-glibc2.34",
+  "python": "CPython 3.10.16",
+  "startedAt": "2025-04-12T22:22:04.823396Z",
+  "args": [
+    "DCFT_experiments/configs/s1k-11-test-192_train_config.yaml"
+  ],
+  "program": "/data/horse/ws/ryma833h-DCFT_Shared/dcft_private/dcft/train/llamafactory/src/train.py",
+  "codePath": "dcft/train/llamafactory/src/train.py",
+  "git": {
+    "remote": "[email protected]:mlfoundations/dcft_private.git",
+    "commit": "aa9216d55a4bbf475343afa165e6a9dd8e34241b"
+  },
+  "email": "[email protected]",
+  "root": "/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192",
+  "host": "c126",
+  "executable": "/data/horse/ws/ryma833h-DCFT_Shared/dcft_private/env/dcft_private/bin/python",
+  "codePathLocal": "dcft/train/llamafactory/src/train.py",
+  "cpu_count": 64,
+  "cpu_count_logical": 64,
+  "gpu": "NVIDIA H100",
+  "gpu_count": 4,
+  "disk": {
+    "/": {
+      "total": "42882564096",
+      "used": "39661744128"
+    }
+  },
+  "memory": {
+    "total": "810822434816"
+  },
+  "cpu": {
+    "count": 64,
+    "countLogical": 64
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    },
+    {
+      "name": "NVIDIA H100",
+      "memoryTotal": "100485038080",
+      "cudaCores": 16896,
+      "architecture": "Hopper"
+    }
+  ],
+  "slurm": {
+    "cluster_name": "capella",
+    "conf": "/etc/slurm/slurm.conf",
+    "cpu_bind": "quiet,mask_cpu:0x0000000FFFFF3F3F",
+    "cpu_bind_list": "0x0000000FFFFF3F3F",
+    "cpu_bind_type": "mask_cpu:",
+    "cpu_bind_verbose": "quiet",
+    "cpus_on_node": "32",
+    "cpus_per_task": "32",
+    "distribution": "cyclic",
+    "gpus_on_node": "4",
+    "gtids": "0",
+    "hint": "nomultithread",
+    "job_account": "p_finetuning",
+    "job_cpus_per_node": "56",
+    "job_end_time": "1744499811",
+    "job_gid": "203360",
+    "job_gpus": "0,1,2,3",
+    "job_id": "200676",
+    "job_name": "s1k-11-test-192",
+    "job_nodelist": "c126",
+    "job_num_nodes": "1",
+    "job_partition": "capella",
+    "job_qos": "normal",
+    "job_start_time": "1744496211",
+    "job_uid": "2215941",
+    "job_user": "ryma833h",
+    "jobid": "200676",
+    "launch_node_ipaddr": "172.24.74.136",
+    "localid": "0",
+    "mem_per_node": "727040",
+    "mpi_type": "pmix",
+    "nnodes": "1",
+    "nodeid": "0",
+    "nodelist": "c126",
+    "nprocs": "1",
+    "ntasks": "1",
+    "pmix_mapping_serv": "(vector,(0,1,1))",
+    "pmixp_abort_agent_port": "61937",
+    "prio_process": "0",
+    "procid": "0",
+    "srun_comm_host": "172.24.74.136",
+    "srun_comm_port": "61938",
+    "step_gpus": "0,1,2,3",
+    "step_id": "0",
+    "step_launcher_port": "61938",
+    "step_nodelist": "c126",
+    "step_num_nodes": "1",
+    "step_num_tasks": "1",
+    "step_tasks_per_node": "1",
+    "stepid": "0",
+    "submit_dir": "/data/horse/ws/ryma833h-DCFT_Shared/dcft_private",
+    "submit_host": "c2",
+    "task_pid": "402713",
+    "tasks_per_node": "1",
+    "threads_per_core": "1",
+    "topology_addr": "spines.leaf8.c126",
+    "topology_addr_pattern": "switch.switch.node",
+    "tres_per_task": "cpu:32",
+    "umask": "0022"
+  },
+  "cudaVersion": "12.6"
+}

wandb/run-20250413_002204-33xvut2k/logs/debug-core.log

@@ -0,0 +1,6 @@
+{"time":"2025-04-13T00:22:04.257805259+02:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpoela46y5/port-402765.txt","pid":402765,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false}
+{"time":"2025-04-13T00:22:04.25833727+02:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":402765}
+{"time":"2025-04-13T00:22:04.25833602+02:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":36323,"Zone":""}}
+{"time":"2025-04-13T00:22:04.372792088+02:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:39750"}
+{"time":"2025-04-13T00:22:04.824089349+02:00","level":"INFO","msg":"handleInformInit: received","streamId":"33xvut2k","id":"127.0.0.1:39750"}
+{"time":"2025-04-13T00:22:05.128600864+02:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"33xvut2k","id":"127.0.0.1:39750"}

wandb/run-20250413_002204-33xvut2k/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-04-13T00:22:04.825520919+02:00","level":"INFO","msg":"stream: starting","core version":"0.19.8","symlink path":"/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250413_002204-33xvut2k/logs/debug-core.log"}
+{"time":"2025-04-13T00:22:05.128494964+02:00","level":"INFO","msg":"created new stream","id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128573564+02:00","level":"INFO","msg":"stream: started","id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128604494+02:00","level":"INFO","msg":"writer: Do: started","stream_id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128621084+02:00","level":"INFO","msg":"sender: started","stream_id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.128629264+02:00","level":"INFO","msg":"handler: started","stream_id":"33xvut2k"}
+{"time":"2025-04-13T00:22:05.425669006+02:00","level":"INFO","msg":"Starting system monitor"}

wandb/run-20250413_002204-33xvut2k/logs/debug.log

@@ -0,0 +1,25 @@
+2025-04-13 00:22:04,770 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Current SDK version is 0.19.8
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Configure stats pid to 402765
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Loading settings from /home/ryma833h/.config/wandb/settings
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Loading settings from /data/horse/ws/ryma833h-DCFT_Shared/dcft_private/wandb/settings
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_setup.py:_flush():67] Loading settings from environment variables
+2025-04-13 00:22:04,778 INFO    MainThread:402765 [wandb_init.py:setup_run_log_directory():647] Logging user logs to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250413_002204-33xvut2k/logs/debug.log
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:setup_run_log_directory():648] Logging internal logs to /data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/wandb/run-20250413_002204-33xvut2k/logs/debug-internal.log
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():761] calling init triggers
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():766] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():784] starting backend
+2025-04-13 00:22:04,779 INFO    MainThread:402765 [wandb_init.py:init():788] sending inform_init request
+2025-04-13 00:22:04,822 INFO    MainThread:402765 [backend.py:_multiprocessing_setup():101] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2025-04-13 00:22:04,823 INFO    MainThread:402765 [wandb_init.py:init():798] backend started and connected
+2025-04-13 00:22:04,824 INFO    MainThread:402765 [wandb_init.py:init():891] updated telemetry
+2025-04-13 00:22:04,991 INFO    MainThread:402765 [wandb_init.py:init():915] communicating run to backend with 90.0 second timeout
+2025-04-13 00:22:05,420 INFO    MainThread:402765 [wandb_init.py:init():990] starting run threads in backend
+2025-04-13 00:22:07,461 INFO    MainThread:402765 [wandb_run.py:_console_start():2375] atexit reg
+2025-04-13 00:22:07,462 INFO    MainThread:402765 [wandb_run.py:_redirect():2227] redirect: wrap_raw
+2025-04-13 00:22:07,462 INFO    MainThread:402765 [wandb_run.py:_redirect():2292] Wrapping output streams.
+2025-04-13 00:22:07,462 INFO    MainThread:402765 [wandb_run.py:_redirect():2315] Redirects installed.
+2025-04-13 00:22:07,505 INFO    MainThread:402765 [wandb_init.py:init():1032] run started, returning control to user process
+2025-04-13 00:22:07,506 INFO    MainThread:402765 [wandb_run.py:_config_callback():1261] config_cb None None {'vocab_size': 152064, 'max_position_embeddings': 32768, 'hidden_size': 3584, 'intermediate_size': 18944, 'num_hidden_layers': 28, 'num_attention_heads': 28, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 28, 'num_key_value_heads': 4, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'rope_scaling': None, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 151643, 'pad_token_id': None, 'eos_token_id': 151645, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'Qwen/Qwen2.5-7B-Instruct', '_attn_implementation_autoset': True, 'transformers_version': '4.46.1', 'model_type': 'qwen2', 'output_dir': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192', 'overwrite_output_dir': False, 'do_train': True, 'do_eval': False, 'do_predict': False, 'eval_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 24, 'eval_accumulation_steps': None, 'eval_delay': 0, 'torch_empty_cache_steps': None, 'learning_rate': 2e-05, 'weight_decay': 0.0, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1.0, 'max_steps': -1, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.1, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192/runs/Apr13_00-20-44_c126', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'epoch', 'save_steps': 500, 'save_total_limit': None, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'restore_callback_states_from_checkpoint': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 4, 'dataloader_prefetch_factor': None, 'past_index': -1, 'run_name': '/data/horse/ws/ryma833h-DCFT_Shared/checkpoints/s1k-11-test-192', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'accelerator_config': {'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True, 'non_blocking': False, 'gradient_accumulation_kwargs': None}, 'deepspeed': 'dcft/train/zero3.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': True, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': True, 'resume_from_checkpoint': None, 'hub_model_id': 's1k-11-test-192', 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': False, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'include_for_metrics': [], 'eval_do_concat_batches': True, 'fp16_backend': 'auto', 'evaluation_strategy': None, 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 180000000, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': None, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None, 'optim_target_modules': None, 'batch_eval_metrics': False, 'eval_on_start': False, 'use_liger_kernel': False, 'eval_use_gather_object': False, 'sortish_sampler': False, 'predict_with_generate': False, 'generation_max_length': 16384, 'generation_num_beams': None, 'generation_config': None}
+2025-04-13 00:22:07,508 INFO    MainThread:402765 [wandb_config.py:__setitem__():154] config set model/num_parameters = 0 - <bound method Run._config_callback of <wandb.sdk.wandb_run.Run object at 0x7faa58768340>>
+2025-04-13 00:22:07,508 INFO    MainThread:402765 [wandb_run.py:_config_callback():1261] config_cb model/num_parameters 0 None