upload 12k inference checkpoint

12k/checkpoint.json

@@ -0,0 +1,18 @@
+{
+  "checkpoint_cache_dir": null,
+  "checkpoint_hf": null,
+  "checkpoint_path": "/project/peilab/wam/cosmos3_cy/reports/cosmos3_piper14/cosmos3_action/battery_piper14/battery_piper14_cosmos3_nano_20000step_4gpu_b8_acc1_offline/checkpoints/iter_000012000/model",
+  "checkpoint_type": "dcp",
+  "config_file": "/project/peilab/wam/cosmos3_cy/reports/cosmos3_piper14/cosmos3_action/battery_piper14/battery_piper14_cosmos3_nano_20000step_4gpu_b8_acc1_offline/config.yaml",
+  "config_file_type": "yaml",
+  "credential_path": "credentials/gcp_checkpoint.secret",
+  "experiment": "",
+  "experiment_overrides": [
+    "model.config.diffusion_expert_config.load_weights_from_pretrained=False",
+    "model.config.vlm_config.pretrained_weights.enabled=False",
+    "checkpoint.load_from_object_store.enabled=False"
+  ],
+  "model_memory_bytes": null,
+  "use_ema_weights": true,
+  "vlm_processor_from_checkpoint": false
+}

12k/config.json

@@ -0,0 +1,183 @@
+{
+  "architectures": [
+    "Cosmos3OmniModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "cosmos3.model.Cosmos3OmniConfig",
+    "AutoModel": "cosmos3.model.Cosmos3OmniModel"
+  },
+  "dtype": "bfloat16",
+  "model": {
+    "_recursive_": false,
+    "_target": "omni_mot_model",
+    "config": {
+      "action_gen": true,
+      "activation_checkpointing": {
+        "determinism_check": "default",
+        "mode": "full",
+        "preserve_rng_state": true,
+        "save_ops_regex": [
+          "fmha"
+        ]
+      },
+      "causal_training_strategy": "none",
+      "compile": {
+        "compile_dynamic": true,
+        "compiled_region": "language",
+        "coordinate_descent_tuning": false,
+        "enabled": true,
+        "max_autotune_pointwise": false,
+        "use_cuda_graphs": false
+      },
+      "diffusion_expert_config": {
+        "base_fps": 24,
+        "enable_fps_modulation": true,
+        "load_weights_from_pretrained": true,
+        "max_vae_latent_side_after_patchify": 20,
+        "patch_spatial": 2,
+        "position_embedding_type": "unified_3d_mrope",
+        "rope_h_extrapolation_ratio": 1.0,
+        "rope_t_extrapolation_ratio": 1.0,
+        "rope_w_extrapolation_ratio": 1.0,
+        "sound_base_temporal_compression_factor": null,
+        "timestep_range": 1.0,
+        "unified_3d_mrope_reset_spatial_ids": true,
+        "unified_3d_mrope_temporal_modality_margin": 15000,
+        "vision_temporal_position_mode": "latent_index"
+      },
+      "ema": {
+        "enabled": false,
+        "iteration_shift": 0,
+        "rate": 0.1
+      },
+      "exclude_reasoner_weights_from_checkpoint": false,
+      "fixed_step_sampler_config": null,
+      "input_caption_key": "ai_caption",
+      "input_image_key": "images",
+      "input_video_key": "video",
+      "joint_attn_implementation": "two_way",
+      "latent_downsample_factor": 16,
+      "lbl": {
+        "coeff_gen": null,
+        "coeff_und": null,
+        "method": "local"
+      },
+      "log_enc_time_every_n": 100,
+      "lora_alpha": 32,
+      "lora_enabled": false,
+      "lora_rank": 16,
+      "lora_target_modules": "q_proj_moe_gen,k_proj_moe_gen,v_proj_moe_gen,o_proj_moe_gen",
+      "max_action_dim": 64,
+      "max_num_tokens_after_packing": -1,
+      "natten_parameter_list": null,
+      "net": null,
+      "num_embodiment_domains": 32,
+      "parallelism": {
+        "attention_io_layout": "sequence_sharded",
+        "cfg_parallel_shard_degree": 1,
+        "context_parallel_shard_degree": 1,
+        "data_parallel_replicate_degree": 1,
+        "data_parallel_shard_degree": 4,
+        "enable_inference_mode": false,
+        "fsdp_master_dtype": "float32"
+      },
+      "precision": "bfloat16",
+      "rectified_flow_inference_config": {
+        "num_train_timesteps": 1000,
+        "scheduler_type": "unipc",
+        "shift": 1,
+        "use_dynamic_shifting": false
+      },
+      "rectified_flow_training_config": {
+        "action_loss_weight": 10.0,
+        "image_loss_scale": 1.0,
+        "independent_action_schedule": false,
+        "independent_sound_schedule": false,
+        "loss_scale": 10.0,
+        "normalize_loss_by_active": false,
+        "shift": {
+          "256": 3,
+          "480": 5,
+          "720": 10
+        },
+        "shift_action": null,
+        "shift_sound": null,
+        "sound_loss_scale": null,
+        "train_time_action_distribution": "logitnormal",
+        "train_time_image_distribution": "logitnormal",
+        "train_time_sound_distribution": "logitnormal",
+        "train_time_video_distribution": "waver",
+        "train_time_weight": "uniform",
+        "use_discrete_rf": false,
+        "use_dynamic_shift": false
+      },
+      "resolution": "720",
+      "sound_dim": null,
+      "sound_gen": false,
+      "sound_latent_fps": 25,
+      "sound_tokenizer": null,
+      "state_ch": 48,
+      "state_t": 300,
+      "tokenizer": {
+        "_target": "wan2pt2_vae_interface",
+        "bucket_name": "",
+        "causal": true,
+        "chunk_duration": 93,
+        "encode_bucket_multiple": null,
+        "encode_chunk_frames": {
+          "256": 68,
+          "480": 24,
+          "720": 12
+        },
+        "encode_exact_durations": [
+          33
+        ],
+        "keep_decoder_cache": false,
+        "object_store_credential_path_pretrained": "",
+        "spatial_compression_factor": 16,
+        "temporal_compression_factor": 4,
+        "temporal_window": null,
+        "use_streaming_encode": false,
+        "vae_path": "/project/peilab/wam/cosmos3_cy/external/cosmos/checkpoints/hf_home/hub/models--Wan-AI--Wan2.2-TI2V-5B/snapshots/921dbaf3f1674a56f47e83fb80a34bac8a8f203e/Wan2.2_VAE.pth"
+      },
+      "video_temporal_causal": false,
+      "vision_gen": true,
+      "vlm_config": {
+        "layer_module": "Qwen2MoTDecoderLayer",
+        "model_instance": {
+          "_target": "qwen3_vl_text_for_causal_lm",
+          "config": {
+            "_target": "create_vlm_config",
+            "base_config": {
+              "_target": "qwen3_vl_mot_config_from_json_file",
+              "json_file": "cosmos3://vfm/models/vlm/qwen3_vl/configs/Qwen3-VL-8B-Instruct.json"
+            },
+            "freeze_und": false,
+            "layer_module": "MoTDecoderLayer",
+            "qk_norm_for_text": true,
+            "tie_word_embeddings": true
+          }
+        },
+        "model_name": "Qwen/Qwen3-VL-8B-Instruct",
+        "pretrained_weights": {
+          "backbone_path": "s3://bucket0/cosmos3/pretrained/huggingface/Qwen/Qwen3-VL-8B-Instruct/",
+          "checkpoint_format": null,
+          "credentials_path": "",
+          "enable_gcs_patch_in_boto3": true,
+          "enabled": false
+        },
+        "qk_norm": false,
+        "safetensors_path": "",
+        "tie_word_embeddings": false,
+        "tokenizer": {
+          "_target": "create_qwen2_tokenizer_with_download",
+          "config_variant": "hf",
+          "pretrained_model_name": "Qwen/Qwen3-VL-8B-Instruct"
+        },
+        "use_system_prompt": false
+      }
+    }
+  },
+  "model_type": "cosmos3_omni",
+  "transformers_version": "4.57.6"
+}

12k/model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:531c6f6c30b55db8147719539c2c8e4da6e7ea897e45108a042de4bbb4c55728
+size 4994798328

12k/model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1e8b02dac94f7b105f125c0968dc52c68fa42c47135d45c106b9e1c231c66a5
+size 4983055752

12k/model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c9b8810ee35de8448a53c6ed7f9078ad9acfa8c082b82f0e1af25a1cd0b87c2
+size 4983089216

12k/model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a19dd0c966106d8fd2082315f4a15e2582656139490a27032e3293eda55b6723
+size 4949503184

12k/model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f525efaf0e083056532f0899caf076e98ce78531980545f07d70225f69c37569
+size 4932723808

12k/model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a15ce1829ea694b16641073706dd7e1fc04ce285ac3cbac6449dc4badd63e66
+size 4999868608

12k/model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eae10ca83e98d579736a3160fef4ec1d338bb41ff9b2a2d6a7ae057dda6f7e89
+size 1656173544