{ "metadata": { "ParamSize": 594, "ParamBytes": 8953224192.0, "BitsPerParam": 4.33833552941715 }, "records": [ { "dataPath": "params_shard_0.bin", "format": "raw-shard", "nbytes": 104857600, "records": [ { "name": "lm_head.q_weight", "shape": [ 102400, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 104857600, "byteOffset": 0 } ], "md5sum": "4a468bcc5038248d695c290e9d42877d" }, { "dataPath": "params_shard_1.bin", "format": "raw-shard", "nbytes": 104857600, "records": [ { "name": "model.embed_tokens.q_weight", "shape": [ 102400, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 104857600, "byteOffset": 0 } ], "md5sum": "6f114319338f4c6be823dc1ec3d6a986" }, { "dataPath": "params_shard_2.bin", "format": "raw-shard", "nbytes": 26218496, "records": [ { "name": "lm_head.q_scale", "shape": [ 102400, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 0 }, { "name": "model.embed_tokens.q_scale", "shape": [ 102400, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 13107200, "byteOffset": 13107200 }, { "name": "model.layers.0.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 26214400 } ], "md5sum": "b2ffcb21f36cfbd2c1456b717f5f4344" }, { "dataPath": "params_shard_3.bin", "format": "raw-shard", "nbytes": 22413312, "records": [ { "name": "model.layers.0.mlp.gate_up_proj.q_weight", "shape": [ 21888, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 22413312, "byteOffset": 0 } ], "md5sum": "61d7091d848752d073652041fa266a04" }, { "dataPath": "params_shard_4.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.1.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "c7198c364439fc8fe3e43725944be8c1" }, { "dataPath": "params_shard_5.bin", "format": "raw-shard", "nbytes": 27354112, "records": [ { "name": "model.layers.0.mlp.down_proj.q_weight", "shape": [ 2048, 1368 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 11206656, "byteOffset": 0 }, { "name": "model.layers.0.mlp.down_proj.q_scale", "shape": [ 2048, 342 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1400832, "byteOffset": 11206656 }, { "name": "model.layers.0.mlp.gate_up_proj.q_scale", "shape": [ 21888, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2801664, "byteOffset": 12607488 }, { "name": "model.layers.0.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 15409152 }, { "name": "model.layers.0.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 15413248 }, { "name": "model.layers.0.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 15414272 }, { "name": "model.layers.0.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 16004096 }, { "name": "model.layers.0.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 16077824 }, { "name": "model.layers.0.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 18174976 }, { "name": "model.layers.0.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 20272128 }, { "name": "model.layers.0.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 21320704 }, { "name": "model.layers.0.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 21451776 }, { "name": "model.layers.0.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 23548928 }, { "name": "model.layers.0.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 23811072 }, { "name": "model.layers.0.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 26956800 }, { "name": "model.layers.1.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 27350016 } ], "md5sum": "196f6f16fb3d00ff63f86f0276547ffe" }, { "dataPath": "params_shard_6.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.1.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "6911eb57b091d9f8821cd4fd48183e00" }, { "dataPath": "params_shard_7.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.1.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "6e58910b8151d18f3b23974b593d037c" }, { "dataPath": "params_shard_8.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.2.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "20e1b04ea4edda0477066cbb9b88972a" }, { "dataPath": "params_shard_9.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.1.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.1.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.1.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.1.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.1.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.1.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.1.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.1.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.1.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.1.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.1.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.1.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.1.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.1.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.1.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.1.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.1.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.1.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.2.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "8f5e96020725efbd7e0afbf67f7f678f" }, { "dataPath": "params_shard_10.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.2.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "455c9ef0f34196b78a86db88f2006130" }, { "dataPath": "params_shard_11.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.2.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "63b83d481228213aadf67014a944331a" }, { "dataPath": "params_shard_12.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.3.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "200cf957e20e88f2db62acd104dac7fb" }, { "dataPath": "params_shard_13.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.2.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.2.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.2.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.2.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.2.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.2.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.2.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.2.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.2.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.2.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.2.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.2.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.2.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.2.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.2.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.2.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.2.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.2.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.3.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "6fb7371ffb90b8a809d4d6c50688dff8" }, { "dataPath": "params_shard_14.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.3.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "11348c610c074130d847ffdccc0de5de" }, { "dataPath": "params_shard_15.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.3.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "75ef5d312e73c602762c7799e01ef7f5" }, { "dataPath": "params_shard_16.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.4.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "4b9180d520907ca6ce0ea17b9200d566" }, { "dataPath": "params_shard_17.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.3.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.3.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.3.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.3.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.3.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.3.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.3.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.3.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.3.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.3.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.3.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.3.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.3.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.3.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.3.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.3.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.3.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.3.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.4.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "4f5b8cbae8fe84e54413ec2d66471f8b" }, { "dataPath": "params_shard_18.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.4.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "fadd95bd8eaca8784b289824d7c2cfff" }, { "dataPath": "params_shard_19.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.4.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "f371c4bf6a6a78ad934a5ad24a232aef" }, { "dataPath": "params_shard_20.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.5.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "b1726000425c1b12cd93c7aa2aa8f20e" }, { "dataPath": "params_shard_21.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.4.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.4.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.4.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.4.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.4.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.4.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.4.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.4.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.4.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.4.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.4.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.4.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.4.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.4.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.4.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.4.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.4.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.4.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.5.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "e4e879b1c2a4cc5e780c7645c10f9241" }, { "dataPath": "params_shard_22.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.5.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "dcbacc42022e4e8379d6af2a18f127b0" }, { "dataPath": "params_shard_23.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.5.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "c04f4e8e1690ef6d3d956949f3f3d742" }, { "dataPath": "params_shard_24.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.6.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "e67b245cef88abd034bab9b98e9cf608" }, { "dataPath": "params_shard_25.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.5.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.5.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.5.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.5.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.5.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.5.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.5.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.5.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.5.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.5.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.5.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.5.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.5.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.5.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.5.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.5.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.5.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.5.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.6.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "a0a4fe7beaede86170760cd67229d0fa" }, { "dataPath": "params_shard_26.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.6.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "a2780fff66afa62efa73f264c3dd2e28" }, { "dataPath": "params_shard_27.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.6.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "bfdee4c3ff7a2e86241355c4d94c3ccc" }, { "dataPath": "params_shard_28.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.7.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "3400097090252d5180ee6b662aa0cda5" }, { "dataPath": "params_shard_29.bin", "format": "raw-shard", "nbytes": 33469440, "records": [ { "name": "model.layers.6.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.6.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.6.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.6.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.6.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.6.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.6.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.6.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.6.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.6.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.6.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.6.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.6.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.6.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.6.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.6.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.6.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.6.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 } ], "md5sum": "df4a1b799c9826c645d23e139120fed9" }, { "dataPath": "params_shard_30.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.7.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "f55892b66d63509d700a260e9e4093ed" }, { "dataPath": "params_shard_31.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.7.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "dae599ff7ed6c89c666ab3b633c1b66d" }, { "dataPath": "params_shard_32.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.10.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "baba412fac077c9b64f9ed4f79470249" }, { "dataPath": "params_shard_33.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.7.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.7.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.7.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.7.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.7.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.7.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.7.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21528576 }, { "name": "model.layers.7.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21529600 }, { "name": "model.layers.7.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22119424 }, { "name": "model.layers.7.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22193152 }, { "name": "model.layers.7.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24290304 }, { "name": "model.layers.7.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26387456 }, { "name": "model.layers.7.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27436032 }, { "name": "model.layers.7.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27567104 }, { "name": "model.layers.7.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29664256 }, { "name": "model.layers.7.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29926400 }, { "name": "model.layers.7.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33072128 }, { "name": "model.norm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33465344 }, { "name": "model.layers.10.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "7152b63d5203ceed98f05a7a1b4cf3d8" }, { "dataPath": "params_shard_34.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.10.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "89f59af0937350e67cde252b5325aef2" }, { "dataPath": "params_shard_35.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.10.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "bf0b8dd975522d798dfcf403fd7b207f" }, { "dataPath": "params_shard_36.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.11.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "688b594cf3450eb742cdd4fa0886b7ab" }, { "dataPath": "params_shard_37.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.10.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.10.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.10.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.10.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.10.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.10.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.10.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.10.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.10.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.10.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.10.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.10.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.10.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.10.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.10.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.10.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.10.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.10.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.11.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "90966ead88ed42916c5513b7c0723d7e" }, { "dataPath": "params_shard_38.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.11.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "eb10ff550398c9b69b349b704863dc99" }, { "dataPath": "params_shard_39.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.11.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "faab93bb3e10c820046f3da57ee1d9d1" }, { "dataPath": "params_shard_40.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.12.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "f801024264d291975fa593eecb64f95a" }, { "dataPath": "params_shard_41.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.11.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.11.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.11.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.11.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.11.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.11.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.11.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.11.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.11.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.11.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.11.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.11.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.11.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.11.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.11.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.11.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.11.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.11.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.12.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "8171f1f5168384fa3835c5dca91968e1" }, { "dataPath": "params_shard_42.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.12.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "0e799f44405da3cb66374f9e7ac10ec7" }, { "dataPath": "params_shard_43.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.12.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "06ed6077ea58f01a47e8e5690d88dd89" }, { "dataPath": "params_shard_44.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.13.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "3ad1c998296cd6b6e58f58849a4e4236" }, { "dataPath": "params_shard_45.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.12.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.12.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.12.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.12.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.12.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.12.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.12.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.12.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.12.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.12.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.12.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.12.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.12.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.12.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.12.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.12.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.12.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.12.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.13.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "c78839d066566f804336f49c9e602ef4" }, { "dataPath": "params_shard_46.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.13.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "cf3e74c9fd4650ba02cc7bb41c02d4bd" }, { "dataPath": "params_shard_47.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.13.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "dc33b1e36cce4fb67ff54d090a3376d1" }, { "dataPath": "params_shard_48.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.14.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "01bac8cef8036e885c40b0787c03fac1" }, { "dataPath": "params_shard_49.bin", "format": "raw-shard", "nbytes": 33469440, "records": [ { "name": "model.layers.13.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.13.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.13.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.13.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.13.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.13.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.13.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.13.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.13.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.13.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.13.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.13.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.13.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.13.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.13.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.13.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.13.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.13.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 } ], "md5sum": "ddbd77fa73a7033eadedae99ac9206b8" }, { "dataPath": "params_shard_50.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.14.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "69a3c78fa5b23ed650dd59bf0ff4b206" }, { "dataPath": "params_shard_51.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.14.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "14eaff40a030777072c8d6748c619d17" }, { "dataPath": "params_shard_52.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.8.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "d0a0566ff97efb1bbed9525812b00b9f" }, { "dataPath": "params_shard_53.bin", "format": "raw-shard", "nbytes": 33477632, "records": [ { "name": "model.layers.14.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.14.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.14.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.14.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.14.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.14.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.14.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21528576 }, { "name": "model.layers.14.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21529600 }, { "name": "model.layers.14.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22119424 }, { "name": "model.layers.14.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22193152 }, { "name": "model.layers.14.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24290304 }, { "name": "model.layers.14.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26387456 }, { "name": "model.layers.14.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27436032 }, { "name": "model.layers.14.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27567104 }, { "name": "model.layers.14.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29664256 }, { "name": "model.layers.14.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29926400 }, { "name": "model.layers.14.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33072128 }, { "name": "model.layers.7.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33465344 }, { "name": "model.layers.7.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 }, { "name": "model.layers.8.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33473536 } ], "md5sum": "172116b2b8eb83291bfba5856a470b8a" }, { "dataPath": "params_shard_54.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.8.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "952e9d58295b4685501929625c8c3258" }, { "dataPath": "params_shard_55.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.8.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "9c21c70bff9bb2af39ec275b39e7273a" }, { "dataPath": "params_shard_56.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.9.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "42b94ea9b940de4487c749aaede91fc9" }, { "dataPath": "params_shard_57.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.8.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.8.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.8.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.8.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.8.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.8.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.8.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.8.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.8.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.8.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.8.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.8.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.8.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.8.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.8.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.8.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.8.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.8.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.9.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "647a36317901e622efa1739f48c6c285" }, { "dataPath": "params_shard_58.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.9.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "db1080f7ca68dc51b66372505b1ccac4" }, { "dataPath": "params_shard_59.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.9.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "a98c7ac78aa9bdfc41065131f15431d7" }, { "dataPath": "params_shard_60.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.15.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "819a5c8873f808f1475eccd399294984" }, { "dataPath": "params_shard_61.bin", "format": "raw-shard", "nbytes": 33481728, "records": [ { "name": "model.layers.9.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.9.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.9.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.9.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.9.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.9.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.9.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.9.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.9.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.9.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.9.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.9.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.9.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.9.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.9.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.9.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.9.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.9.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.14.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 }, { "name": "model.layers.14.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33473536 }, { "name": "model.layers.15.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33477632 } ], "md5sum": "c5a83a9e0e63aa8285588305b15b7e22" }, { "dataPath": "params_shard_62.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.15.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "f3cf48099d985cb97592c8ec3f55b639" }, { "dataPath": "params_shard_63.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.15.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "65096cf726cdc1eb136cda0ec2ae669d" }, { "dataPath": "params_shard_64.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.16.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "5e85e22200470b3fab78e8797ab19553" }, { "dataPath": "params_shard_65.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.15.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.15.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.15.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.15.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.15.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.15.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.15.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.15.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.15.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.15.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.15.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.15.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.15.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.15.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.15.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.15.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.15.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.15.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.16.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "e473e58e0d93ef881da8a6ccccf5f9c0" }, { "dataPath": "params_shard_66.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.16.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "7d8dbdecefca7e9a37507fe3fe705521" }, { "dataPath": "params_shard_67.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.16.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "c4c1e6d13e8bffa2f32770215d11de71" }, { "dataPath": "params_shard_68.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.17.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "455ad9d73d28788b34e8c9aa0ca2fcb2" }, { "dataPath": "params_shard_69.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.16.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.16.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.16.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.16.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.16.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.16.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.16.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.16.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.16.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.16.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.16.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.16.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.16.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.16.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.16.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.16.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.16.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.16.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.17.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "08fe03cc28ffefa0a87c79811701ef17" }, { "dataPath": "params_shard_70.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.17.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "3a1834043f066eab6b83cc47661a394c" }, { "dataPath": "params_shard_71.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.17.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "12529b119679fbdca2b3cd2e98e81255" }, { "dataPath": "params_shard_72.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.18.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "b84b6f8cc1184b4f9a187f41097fa87f" }, { "dataPath": "params_shard_73.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.17.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.17.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.17.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.17.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.17.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.17.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.17.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.17.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.17.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.17.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.17.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.17.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.17.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.17.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.17.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.17.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.17.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.17.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.18.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "3c8d10a063cf3eea86329a03eae4dea5" }, { "dataPath": "params_shard_74.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.18.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "5501ba30b9e4a048ef903549927282fb" }, { "dataPath": "params_shard_75.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.18.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "b23b05b768a004a78c42733766aa1ffa" }, { "dataPath": "params_shard_76.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.19.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "faf32894d81d74763f70f6632db11132" }, { "dataPath": "params_shard_77.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.18.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.18.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.18.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.18.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.18.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.18.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.18.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.18.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.18.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.18.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.18.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.18.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.18.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.18.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.18.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.18.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.18.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.18.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.19.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "34b370647ac68ee747748b17677d2d1f" }, { "dataPath": "params_shard_78.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.19.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "24d706ef3385c79d8283c2259f7c1be6" }, { "dataPath": "params_shard_79.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.19.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "90c25f7df79792bda4a51474388ac0b8" }, { "dataPath": "params_shard_80.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.20.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "faf6504a3b292599c4ad1e8966d162cb" }, { "dataPath": "params_shard_81.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.19.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.19.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.19.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.19.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.19.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.19.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.19.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.19.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.19.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.19.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.19.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.19.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.19.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.19.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.19.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.19.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.19.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.19.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.20.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "a20acedf0f6078128ffc2018a7fe2e78" }, { "dataPath": "params_shard_82.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.20.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "15ccc15a659637ea419f2f0469ab6332" }, { "dataPath": "params_shard_83.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.20.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "e6d4b54cf602a2f20494db384e63e23b" }, { "dataPath": "params_shard_84.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.21.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "6823f59b63b3b651a6d038591eec8013" }, { "dataPath": "params_shard_85.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.20.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.20.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.20.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.20.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.20.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.20.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.20.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.20.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.20.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.20.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.20.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.20.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.20.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.20.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.20.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.20.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.20.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.20.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.21.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "56f7e1953d350688f0f2806764b81a36" }, { "dataPath": "params_shard_86.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.21.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "e9ddb2903fbe7ab1c64f2bd91d0f5436" }, { "dataPath": "params_shard_87.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.21.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "345aaa144c6ca7709c5c6bee1b93f6b4" }, { "dataPath": "params_shard_88.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.22.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "d9f9e723524292fec67da4f4b4e1bfd4" }, { "dataPath": "params_shard_89.bin", "format": "raw-shard", "nbytes": 33469440, "records": [ { "name": "model.layers.21.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.21.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.21.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.21.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.21.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.21.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.21.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.21.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.21.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.21.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.21.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.21.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.21.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.21.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.21.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.21.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.21.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.21.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 } ], "md5sum": "59d67a2060973881e6188950efb3e7f5" }, { "dataPath": "params_shard_90.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.22.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "f198e7165d4286e290363fb349a45abd" }, { "dataPath": "params_shard_91.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.22.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "2c1b20cc5ffdff1bd92c54e096423f03" }, { "dataPath": "params_shard_92.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.23.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "98f16dc30cb4a1f69598ba53bf2be31e" }, { "dataPath": "params_shard_93.bin", "format": "raw-shard", "nbytes": 33477632, "records": [ { "name": "model.layers.22.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.22.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.22.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.22.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.22.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.22.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.22.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21528576 }, { "name": "model.layers.22.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21529600 }, { "name": "model.layers.22.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22119424 }, { "name": "model.layers.22.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22193152 }, { "name": "model.layers.22.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24290304 }, { "name": "model.layers.22.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26387456 }, { "name": "model.layers.22.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27436032 }, { "name": "model.layers.22.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27567104 }, { "name": "model.layers.22.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29664256 }, { "name": "model.layers.22.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29926400 }, { "name": "model.layers.22.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33072128 }, { "name": "model.layers.22.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33465344 }, { "name": "model.layers.22.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 }, { "name": "model.layers.23.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33473536 } ], "md5sum": "37e861bb67994ae2e9445f56a3ce7453" }, { "dataPath": "params_shard_94.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.23.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "3819a43c38dbbe056a590b0e0cda0f1d" }, { "dataPath": "params_shard_95.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.23.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "169c6effab39d0862a78da600d2cbd90" }, { "dataPath": "params_shard_96.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.24.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "a61a08fd1f19c742eb4c1794f678e269" }, { "dataPath": "params_shard_97.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.23.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.23.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.23.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.23.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.23.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.23.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.23.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.23.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.23.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.23.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.23.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.23.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.23.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.23.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.23.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.23.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.23.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.23.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.24.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "12595ffea622edf35397d0cc217c2157" }, { "dataPath": "params_shard_98.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.24.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "38d9b0b0615480c7e09f74c0d7c052cc" }, { "dataPath": "params_shard_99.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.24.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "b40965b3cbafd63573c81b32b2c1e868" }, { "dataPath": "params_shard_100.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.25.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "d15ccb0adf8a049991ab87e92a232ac4" }, { "dataPath": "params_shard_101.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.24.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.24.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.24.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.24.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.24.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.24.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.24.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.24.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.24.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.24.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.24.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.24.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.24.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.24.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.24.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.24.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.24.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.24.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.25.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "ad90fab77a382dc5937ef17bdbf43207" }, { "dataPath": "params_shard_102.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.25.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "f756383a0aa08ac96f24372b84e9e603" }, { "dataPath": "params_shard_103.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.25.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "a1b556c1344bffa7ce687442268e3b20" }, { "dataPath": "params_shard_104.bin", "format": "raw-shard", "nbytes": 92274688, "records": [ { "name": "model.layers.26.mlp.moe_down_proj.q_weight", "shape": [ 64, 2048, 176 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 92274688, "byteOffset": 0 } ], "md5sum": "661ff2af5b74d24c283bfa1c48bc60fa" }, { "dataPath": "params_shard_105.bin", "format": "raw-shard", "nbytes": 33473536, "records": [ { "name": "model.layers.25.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.25.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.25.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.25.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.25.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.25.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.25.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.25.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.25.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.25.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.25.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.25.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.25.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.25.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.25.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.25.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.25.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.25.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 }, { "name": "model.layers.26.input_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 33469440 } ], "md5sum": "1d0229a0f632e03f24af29e4229f4072" }, { "dataPath": "params_shard_106.bin", "format": "raw-shard", "nbytes": 184549376, "records": [ { "name": "model.layers.26.mlp.moe_gate_up_proj.q_weight", "shape": [ 64, 2816, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 184549376, "byteOffset": 0 } ], "md5sum": "e998272049382b428afc43f4dc320ff7" }, { "dataPath": "params_shard_107.bin", "format": "raw-shard", "nbytes": 23068672, "records": [ { "name": "model.layers.26.mlp.moe_gate_up_proj.q_scale", "shape": [ 64, 2816, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 23068672, "byteOffset": 0 } ], "md5sum": "c7d2865e443b4b5cb89b20a3f836c5ab" }, { "dataPath": "params_shard_108.bin", "format": "raw-shard", "nbytes": 33469440, "records": [ { "name": "model.layers.26.mlp.moe_down_proj.q_scale", "shape": [ 64, 2048, 44 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 11534336, "byteOffset": 0 }, { "name": "model.layers.26.mlp.gate.weight", "shape": [ 64, 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 11534336 }, { "name": "model.layers.26.mlp.shared_experts.down_proj.q_weight", "shape": [ 2048, 352 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2883584, "byteOffset": 11796480 }, { "name": "model.layers.26.mlp.shared_experts.down_proj.q_scale", "shape": [ 2048, 88 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 360448, "byteOffset": 14680064 }, { "name": "model.layers.26.mlp.shared_experts.gate_up_proj.q_weight", "shape": [ 5632, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 5767168, "byteOffset": 15040512 }, { "name": "model.layers.26.mlp.shared_experts.gate_up_proj.q_scale", "shape": [ 5632, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 720896, "byteOffset": 20807680 }, { "name": "model.layers.26.post_attention_layernorm.weight", "shape": [ 2048 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 4096, "byteOffset": 21528576 }, { "name": "model.layers.26.self_attn.kv_a_layernorm.weight", "shape": [ 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 1024, "byteOffset": 21532672 }, { "name": "model.layers.26.self_attn.kv_a_proj_with_mqa.q_weight", "shape": [ 576, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 589824, "byteOffset": 21533696 }, { "name": "model.layers.26.self_attn.kv_a_proj_with_mqa.q_scale", "shape": [ 576, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 73728, "byteOffset": 22123520 }, { "name": "model.layers.26.self_attn.w_uk", "shape": [ 16, 512, 128 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 22197248 }, { "name": "model.layers.26.self_attn.w_uv", "shape": [ 16, 128, 512 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 24294400 }, { "name": "model.layers.26.self_attn.kv_b_proj.q_weight", "shape": [ 4096, 64 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 1048576, "byteOffset": 26391552 }, { "name": "model.layers.26.self_attn.kv_b_proj.q_scale", "shape": [ 4096, 16 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 131072, "byteOffset": 27440128 }, { "name": "model.layers.26.self_attn.o_proj.q_weight", "shape": [ 2048, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 2097152, "byteOffset": 27571200 }, { "name": "model.layers.26.self_attn.o_proj.q_scale", "shape": [ 2048, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 262144, "byteOffset": 29668352 }, { "name": "model.layers.26.self_attn.q_proj.q_weight", "shape": [ 3072, 256 ], "dtype": "uint32", "format": "f32-to-bf16", "nbytes": 3145728, "byteOffset": 29930496 }, { "name": "model.layers.26.self_attn.q_proj.q_scale", "shape": [ 3072, 64 ], "dtype": "float16", "format": "f32-to-bf16", "nbytes": 393216, "byteOffset": 33076224 } ], "md5sum": "a8a79f847d0d7d5d0a62ba7e8723df75" } ] }