Upload configs

config.json

@@ -0,0 +1,95 @@
+{
+  "activation_type": "swiglu",
+  "alibi": false,
+  "alibi_bias_max": 8.0,
+  "architectures": [
+    "AIGCodeXMoEForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "attention_layer_norm": false,
+  "attention_layer_norm_with_affine": false,
+  "batch_size": 4,
+  "bias_for_layer_norm": false,
+  "block_group_size": 1,
+  "block_type": "sequential",
+  "clip_qkv": null,
+  "d_model": 4096,
+  "deepnorm": false,
+  "embedding_dropout": 0.0,
+  "embedding_size": 65280,
+  "encoder_decoder": false,
+  "eos_token_id": 2,
+  "eval_max_sequence_length": null,
+  "exp_dim_ratio": 1,
+  "flash_attention": false,
+  "gate_level": "token",
+  "gate_sample_ratio": 1,
+  "gate_softmax_temperature": 8.0,
+  "gshard": false,
+  "include_bias": false,
+  "init_cutoff_factor": null,
+  "init_device": "meta",
+  "init_fn": "normal",
+  "init_std": 0.01,
+  "intermediate_size": 16384,
+  "latent_attention": false,
+  "latent_attention_dim": 512,
+  "layer_norm_eps": 1e-05,
+  "layer_norm_type": "default",
+  "layer_norm_with_affine": false,
+  "layer_share": false,
+  "layer_share_mlp_version": 1,
+  "layer_std_check": false,
+  "max_sequence_length": 4096,
+  "mlp_hidden_size": null,
+  "mlp_ratio": 4,
+  "mobile_llm_repeat_num": 1,
+  "model_type": "hf_aigcodexmoe",
+  "moe_act_ckpt_ratio": 1,
+  "moe_auxiliary_loss": false,
+  "moe_auxiliary_loss_weight": 0.0,
+  "moe_batch_prioritized_routing": false,
+  "moe_eval_capacity_token_fraction": 0.25,
+  "moe_expert_count": 4,
+  "moe_expert_count_mluti_level": null,
+  "moe_freq": 2,
+  "moe_freq_pos": 0,
+  "moe_gate_input_type": "concat",
+  "moe_gate_loss_combine_method": "average",
+  "moe_gate_loss_weight": 0.0,
+  "moe_gate_no_grad": false,
+  "moe_gating_use_fp32": true,
+  "moe_logging": false,
+  "moe_normalize_gate_prob_before_dropping": false,
+  "moe_second_expert_policy": "sampling",
+  "moe_share_expert_count": 0,
+  "moe_top1_expert": true,
+  "moe_topn_expert": 1,
+  "moe_version": 1,
+  "multi_query_attention": false,
+  "n_heads": 32,
+  "n_kv_heads": null,
+  "n_layers": 22,
+  "pad_token_id": 0,
+  "ple_layer_num": 0,
+  "ple_layernorm": false,
+  "precision": "amp_bf16",
+  "residual_dropout": 0.0,
+  "rope": true,
+  "rope_base": 30000,
+  "rope_ext_ratio": 1,
+  "rope_full_precision": true,
+  "scale_logits": false,
+  "sft_ans_mask": false,
+  "share_layer_groups": 1,
+  "share_moe_groups": 1,
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.2",
+  "use_cache": true,
+  "use_mobile_llm": false,
+  "use_moe": false,
+  "use_ple": false,
+  "use_xmoe": true,
+  "vocab_size": 64003,
+  "weight_tying": false
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": 2,
+  "pad_token_id": 0,
+  "transformers_version": "4.40.2"
+}

model.safetensors.index.json

@@ -0,0 +1,174 @@
+{
+  "metadata": {
+    "total_size": 30191321088
+  },
+  "weight_map": {
+    "model.transformer.blocks.0.att_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.attn_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.0.ff_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.0.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.1.ff_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.1.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.2.ff_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.2.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.3.ff_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.experts.3.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.0.gate.wg.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.1.att_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.1.attn_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.1.ff_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.1.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.10.att_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.10.attn_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.0.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.0.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.1.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.1.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.2.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.2.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.3.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.10.experts.3.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.10.gate.wg.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.11.att_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.11.attn_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.11.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.11.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.att_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.attn_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.0.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.0.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.1.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.1.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.2.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.2.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.3.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.experts.3.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.12.gate.wg.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.13.att_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.13.attn_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.13.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.13.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.14.att_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.14.attn_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.0.ff_out.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.0.ff_proj.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.1.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.1.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.2.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.2.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.3.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.14.experts.3.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.14.gate.wg.weight": "model-00004-of-00007.safetensors",
+    "model.transformer.blocks.15.att_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.15.attn_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.15.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.15.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.att_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.attn_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.0.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.0.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.1.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.1.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.2.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.2.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.3.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.experts.3.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.16.gate.wg.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.17.att_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.17.attn_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.17.ff_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.17.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.18.att_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.18.attn_out.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.0.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.0.ff_proj.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.1.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.1.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.2.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.2.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.3.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.18.experts.3.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.18.gate.wg.weight": "model-00005-of-00007.safetensors",
+    "model.transformer.blocks.19.att_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.19.attn_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.19.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.19.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.2.att_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.2.attn_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.0.ff_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.0.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.1.ff_out.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.1.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.2.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.2.ff_proj.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.3.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.2.experts.3.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.2.gate.wg.weight": "model-00001-of-00007.safetensors",
+    "model.transformer.blocks.20.att_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.attn_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.0.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.0.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.1.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.1.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.2.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.2.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.3.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.experts.3.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.20.gate.wg.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.21.att_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.21.attn_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.21.ff_out.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.21.ff_proj.weight": "model-00006-of-00007.safetensors",
+    "model.transformer.blocks.3.att_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.3.attn_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.3.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.3.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.att_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.attn_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.0.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.0.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.1.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.1.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.2.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.2.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.3.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.experts.3.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.4.gate.wg.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.5.att_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.5.attn_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.5.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.5.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.6.att_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.6.attn_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.0.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.0.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.1.ff_out.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.1.ff_proj.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.2.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.2.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.3.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.6.experts.3.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.6.gate.wg.weight": "model-00002-of-00007.safetensors",
+    "model.transformer.blocks.7.att_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.7.attn_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.7.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.7.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.att_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.attn_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.0.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.0.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.1.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.1.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.2.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.2.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.3.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.experts.3.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.8.gate.wg.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.9.att_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.9.attn_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.9.ff_out.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.blocks.9.ff_proj.weight": "model-00003-of-00007.safetensors",
+    "model.transformer.ff_out.weight": "model-00007-of-00007.safetensors",
+    "model.transformer.wte.weight": "model-00001-of-00007.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,32 @@
+{
+  "additional_special_tokens": [
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "pad_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": true
+  }
+}

tokenizer_config.json

@@ -0,0 +1,42 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": true,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": true,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+  ],
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "chat_template": "{% set system_message = 'You are a helpful AI assistant.' %}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if loop.index0 == 0 and system_message is defined %}{% set content = eos_token + '<system>' + system_message + eos_token + '\\n' %}{% endif %}{% if message['role'] == 'user' %}{{ '<user>' + + content + eos_token + '\\n' + '<assistant>' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "model_max_length": 4096,
+  "pad_token": "<unk>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<unk>"
+}