Hugging Face's logo

trl-internal-testing
/
tiny-DbrxForCausalLM

Text Generation
Transformers
Safetensors
dbrx
trl
conversational
text-generation-inference
Model card Files
xet
Community
qgallouedec HF Staff commited on
Commit
f88dd1d
·
verified ·
1 Parent(s): 3faf829

Upload DbrxForCausalLM

Browse files
Files changed (3) hide show
  1. config.json +16 -3
  2. generation_config.json +1 -1
  3. model.safetensors +1 -1
config.json CHANGED
@@ -3,12 +3,25 @@
3
  "DbrxForCausalLM"
4
  ],
5
  "attn_config": {
6
- "model_type": ""
 
 
 
 
7
  },
8
  "d_model": 8,
9
  "emb_pdrop": 0.0,
10
  "ffn_config": {
11
- "model_type": ""
 
 
 
 
 
 
 
 
 
12
  },
13
  "initializer_range": 0.02,
14
  "intermediate_size": 32,
@@ -21,7 +34,7 @@
21
  "resid_pdrop": 0.0,
22
  "tie_word_embeddings": false,
23
  "torch_dtype": "float32",
24
- "transformers_version": "4.47.0.dev0",
25
  "use_cache": true,
26
  "vocab_size": 100302
27
  }
 
3
  "DbrxForCausalLM"
4
  ],
5
  "attn_config": {
6
+ "attn_pdrop": 0.0,
7
+ "clip_qkv": null,
8
+ "kv_n_heads": 1,
9
+ "model_type": "",
10
+ "rope_theta": 10000.0
11
  },
12
  "d_model": 8,
13
  "emb_pdrop": 0.0,
14
  "ffn_config": {
15
+ "ffn_act_fn": {
16
+ "name": "silu"
17
+ },
18
+ "ffn_hidden_size": 3584,
19
+ "model_type": "",
20
+ "moe_jitter_eps": null,
21
+ "moe_loss_weight": 0.01,
22
+ "moe_normalize_expert_weights": 1.0,
23
+ "moe_num_experts": 4,
24
+ "moe_top_k": 1
25
  },
26
  "initializer_range": 0.02,
27
  "intermediate_size": 32,
 
34
  "resid_pdrop": 0.0,
35
  "tie_word_embeddings": false,
36
  "torch_dtype": "float32",
37
+ "transformers_version": "4.55.0.dev0",
38
  "use_cache": true,
39
  "vocab_size": 100302
40
  }
generation_config.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
  "_from_model_config": true,
3
- "transformers_version": "4.47.0.dev0"
4
  }
 
1
  {
2
  "_from_model_config": true,
3
+ "transformers_version": "4.55.0.dev0"
4
  }
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:ef906b4a86e653fbd0790ab564ae1f123c15a6b93499604707aee242c7a050cc
3
  size 9175648
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:063227ec2cf1f4b79f7aa7303219218ac50c8f9fbefce44eb4ffc7fa08bca425
3
  size 9175648