| { | |
| "vocab_size": 248320, | |
| "n_meta_tokens": 16, | |
| "sliding_window": 512, | |
| "d_model": 1024, | |
| "d_state": 256, | |
| "d_latent": 2048, | |
| "d_ff": 8192, | |
| "n_heads": 16, | |
| "n_kv_heads": 8, | |
| "n_prelude": 1, | |
| "n_coda": 1, | |
| "max_loop_iters": 6, | |
| "max_seq_len": 2048, | |
| "n_experts": 24, | |
| "n_shared": 1, | |
| "n_experts_tok": 2, | |
| "expert_dim": 4096, | |
| "loop_min": 2, | |
| "loop_max": 6, | |
| "rho_target": 0.37 | |
| } |