| { | |
| "model_type": "glm4_moe_lite", | |
| "src": { | |
| "model_type": "glm4_moe_lite", | |
| "vocab_size": 154880, | |
| "hidden_size": 2048, | |
| "intermediate_size": 10240, | |
| "moe_intermediate_size": 1536, | |
| "num_hidden_layers": 47, | |
| "num_nextn_predict_layers": 1, | |
| "num_attention_heads": 20, | |
| "num_key_value_heads": 20, | |
| "q_lora_rank": 768, | |
| "kv_lora_rank": 512, | |
| "qk_nope_head_dim": 192, | |
| "qk_rope_head_dim": 64, | |
| "v_head_dim": 256, | |
| "n_routed_experts": 64, | |
| "n_shared_experts": 1, | |
| "num_experts_per_tok": 4, | |
| "first_k_dense_replace": 1, | |
| "moe_layer_freq": 1, | |
| "rms_norm_eps": 1e-05, | |
| "tie_word_embeddings": false | |
| }, | |
| "tgt": { | |
| "hidden_size": 2048, | |
| "intermediate_size": 10240, | |
| "moe_intermediate_size": 1536, | |
| "num_hidden_layers": 75, | |
| "rms_norm_eps": 1e-05 | |
| }, | |
| "operators": { | |
| "hidden_operator": "vzero", | |
| "mlp_operator": "dormant_prime", | |
| "new_layer_init": "copy_base", | |
| "depth_identity_operator": "inproj_zero_prime", | |
| "seed": 42 | |
| } | |
| } |