{
  "model_type": "glm4_moe_lite",
  "src": {
    "model_type": "glm4_moe_lite",
    "vocab_size": 154880,
    "hidden_size": 2048,
    "intermediate_size": 10240,
    "moe_intermediate_size": 1536,
    "num_hidden_layers": 47,
    "num_nextn_predict_layers": 1,
    "num_attention_heads": 20,
    "num_key_value_heads": 20,
    "q_lora_rank": 768,
    "kv_lora_rank": 512,
    "qk_nope_head_dim": 192,
    "qk_rope_head_dim": 64,
    "v_head_dim": 256,
    "n_routed_experts": 64,
    "n_shared_experts": 1,
    "num_experts_per_tok": 4,
    "first_k_dense_replace": 1,
    "moe_layer_freq": 1,
    "rms_norm_eps": 1e-05,
    "tie_word_embeddings": false
  },
  "tgt": {
    "hidden_size": 2048,
    "intermediate_size": 10240,
    "moe_intermediate_size": 1536,
    "num_hidden_layers": 75,
    "rms_norm_eps": 1e-05
  },
  "operators": {
    "hidden_operator": "vzero",
    "mlp_operator": "dormant_prime",
    "new_layer_init": "copy_base",
    "depth_identity_operator": "inproj_zero_prime",
    "seed": 42
  }
}