config.json

{
    "activation": "gelu",
    "architectures": [
        "LNNModel"
    ],
    "dt": 0.1,
    "eos_token_id": 1,
    "expert_dim": 32768,
    "hidden_size": 3048,
    "initializer_range": 0.02,
    "lambda_res": 0.0,
    "model_type": "lnn",
    "moe_load_balance_loss_weight": 0.01,
    "num_experts": 14,
    "num_experts_per_tok": 2,
    "num_hidden_layers": 6,
    "pad_token_id": 1,
    "pmb_num_blocks": 1024,
    "pmb_slots_per_block": 4096,
    "pmb_top_k": 1,
    "torch_dtype": "float32",
    "transformers_version": "4.35.2",
    "use_moe": true,
    "use_pmb": false,
    "vocab_size": 128815,
    "auto_map": {
        "AutoModel": "quasar.lnn.LNNModel"
    }
}