config.json

{
  "architectures": [
    "CloverLMForCausalLM"
  ],
  "attn_backend": "pytorch",
  "auto_map": {
    "AutoConfig": "configuration_cloverlm.CloverLMConfig",
    "AutoModelForCausalLM": "modeling_cloverlm.CloverLMForCausalLM",
    "AutoTokenizer": [
      "tokenization_cloverlm.CloverLMTokenizer",
      null
    ]
  },
  "d_head": 128,
  "dtype": "bfloat16",
  "head_dim": 128,
  "heads": 28,
  "hidden_size": 3584,
  "intermediate_size": 14336,
  "max_context": 1024,
  "max_position_embeddings": 1024,
  "model_type": "cloverlm",
  "num_attention_heads": 28,
  "num_blocks": 29,
  "num_hidden_layers": 29,
  "num_key_value_heads": 7,
  "quartet_2_impl": "pseudoquant",
  "ratio": 4,
  "scale_type": "1/sqrt(d)",
  "tie_word_embeddings": true,
  "transformers_version": "5.4.0",
  "vocab_size": 32000,
  "weight_tying": true
}