Upload autoresearch experiment (val_bpb=0.778052)

ed064c1 verified 3 months ago

266 Bytes

{
  "attention_backend": "sdpa",
  "compute_dtype": "torch.bfloat16",
  "n_embd": 1408,
  "n_head": 11,
  "n_kv_head": 11,
  "n_layer": 22,
  "sequence_len": 2048,
  "use_activation_checkpointing": true,
  "vocab_size": 16384,
  "window_pattern": "SSSL"
}

	{
	"attention_backend": "sdpa",
	"compute_dtype": "torch.bfloat16",
	"n_embd": 1408,
	"n_head": 11,
	"n_kv_head": 11,
	"n_layer": 22,
	"sequence_len": 2048,
	"use_activation_checkpointing": true,
	"vocab_size": 16384,
	"window_pattern": "SSSL"
	}