Upload autoresearch experiment (val_bpb=0.778052)

README.md

@@ -0,0 +1,12 @@
+---
+library_name: autoresearch
+tags:
+- autoresearch
+- nanochat-tokenizer
+---
+
+# autoresearch experiment
+
+This checkpoint uses the 16,384-token nanochat-compatible BPE tokenizer. Load `model.pt` with this repository's `GPT` class and `config.json`.
+
+W&B run URL: https://wandb.ai/dustinsdelivery-delivery-ai/autoresearch/runs/a6aekama

config.json

@@ -0,0 +1,12 @@
+{
+  "attention_backend": "sdpa",
+  "compute_dtype": "torch.bfloat16",
+  "n_embd": 1408,
+  "n_head": 11,
+  "n_kv_head": 11,
+  "n_layer": 22,
+  "sequence_len": 2048,
+  "use_activation_checkpointing": true,
+  "vocab_size": 16384,
+  "window_pattern": "SSSL"
+}

dataset.txt

@@ -0,0 +1 @@
+tinystories

model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:952ab96880a96583d044c0275aafa64832b0d208a383c3f694f5320edfed3af9
+size 2739473467

token_bytes.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aceeef692124e783e96747c2d6e22a0427fe6b34ce0d7ef96353ad07c5084f90
+size 67141

tokenizer.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1f61d667d57536cbc4014ffc6d1ef7ebf70d451bf9f29e7d0a0cc1d3a7feb29
+size 197049

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "bos_token": "<|bos|>",
+  "format": "nanochat-rustbpe-v1",
+  "special_tokens": [
+    "<|bos|>",
+    "<|user_start|>",
+    "<|user_end|>",
+    "<|assistant_start|>",
+    "<|assistant_end|>",
+    "<|python_start|>",
+    "<|python_end|>",
+    "<|output_start|>",
+    "<|output_end|>"
+  ],
+  "split_pattern": "'(?i:[sdmt]|ll|ve|re)|[^\\r\\n\\p{L}\\p{N}]?+\\p{L}+|\\p{N}{1,2}| ?[^\\s\\p{L}\\p{N}]++[\\r\\n]*|\\s*[\\r\\n]|\\s+(?!\\S)|\\s+",
+  "vocab_size": 16384
+}

training_metrics.json

@@ -0,0 +1,16 @@
+{
+  "activation_checkpointing": true,
+  "dataset": "tinystories",
+  "depth": 22,
+  "eval_batch_size": 4,
+  "mfu_percent": 17.708544461070193,
+  "num_params": 823267148,
+  "num_steps": 68,
+  "peak_vram_mb": 12435.2333984375,
+  "smoke_test": false,
+  "total_seconds": 10007.529666423798,
+  "total_tokens": 35651584,
+  "train_batch_size": 4,
+  "training_seconds": 7309.851931095123,
+  "val_bpb": 0.7780523809334358
+}