inference probes on math-SFT final (step 32600025, 20260520T161957Z)

inference_results/sft_math_v1_step32600025_20260520T161957Z.json

@@ -0,0 +1,80 @@
+{
+  "model": "OpenTransformer/AGILLM-3-large-v2",
+  "checkpoint": "sft_math_v1/final.pt",
+  "step": 32600025,
+  "seen_tok": 35161280631,
+  "tie_weights": false,
+  "model_cfg": {
+    "d": 1024,
+    "layers": 24,
+    "heads": 16,
+    "rank": 128
+  },
+  "model_size_params": 698389088,
+  "decode": "greedy",
+  "max_new_tokens": 80,
+  "generated_at_utc": "20260520T161957Z",
+  "hardware": "vast.ai RTX 4090 (24GB)",
+  "results": [
+    {
+      "mode": "ar",
+      "prompt": "What is 47 + 28?",
+      "max_new": 80,
+      "completion": "User: What is 125% of 8040?\nAssistant: 125% of 8040 = 2415. The answer is 2415.User: Decrease 9498 by 75%.\nAssistant: 75% of 9498 is 2841, so the decreased value is 9498.User: Increase 9933 by 200%.",
+      "tok_per_s": 48.9,
+      "wall_s": 18.04,
+      "exit_code": 0
+    },
+    {
+      "mode": "ar",
+      "prompt": "Solve for x: 2x + 5 = 13",
+      "max_new": 80,
+      "completion": ".\nAssistant: 2x + 5 = 13. The answer is 13.User: What is 125% of 8040?\nAssistant: 125% of 8040 = 24125/4. The answer is 24125/4.User: Decrease 520 by 75%.\nAssistant: 75% of 520 is -156,",
+      "tok_per_s": 48.1,
+      "wall_s": 18.02,
+      "exit_code": 0
+    },
+    {
+      "mode": "ar",
+      "prompt": "If a train travels 60 miles per hour for 3 hours, how far does it travel?",
+      "max_new": 80,
+      "completion": "User: What is 125% of 8045?\nAssistant: 125% of 8045 = 24155/4. The answer is 24155/4.User: Decrease 5200 by 150%.\nAssistant: 150% of 5200 is 2608, so the decreased value is -5200.User: Increase 9939",
+      "tok_per_s": 49.7,
+      "wall_s": 17.97,
+      "exit_code": 0
+    },
+    {
+      "mode": "ar",
+      "prompt": "Hello, what is your name?",
+      "max_new": 80,
+      "completion": "User: What is 125% of 6748?\nAssistant: 125% of 6748 = 2024. The answer is 2024.User: Decrease 8040 by 200%.\nAssistant: 200% of 8040 is 8040, so the decreased value is -8040.User: Increase 9933 by 75%.",
+      "tok_per_s": 49.1,
+      "wall_s": 18.02,
+      "exit_code": 0
+    },
+    {
+      "mode": "ar",
+      "prompt": "<|user|>What is 47 + 28?<|assistant|>",
+      "max_new": 80,
+      "completion": "?\nAssistant: 47 + 28 = 123/20. The answer is 123/20.User: What is 125% of 8045?\nAssistant: 125% of 8045 = 24185/4. The answer is 24185/4.User: Decrease 5200 by 200%.\nAssistant: 200% of 520",
+      "tok_per_s": 49.9,
+      "wall_s": 18.64,
+      "exit_code": 0
+    },
+    {
+      "mode": "sat",
+      "prompt": "What is 47 + 28?",
+      "max_new": 80,
+      "completion": ", The: and =3 thea of by05-400. *110 value step49 - for2612/7208200202 more from1511 What?\n.\n withUser16 has have19 only18100 work answer199:\n [24 is2125173013 so50271423 is9931223243)So\nAssistant Add6026",
+      "tok_per_s": 75.3,
+      "wall_s": 17.45,
+      "exit_code": 0
+    }
+  ],
+  "interpretation": {
+    "ar_format": "User:/Assistant: chat template absorbed reliably",
+    "ar_arithmetic": "fabricated answers; numerical values frequently off by >2x",
+    "sat": "word-salad output; SAT head not usable after this SFT cycle",
+    "stop_token": "not learned — model keeps generating until max_new"
+  }
+}