/* 3amBench Replay: renders data/runs.json (written by scripts/export_runs.py). No dependencies. board.js (loaded first) holds the tier switch, leaderboard, expert requirement panel and theme button. */ "use strict"; const GROUPS = ["alerts", "repairs", "recording", "routing", "inhibition", "preserved"]; const $ = (id) => document.getElementById(id); const S = { data: null, tier: "core", task: null, run: null, i: 0, timer: null, resize: null }; function esc(s) { return String(s == null ? "" : s).replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); } const fmt = (x, d = 3) => (x == null || isNaN(x) ? "–" : Number(x).toFixed(d)); const signed = (x) => (x > 0 ? "+" : "") + fmt(x, 3); /* ---------- tiny markdown: headings, bullets, paragraphs, **bold**, `code` ---------- */ function inline(s) { return esc(s).replace(/`([^`]+)`/g, "$1").replace(/\*\*([^*]+)\*\*/g, "$1"); } function markdown(src) { const out = []; let list = false; let para = []; const flush = () => { if (para.length) { out.push("

" + inline(para.join(" ")) + "

"); para = []; } }; for (const line of src.split("\n")) { const h = line.match(/^(#{1,4})\s+(.*)$/); const li = line.match(/^\s*[-*]\s+(.*)$/); if (h || li || !line.trim()) flush(); if (!li && list) { out.push(""); list = false; } if (h) out.push(`

${inline(h[2])}

`); else if (li) { if (!list) { out.push(""); return out.join(""); } /* ---------- tiers and pickers ---------- */ const isExpert = (tid) => taskTier(S.data.tasks[tid]) === "expert"; function tierTasks(tier) { return Object.keys(S.data.tasks).filter((t) => taskTier(S.data.tasks[t]) === tier).sort(); } function setTier(tier, tid, runId) { stop(); S.tier = tier; renderTierSwitch(); leaderboard(); fillTasks(); const ids = tierTasks(tier); $("replay-section").hidden = !ids.length; if (ids.length) selectTask(tid && ids.includes(tid) ? tid : ids[0], runId); } function runLabel(r) { const m = r.model ? ` · ${r.model}` : ""; return `${r.agent}${m} · reward ${fmt(r.reward)}${r.trial ? " · " + r.trial.slice(-8) : ""}${r.invalid ? " · invalid" : ""}`; } function fillTasks() { $("task-pick").innerHTML = tierTasks(S.tier).map((t) => { const T = S.data.tasks[t]; const n = S.data.runs.filter((r) => r.task_id === t).length; const what = isExpert(t) ? ` · ${T.workflow || T.family || "expert"}` : ""; const diff = isExpert(t) ? "" : `${T.difficulty || T.tier}, `; return ``; }).join(""); } function selectTask(tid, runId) { S.task = tid; $("task-pick").value = tid; const runs = S.data.runs.filter((r) => r.task_id === tid); $("run-pick").innerHTML = runs.map((r) => ``).join(""); $("instruction").innerHTML = markdown(S.data.tasks[tid].instruction); selectRun(runId && runs.some((r) => r.id === runId) ? runId : runs[0].id); } function runMeta(r, T) { const k = r.keys || {}; const exp = taskTier(T) === "expert"; const where = exp ? [T.workflow, T.family] : [T.workflow, T.difficulty || T.tier]; const reward = r.reward_original != null ? `re-graded ${fmt(r.reward)} (Harbor ${fmt(r.reward_original)})` : `final reward ${fmt(r.reward)}`; const extra = exp ? ` · diagnosis ${fmt(k.diagnosis, 2)} · routine ${fmt(k.routine, 2)} · preserved ${fmt(k.preservation, 2)}` : ""; let h = r.invalid ? `` : ""; h += `
${esc(r.agent)}${r.model ? " · " + esc(r.model) : ""} · ${where.filter(Boolean).map(esc).join(" · ")} · ${reward} · solved ${k.solved ? "yes" : "no"}${extra}${r.cost_usd ? " · $" + fmt(r.cost_usd, 2) : ""}
`; if (!exp) { const reqs = T.requirements.map((q) => { const s = r.req_scores ? r.req_scores[q.id] : null; return `${esc(q.id)} ${s == null ? "–" : fmt(s, 2)}`; }).join(" · "); h += `
Per-requirement score: ${reqs}
`; } if (r.reward_original != null) { const ko = r.keys_original || {}; h += `
Re-graded with the current hidden tests${r.regraded ? ` (${esc(r.regraded)})` : ""}; Harbor's own verifier gave ${fmt(r.reward_original)}${ko.solved != null ? `, solved ${ko.solved ? "yes" : "no"}` : ""}.
`; } if (r.exception) h += `
Harbor recorded ${esc(r.exception)} for this run; it still counts.
`; return h + `
${esc(r.curve_note || "")}
`; } function selectRun(id) { stop(); S.run = S.data.runs.find((r) => r.id === id); $("run-pick").value = id; const r = S.run; const T = S.data.tasks[r.task_id]; $("run-meta").innerHTML = runMeta(r, T); $("scrub").max = Math.max(0, r.steps.length - 1); buildTimeline(); drawCurve(); go(0); const h = `#${S.tier}/${encodeURIComponent(r.task_id)}/${r.id}`; if (location.hash !== h) history.replaceState(null, "", h); } /* ---------- step view ---------- */ function stepTitle(s) { if (s.kind === "think") return (s.text || "").split("\n")[0]; if (s.kind === "tool_call") return `${s.tool} ${s.args || ""}`; return (s.output || "").split("\n")[0] || "(no output)"; } function buildTimeline() { $("timeline").innerHTML = S.run.steps.map((s, i) => { const dr = s.r ? `${s.r > 0 ? "▲" : "▼"} ${signed(s.r)}` : ""; return `
  • ${i}${esc(s.kind.replace("_", " "))} ${esc(stepTitle(s))}${dr}
  • `; }).join(""); } function diffHtml(d) { return d.split("\n").map((l) => { const c = l.startsWith("+++") || l.startsWith("---") ? "hunk" : l.startsWith("+") ? "add" : l.startsWith("-") ? "del" : l.startsWith("@@") ? "hunk" : ""; return c ? `${esc(l)}` : esc(l) + "\n"; }).join(""); } function renderStep() { const s = S.run.steps[S.i]; let h = ""; const cum = cumAt(S.i); h += `
    step ${S.i}${esc(s.kind === "think" ? "thinking" : s.tool)}${s.r != null ? `step reward ${signed(s.r)}` : ""} cumulative ${fmt(cum)}${s.phi != null ? `progress ${fmt(s.phi)}` : ""} ${s.ok === false ? 'tool error' : ""}
    `; if (s.kind === "think") h += `

    ${esc(s.text)}

    `; if (s.args) h += `
    arguments
    ${esc(s.args)}
    `; if (s.diff) h += `
    file diff
    ${diffHtml(s.diff)}
    `; if (s.kind === "observation") h += `
    output
    ${esc(s.output || "(empty)")}
    `; $("step-detail").innerHTML = h; } function cumAt(i) { for (let j = i; j >= 0; j--) if (S.run.steps[j].cum != null) return S.run.steps[j].cum; return 0; } function go(i) { const n = S.run.steps.length; S.i = Math.max(0, Math.min(n - 1, i)); $("scrub").value = S.i; $("step-label").textContent = `${S.i} / ${n - 1}`; document.querySelectorAll("#timeline li").forEach((li) => { const k = +li.dataset.i; li.classList.toggle("cur", k === S.i); li.classList.toggle("future", k > S.i); }); const cur = document.querySelector("#timeline li.cur"); if (cur) { const box = $("timeline"); const t = cur.offsetTop - box.offsetTop; if (t < box.scrollTop || t > box.scrollTop + box.clientHeight - 30) box.scrollTop = t - box.clientHeight / 2; } renderStep(); (isExpert(S.run.task_id) ? renderReqs : renderChecks)(); moveCursor(); } /* ---------- checks panel (core tier) ---------- */ function checksAt(i) { let prev = null; let cur = null; for (let j = 0; j <= i; j++) if (S.run.steps[j].checks) { prev = cur; cur = S.run.steps[j].checks; } const last = S.run.steps.length - 1; if (i === last && S.run.final_checks && cur !== S.run.final_checks) { prev = cur; cur = S.run.final_checks; } else if (!S.run.steps[i].checks) prev = null; // highlight only changes made by this very step return { cur, prev }; } function renderChecks() { const T = S.data.tasks[S.run.task_id]; const { cur, prev } = checksAt(S.i); const graded = S.run.steps.some((s) => s.checks); $("checks-title").textContent = "Checks"; $("checks-when").textContent = cur ? (S.i === S.run.steps.length - 1 ? "(end of run)" : `(after step ${S.i})`) : ""; if (!cur) { $("checks-summary").textContent = graded ? "Nothing graded yet at this step: the first edit has not happened." : "This run has checks only at the end (Harbor gives a terminal result). Jump to the last step."; $("checks").innerHTML = ""; return; } const byGroup = {}; let pass = 0; let tot = 0; T.checks.forEach((c, k) => { const v = cur[k]; if (v === "-") return; (byGroup[c.group] = byGroup[c.group] || []).push({ c, v, changed: prev && prev[k] !== v }); tot++; if (v === "1") pass++; }); const reqTitle = Object.fromEntries(T.requirements.map((q) => [q.id, q.title])); $("checks-summary").innerHTML = `${pass} of ${tot} checks pass. Highlighted rows changed at this step.`; $("checks").innerHTML = GROUPS.filter((g) => byGroup[g]).map((g) => { const items = byGroup[g]; const ok = items.filter((x) => x.v === "1").length; const open = g !== "preserved" || ok < items.length; const reqs = [...new Set(items.map((x) => x.c.req || ""))]; const body = reqs.map((rid) => { const li = items.filter((x) => (x.c.req || "") === rid).map((x) => `
  • ${x.v === "1" ? "✓" : "✗"} ${esc(x.c.label)}${x.c.note ? ` ${esc(x.c.note)}` : ""}
  • `).join(""); return `
    ${rid ? `
    ${esc(rid)} · ${esc(reqTitle[rid] || "")}
    ` : ""}
    `; }).join(""); return `
    ${g} ${ok}/${items.length}${body}
    `; }).join(""); } /* ---------- reward curve (inline SVG) ---------- */ const CV = { w: 900, h: 190, l: 44, r: 12, t: 12, b: 26 }; function drawCurve() { const terminal = S.run.curve === "terminal"; $("curve-fig").hidden = terminal; $("curve-none").hidden = !terminal; if (terminal) { $("curve").innerHTML = ""; CV.X = null; $("curve-none").textContent = `Final reward only: Harbor grades the end state of this run (reward ${fmt(S.run.reward)}).`; return; } CV.w = Math.max(320, Math.round($("curve").clientWidth || 900)); // draw at the real width: legible on phones CV.h = CV.w >= 700 ? 230 : 190; const st = S.run.steps; const n = Math.max(1, st.length - 1); const pts = st.map((s, i) => ({ i, y: cumAt(i), r: s.r || 0 })); const X = (i) => CV.l + (i / n) * (CV.w - CV.l - CV.r); const Y = (v) => CV.t + (1 - v) * (CV.h - CV.t - CV.b); let g = ""; for (const v of [0, 0.25, 0.5, 0.75, 1]) { g += ``; g += `${v.toFixed(2)}`; } const every = [1, 2, 5, 10, 20, 50, 100, 200, 500].find((e) => n / e <= 10) || Math.ceil(n / 10); for (let i = 0; i <= n; i += every) g += `${i}`; const d = pts.map((p, k) => `${k ? "L" : "M"}${X(p.i).toFixed(1)},${Y(Math.max(-0.05, Math.min(1.05, p.y))).toFixed(1)}`).join(""); g += ``; for (const p of pts) if (Math.abs(p.r) > 1e-9) { const x = X(p.i); const y = Y(Math.max(-0.05, Math.min(1.05, p.y))); const up = p.r > 0; const tri = up ? `${x},${y - 11} ${x - 5},${y - 3} ${x + 5},${y - 3}` : `${x},${y + 11} ${x - 5},${y + 3} ${x + 5},${y + 3}`; g += ``; } g += ``; g += ``; g += ``; $("curve").innerHTML = `${g}`; const svg = $("curve").firstChild; const hit = svg.querySelector("#hit"); const idxAt = (ev) => { const b = svg.getBoundingClientRect(); const x = ((ev.clientX - b.left) / b.width) * CV.w; return Math.round(((x - CV.l) / (CV.w - CV.l - CV.r)) * n); }; hit.addEventListener("mousemove", (ev) => { const i = Math.max(0, Math.min(n, idxAt(ev))); const s = st[i]; const tip = $("tip"); tip.hidden = false; tip.innerHTML = `step ${i} · ${esc(s.kind === "think" ? "thinking" : s.tool)}
    cumulative ${fmt(cumAt(i))}` + (s.r ? ` · step ${signed(s.r)}` : ""); const b = svg.getBoundingClientRect(); const px = (X(i) / CV.w) * b.width; tip.style.left = Math.max(0, Math.min(px + 10, b.width - 200)) + "px"; tip.style.top = "8px"; }); hit.addEventListener("mouseleave", () => { $("tip").hidden = true; }); hit.addEventListener("click", (ev) => { stop(); go(idxAt(ev)); }); CV.X = X; CV.Y = Y; } function moveCursor() { const c = document.getElementById("cursor"); const dot = document.getElementById("cursor-dot"); if (!c || !CV.X) return; const x = CV.X(S.i); const y = CV.Y(Math.max(-0.05, Math.min(1.05, cumAt(S.i)))); c.setAttribute("x1", x); c.setAttribute("x2", x); dot.setAttribute("cx", x); dot.setAttribute("cy", y); } /* ---------- playback ---------- */ function stop() { if (S.timer) clearInterval(S.timer); S.timer = null; $("btn-play").textContent = "Play"; } function play() { if (S.timer) return stop(); if (S.i >= S.run.steps.length - 1) go(0); $("btn-play").textContent = "Pause"; S.timer = setInterval(() => { if (S.i >= S.run.steps.length - 1) return stop(); go(S.i + 1); }, +$("speed").value); } function wire() { $("tier-switch").addEventListener("click", (e) => { const b = e.target.closest("button[data-tier]"); if (b && b.dataset.tier !== S.tier) setTier(b.dataset.tier); }); $("theme-btn").onclick = cycleTheme; $("task-pick").addEventListener("change", (e) => selectTask(e.target.value)); $("run-pick").addEventListener("change", (e) => selectRun(e.target.value)); $("btn-first").onclick = () => { stop(); go(0); }; $("btn-prev").onclick = () => { stop(); go(S.i - 1); }; $("btn-next").onclick = () => { stop(); go(S.i + 1); }; $("btn-last").onclick = () => { stop(); go(S.run.steps.length - 1); }; $("btn-play").onclick = play; $("speed").onchange = () => { if (S.timer) { stop(); play(); } }; $("scrub").addEventListener("input", (e) => { stop(); go(+e.target.value); }); $("timeline").addEventListener("click", (e) => { const li = e.target.closest("li"); if (li) { stop(); go(+li.dataset.i); } }); window.addEventListener("resize", () => { clearTimeout(S.resize); S.resize = setTimeout(() => { if (S.run) { drawCurve(); moveCursor(); } }, 150); }); document.addEventListener("keydown", (e) => { if (!S.run || /INPUT|SELECT|TEXTAREA/.test(e.target.tagName)) return; if (e.key === "ArrowRight") { stop(); go(S.i + 1); e.preventDefault(); } else if (e.key === "ArrowLeft") { stop(); go(S.i - 1); e.preventDefault(); } else if (e.key === " ") { play(); e.preventDefault(); } }); $("copy-cmd").onclick = () => { const t = $("docker-cmd").textContent; if (navigator.clipboard) navigator.clipboard.writeText(t).then(() => { $("copy-cmd").textContent = "Copied"; }, () => {}); }; } async function main() { wire(); try { const res = await fetch("data/runs.json", { cache: "no-cache" }); if (!res.ok) throw new Error(res.status + " " + res.statusText); S.data = await res.json(); } catch (e) { $("load-status").textContent = "Could not load data/runs.json: " + e.message; return; } if (!S.data.runs.length) { $("load-status").textContent = "No runs recorded yet."; return; } $("load-status").hidden = true; $("board-section").hidden = false; $("replay-section").hidden = false; // #///; the older #// still works (the tier comes from the task) const m = decodeURIComponent(location.hash.slice(1)).split("/"); let tier = TIER_ORDER.includes(m[0]) ? m.shift() : null; const tid = S.data.tasks[m[0]] ? m[0] : null; if (!tier || !tiersPresent().includes(tier)) tier = tid ? taskTier(S.data.tasks[tid]) : defaultTier(); setTier(tier, tid, m[1]); if (m[2] && S.task === tid) go(+m[2] || 0); } main();