davanstrien's picture
davanstrien HF Staff
Open source and provenance links in a new tab (Hub refuses framing)
8e8dd52 verified
Raw History Blame Contribute Delete
19.9 kB
<!doctype html><html lang="en"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1"><title>GLAM extraction benchmark · Overall</title>
<style>/* Base stylesheet for the generated board page. Document first: no card chrome, no pills,
colour only where it carries meaning. */
:root{--ink:#1a1a1a;--muted:#6b7280;--line:#e5e7eb;}
*{box-sizing:border-box}
body{margin:0;background:#fff;color:var(--ink);
font-family:system-ui,-apple-system,sans-serif;font-size:15px;line-height:1.5;}
.wrap{max-width:900px;margin:0 auto;padding:44px 28px 64px;}
h1{font-size:20px;font-weight:600;margin:0 0 6px;letter-spacing:-.01em;}
.sub{font-size:13px;color:var(--muted);margin:0 0 24px;max-width:76ch;line-height:1.6;}
table{width:100%;border-collapse:collapse;font-size:14px;}
th{text-align:left;font-size:11px;font-weight:600;letter-spacing:.04em;text-transform:uppercase;
color:var(--muted);padding:0 14px 8px 0;border-bottom:1px solid var(--ink);}
th.r,td.r{text-align:right;}
td{padding:10px 14px 10px 0;border-bottom:1px solid var(--line);}
th:last-child,td:last-child{padding-right:0;}
tbody tr:last-child td{border-bottom:none;}
.model{font-family:ui-monospace,SFMono-Regular,Menlo,monospace;font-size:13.5px;}
.sz{color:var(--muted);font-size:12px;margin-left:8px;font-family:system-ui,sans-serif;}
.foot{margin:26px 0 0;font-size:12.5px;color:var(--muted);line-height:1.65;max-width:76ch;}
.mono{font-family:ui-monospace,Menlo,monospace;}
a{color:inherit;text-decoration:underline;text-underline-offset:2px;
text-decoration-color:var(--line);}
a:hover{text-decoration-color:currentColor;}
.wrap{max-width:1080px}.notice{border-left:3px solid #8b6c36;padding:8px 16px;background:#faf8f2}
small{color:#666}button,select{font:inherit;margin:0 12px 14px 0}.scroll{overflow-x:auto}
th button{margin:0;padding:6px 0;border:0;background:transparent;color:inherit;
font:inherit;letter-spacing:inherit;text-transform:inherit;cursor:pointer;text-align:inherit}
th button:hover{color:var(--ink)}th button:focus-visible{outline:2px solid #3566a3;outline-offset:4px}
.size-bar{display:flex;align-items:center;flex-wrap:wrap;gap:10px 16px;padding:14px 0;
border-top:1px solid var(--line);margin-top:20px}
.size-options{display:flex;align-items:center;gap:6px;overflow-x:auto;max-width:100%}
.size-options button{margin:0;border:0;border-radius:14px;padding:4px 10px;background:transparent;
color:var(--muted);font-size:14px;white-space:nowrap;cursor:pointer}
.size-options button[aria-pressed="true"]{background:var(--ink);color:#fff}
.size-options button:hover:not(:disabled):not([aria-pressed="true"]){background:#f1f2f4;color:var(--ink)}
.size-options button:disabled{color:#9ca3af;cursor:default}
.size-options button:focus-visible{outline:2px solid #3566a3;outline-offset:2px}
.size-label,#model-count{font-size:13px;color:var(--muted);white-space:nowrap}
#model-count{margin-left:auto}
.filter-note{font-size:13px;color:var(--muted);margin:0 0 18px}[hidden]{display:none!important}
.plot-section{margin-top:30px}.plot-section h2{font-size:18px;margin-bottom:6px}
.plot-section p{font-size:13px;color:var(--muted)}#size-plot{width:100%;height:auto;display:block}
.plot-grid{stroke:var(--line);stroke-width:1}.plot-tick,.plot-axis{font-size:11px;fill:var(--muted)}
.plot-frontier{fill:none;stroke:#286c66;stroke-width:2;stroke-dasharray:5 4;opacity:.7}
.plot-point{fill:#a3aab4;stroke:white;stroke-width:1.5}.frontier-point{fill:#286c66}
#size-plot a:hover .plot-point,#size-plot a:focus .plot-point{stroke:var(--ink);stroke-width:3}
.plot-label{font-size:11px;fill:var(--ink);pointer-events:none}
details{margin-top:24px}img{max-width:100%;max-height:440px}pre{white-space:pre-wrap;font-size:12px}
.plot-label-line{stroke:#82908e;stroke-width:1;opacity:.65;pointer-events:none}
</style></head><body><main class="wrap">
<h1>GLAM extraction benchmark</h1><p>Overall extraction quality</p>
<label for="dataset">Dataset </label><select id="dataset"><option data-config="overall" value="overall.html" selected>Overall</option><option data-config="nls-index-cards" value="index.html">nls-index-cards</option><option data-config="harvard-botany-headers" value="harvard-botany-headers/index.html">harvard-botany-headers</option></select>
<p class="sub">2 datasets · 10 models with complete results</p>
<p>Overall F1 is the average of the dataset F1 scores. Each dataset has equal weight (50% each). Only models evaluated on every selected dataset are included. </p>
<p>Choose a dataset above to inspect its results and methodology. These are small evaluation sets;
nearby scores should not be read as a definitive ranking.</p>
<div class="size-bar"><span class="size-label" id="size-label">Parameter size</span>
<div class="size-options" role="group" aria-labelledby="size-label"><button type="button" data-max-params="all" aria-pressed="true">All</button><button type="button" data-max-params="1" aria-pressed="false" disabled>&lt;1B</button><button type="button" data-max-params="3" aria-pressed="false">&lt;3B</button><button type="button" data-max-params="6" aria-pressed="false">&lt;6B</button><button type="button" data-max-params="12" aria-pressed="false">&lt;12B</button><button type="button" data-max-params="32" aria-pressed="false">&lt;32B</button><button type="button" data-max-params="128" aria-pressed="false">&lt;128B</button><button type="button" data-max-params="500" aria-pressed="false">&lt;500B</button></div>
<span id="model-count" role="status">10 of 10 models</span></div>
<p class="filter-note">Uses total parameters, including for mixture-of-experts models.
Click a column heading to sort; click again to reverse.</p>
<div class="scroll"><table><thead><tr>
<th scope="col"><button type="button" data-column="0" data-direction="asc">Model <span aria-hidden="true">↕</span></button></th>
<th class="r" scope="col"><button type="button" data-column="1" data-direction="desc">Overall F1 <span aria-hidden="true">↕</span></button></th>
</tr></thead><tbody id="results" data-f1-column="1" data-default-column="1">
<tr data-params="321.32"><td data-value="GLM-5.3-Flash"><a href="https://huggingface.co/zai-org/GLM-5.3-Flash" target="_blank" rel="noopener noreferrer" title="zai-org/GLM-5.3-Flash">GLM-5.3-Flash</a> <small>321.32B</small></td><td class="r" data-value="0.9150403061224488">91.5</td></tr>
<tr data-params="27.78"><td data-value="Qwen3.8-27B"><a href="https://huggingface.co/Qwen/Qwen3.8-27B" target="_blank" rel="noopener noreferrer" title="Qwen/Qwen3.8-27B">Qwen3.8-27B</a> <small>27.78B</small></td><td class="r" data-value="0.8881977210884353">88.8</td></tr>
<tr data-params="235.67"><td data-value="Qwen3-VL-235B"><a href="https://huggingface.co/Qwen/Qwen3-VL-235B-A22B-Instruct" target="_blank" rel="noopener noreferrer" title="Qwen/Qwen3-VL-235B-A22B-Instruct">Qwen3-VL-235B</a> <small>235.67B</small></td><td class="r" data-value="0.8789903061224491">87.9</td></tr>
<tr data-params="4.66"><td data-value="Qwen3.5-4B"><a href="https://huggingface.co/Qwen/Qwen3.5-4B" target="_blank" rel="noopener noreferrer" title="Qwen/Qwen3.5-4B">Qwen3.5-4B</a> <small>4.660B</small></td><td class="r" data-value="0.8454842517006803">84.5</td></tr>
<tr data-params="4.539"><td data-value="NuExtract-3"><a href="https://huggingface.co/numind/NuExtract3" target="_blank" rel="noopener noreferrer" title="numind/NuExtract3">NuExtract-3</a> <small>4.539B</small></td><td class="r" data-value="0.8373081632653062">83.7</td></tr>
<tr data-params="9.65"><td data-value="Qwen3.5-9B"><a href="https://huggingface.co/Qwen/Qwen3.5-9B" target="_blank" rel="noopener noreferrer" title="Qwen/Qwen3.5-9B">Qwen3.5-9B</a> <small>9.65B</small></td><td class="r" data-value="0.835791598639456">83.6</td></tr>
<tr data-params="8.767"><td data-value="Qwen3-VL-8B"><a href="https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct" target="_blank" rel="noopener noreferrer" title="Qwen/Qwen3-VL-8B-Instruct">Qwen3-VL-8B</a> <small>8.767B</small></td><td class="r" data-value="0.83271037414966">83.3</td></tr>
<tr data-params="3.997"><td data-value="Granite-Vision-4.1-4B"><a href="https://huggingface.co/ibm-granite/granite-vision-4.1-4b" target="_blank" rel="noopener noreferrer" title="ibm-granite/granite-vision-4.1-4b">Granite-Vision-4.1-4B</a> <small>3.997B</small></td><td class="r" data-value="0.7677061224489796">76.8</td></tr>
<tr data-params="7.996"><td data-value="gemma-4-E4B"><a href="https://huggingface.co/google/gemma-4-E4B-it" target="_blank" rel="noopener noreferrer" title="google/gemma-4-E4B-it">gemma-4-E4B</a> <small>7.996B</small></td><td class="r" data-value="0.7286563605442178">72.9</td></tr>
<tr data-params="2.274"><td data-value="Qwen3.5-2B"><a href="https://huggingface.co/Qwen/Qwen3.5-2B" target="_blank" rel="noopener noreferrer" title="Qwen/Qwen3.5-2B">Qwen3.5-2B</a> <small>2.274B</small></td><td class="r" data-value="0.6328972448979592">63.3</td></tr></tbody></table></div>
<p id="empty-results" hidden>No models match this size limit.</p>
<p class="foot">Scores are shown on a 0–100 scale. Dataset weights, component scores and pinned
revisions are recorded in <a target="_blank" rel="noopener noreferrer" href="overall-scores.json">the score download</a>.</p>
<section class="plot-section" aria-labelledby="plot-title"><h2 id="plot-title">Size vs. overall extraction quality</h2>
<p>Upper-left is better: smaller models, higher overall F1. Teal points show the observed Pareto frontier.</p>
<svg id="size-plot" data-f1-column="1" viewBox="0 0 760 360" role="group" aria-label="Model size versus overall extraction F1"></svg>
<p id="plot-note"></p></section>
<details><summary>Sources and reproducibility</summary>
<p><a target="_blank" rel="noopener noreferrer" href="overall-scores.json">Overall scores and component provenance</a> ·
<a target="_blank" rel="noopener noreferrer" href="configs.json">Dataset configurations and revisions</a> ·
<a target="_blank" rel="noopener noreferrer" href="https://github.com/davanstrien/glam-extraction-benchmark">Code</a> ·
<a target="_blank" rel="noopener noreferrer" href="https://huggingface.co/buckets/small-models-for-glam/glam-extraction-results">Raw predictions</a></p>
<p>The overall view combines the selected dataset results; it does not create a new dataset or change individual scores.</p>
</details></main><script>function drawSizePlot(rows) {
const svg = document.getElementById('size-plot');
const escapeText = value => String(value).replace(/[&<>"']/g, c => ({'&':'&amp;','<':'&lt;','>':'&gt;','"':'&quot;',"'":'&#39;'}[c]));
const points = rows.filter(row => !row.hidden && Number(row.dataset.params) > 0).map(row => ({
name: row.cells[0].dataset.value,
params: Number(row.dataset.params),
f1: Number(row.cells[Number(svg.dataset.f1Column || 4)].dataset.value),
href: row.cells[0].querySelector('a')?.getAttribute('href'),
}));
const omitted = rows.filter(row => !row.hidden).length - points.length;
document.getElementById('plot-note').textContent =
'Each point is a model; hover or focus for details, click to open its Hub page.'
+ (omitted ? ` ${omitted} model(s) with unknown size omitted.` : '');
if (!points.length) {
svg.innerHTML = '<text x="380" y="180" text-anchor="middle">No models with a known size match this filter.</text>';
return;
}
// Keep the axes stable when the size filter changes.
const sizes = rows.map(row => Number(row.dataset.params)).filter(size => size > 0);
const ticks = [0.01,0.025,0.05,0.1,0.25,0.5,1,2,4,8,16,32,64,128,256,512,1024];
const lo = [...ticks].reverse().find(t => t <= Math.min(...sizes)) ?? Math.min(...sizes) / 2;
const hi = ticks.find(t => t > Math.max(...sizes)) ?? Math.max(...sizes) * 2;
const left = 58, top = 20, width = 678, height = 284;
const X = params => left + (Math.log10(params) - Math.log10(lo)) / (Math.log10(hi) - Math.log10(lo)) * width;
const Y = f1 => top + (1 - f1) * height;
const frontier = points.filter(p => !points.some(q =>
q.params <= p.params && q.f1 >= p.f1 && (q.params < p.params || q.f1 > p.f1)
)).sort((a,b) => a.params - b.params);
let chart = '';
for (let value = 0; value <= 100; value += 20) {
const y = Y(value / 100);
chart += `<line class="plot-grid" x1="${left}" y1="${y}" x2="${left+width}" y2="${y}"/>`
+ `<text class="plot-tick" x="${left-10}" y="${y+4}" text-anchor="end">${value}</text>`;
}
ticks.filter(t => t >= lo && t <= hi).forEach(t => {
const x = X(t);
chart += `<line class="plot-grid" x1="${x}" y1="${top}" x2="${x}" y2="${top+height}"/>`
+ `<text class="plot-tick" x="${x}" y="${top+height+20}" text-anchor="middle">${t}B</text>`;
});
chart += `<text class="plot-axis" x="${left+width/2}" y="350" text-anchor="middle">Total parameters (billions · log scale)</text>`
+ '<text class="plot-axis" transform="translate(16,162) rotate(-90)" text-anchor="middle">Extraction F1 (0–100)</text>';
if (frontier.length > 1) chart += `<polyline class="plot-frontier" points="${frontier.map(p => `${X(p.params)},${Y(p.f1)}`).join(' ')}"/>`;
points.forEach(p => {
const best = frontier.includes(p), x = X(p.params), y = Y(p.f1);
const description = `${p.name}: ${p.params}B total parameters, F1 ${(p.f1*100).toFixed(1)}${best ? ', on the observed Pareto frontier' : ''}`;
const opening = p.href ? `<a href="${escapeText(p.href)}" target="_blank" rel="noopener noreferrer" aria-label="${escapeText(description)}">`
: `<g tabindex="0" role="img" aria-label="${escapeText(description)}">`;
chart += `${opening}<title>${escapeText(description)}</title><circle class="plot-point${best ? ' frontier-point' : ''}" cx="${x}" cy="${y}" r="6"/>${p.href ? '</a>' : '</g>'}`;
if (best) {
const right = x < left + width * 0.65;
chart += `<text class="plot-label" data-point-x="${x}" data-point-y="${y}" x="${x+(right ? 10 : -10)}" y="${y-11}" text-anchor="${right ? 'start' : 'end'}">${escapeText(p.name)}</text>`;
}
});
svg.innerHTML = chart;
// Place labels using their rendered dimensions; crowded points keep their true coordinates.
const placed = [];
const intersects = (a, b, gap = 4) => a.x < b.x + b.width + gap
&& a.x + a.width + gap > b.x && a.y < b.y + b.height + gap && a.y + a.height + gap > b.y;
const markers = points.map(p => ({x: X(p.params)-8, y: Y(p.f1)-8, width: 16, height: 16}));
svg.querySelectorAll('.plot-label').forEach(label => {
const x = Number(label.dataset.pointX), y = Number(label.dataset.pointY);
const preferredSide = x < left + width * 0.65 ? 1 : -1;
let chosen;
for (const dy of [-12, 26, -32, 46, -52, 66, -72, 86]) {
for (const side of [preferredSide, -preferredSide]) {
label.setAttribute('text-anchor', side === 1 ? 'start' : 'end');
label.setAttribute('x', x + side * 12);
label.setAttribute('y', y + dy);
const box = label.getBBox();
if (box.x < left + 2 || box.x + box.width > left + width - 2
|| box.y < 5 || box.y + box.height > top + height - 4
|| placed.some(other => intersects(box, other))
|| markers.some(marker => intersects(box, marker, 2))) continue;
chosen = {x: box.x, y: box.y, width: box.width, height: box.height, dy};
break;
}
if (chosen) break;
}
if (!chosen) {
label.remove(); // Every point still has its accessible hover/focus label.
return;
}
placed.push(chosen);
if (chosen.dy !== -12) {
const line = document.createElementNS('http://www.w3.org/2000/svg', 'line');
line.setAttribute('class', 'plot-label-line');
line.setAttribute('x1', x);
line.setAttribute('y1', y);
line.setAttribute('x2', Math.max(chosen.x, Math.min(x, chosen.x + chosen.width)));
line.setAttribute('y2', y < chosen.y ? chosen.y - 2 : chosen.y + chosen.height + 2);
svg.insertBefore(line, label);
}
});
}
const body = document.getElementById('results');
const buttons = [...document.querySelectorAll('[data-column]')];
const sizeButtons = [...document.querySelectorAll('[data-max-params]')];
// Configs without identifier fields start with extraction F1.
const f1Column = Number(body.dataset.f1Column || 4);
let activeColumn = Number(body.dataset.defaultColumn ||
([...body.rows].some(row => row.cells[1].dataset.value !== '') ? 1 : f1Column));
let direction = activeColumn === f1Column ? 'desc' : 'asc';
function sortRows() {
const sign = direction === 'asc' ? 1 : -1;
[...body.rows].sort((a, b) => {
const x = a.cells[activeColumn].dataset.value;
const y = b.cells[activeColumn].dataset.value;
// Missing rates stay last in either direction.
if (x === '' || y === '') return (x === '') - (y === '');
const comparison = activeColumn === 0
? x.localeCompare(y, 'en', { numeric: true, sensitivity: 'base' })
: Number(x) - Number(y);
return sign * comparison || a.cells[0].dataset.value.localeCompare(b.cells[0].dataset.value);
}).forEach(row => body.append(row));
buttons.forEach(button => {
const active = Number(button.dataset.column) === activeColumn;
button.parentElement.removeAttribute('aria-sort');
if (active) button.parentElement.setAttribute('aria-sort', direction === 'asc' ? 'ascending' : 'descending');
button.querySelector('span').textContent = active ? (direction === 'asc' ? '↑' : '↓') : '↕';
});
}
buttons.forEach(button => button.onclick = () => {
const column = Number(button.dataset.column);
direction = column === activeColumn ? (direction === 'asc' ? 'desc' : 'asc') : button.dataset.direction;
activeColumn = column;
sortRows();
});
function filterSize(limit) {
let visible = 0;
[...body.rows].forEach(row => {
row.hidden = limit !== 'all'
&& (row.dataset.params === '' || Number(row.dataset.params) >= Number(limit));
if (!row.hidden) visible++;
});
document.getElementById('model-count').textContent = `${visible} of ${body.rows.length} models`;
document.getElementById('empty-results').hidden = visible !== 0;
sizeButtons.forEach(button => button.setAttribute('aria-pressed', String(button.dataset.maxParams === limit)));
drawSizePlot([...body.rows]);
}
// Keep shareable state in the URL, including when embedded on the Hub.
const datasetSelect = document.getElementById('dataset');
function syncUrl(limit) {
const url = new URL(window.location.href);
if (limit === 'all') url.searchParams.delete('max_params');
else url.searchParams.set('max_params', limit);
url.searchParams.set('dataset', datasetSelect.selectedOptions[0].dataset.config);
window.history.replaceState(null, '', url);
window.parent.postMessage({queryString: url.search}, 'https://huggingface.co');
}
function navigateDataset(option, replace = false) {
const url = new URL(option.value, window.location.href);
url.search = window.location.search;
url.searchParams.set('dataset', option.dataset.config);
url.hash = window.location.hash;
if (replace) window.location.replace(url);
else window.location.assign(url);
}
datasetSelect.addEventListener('change', () => navigateDataset(datasetSelect.selectedOptions[0]));
sizeButtons.forEach(button => button.onclick = () => {
filterSize(button.dataset.maxParams);
syncUrl(button.dataset.maxParams);
});
function restoreUrl() {
const params = new URLSearchParams(window.location.search);
const requested = [...datasetSelect.options].find(option => option.dataset.config === params.get('dataset'));
if (requested && !requested.selected) {
navigateDataset(requested, true);
return;
}
const value = params.get('max_params');
const limit = sizeButtons.some(button => button.dataset.maxParams === value) ? value : 'all';
filterSize(limit);
syncUrl(limit);
}
window.addEventListener('popstate', restoreUrl);
sortRows();
restoreUrl();
</script></body></html>