diff --git a/.gitattributes b/.gitattributes index 8a9a258cfebe5b0e42bb6ddaf9acbd8d06c00752..206f80ba26f8648a5fe9b5bd402536b1b1f9ff24 100644 --- a/.gitattributes +++ b/.gitattributes @@ -40,6 +40,3 @@ huggies/paper.png filter=lfs diff=lfs merge=lfs -text huggies/rocket.png filter=lfs diff=lfs merge=lfs -text huggies/xray.png filter=lfs diff=lfs merge=lfs -text og.png filter=lfs diff=lfs merge=lfs -text -og-news.png filter=lfs diff=lfs merge=lfs -text -huggies/decision.png filter=lfs diff=lfs merge=lfs -text -huggies/decision-favicon.png filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index 7969ede278e8ad277df51ed5e2f3fcb4ce2eb2fb..58d197b4d7b0d6eb37206acd01604a55cb77822e 100644 --- a/README.md +++ b/README.md @@ -1,290 +1,20 @@ --- -title: Jev Decision Index +title: Jev Reproductions Tracker emoji: 🔬 colorFrom: yellow colorTo: blue sdk: static header: mini pinned: false -short_description: Benchmarks and news on various repros of TypeSafe's Jev -thumbnail: https://huggingface.co/spaces/multimodalart/jev-decision-index/resolve/main/og.png -models: -- PelaAI/KnowLine-31B-Gen1 -- fastino/GLiDE -- sthanika-ai/Sieve-31B-v3 -- turion-ai/turion-jev-31b -- vllm-sr/d3-ultra -- tattrongvu/Jeffan-31b-1710 -- h2oai/h2o-lightning-31b -- Rover00/WeMM-JEV -- baselabs/mammoth-1 -- Ainvest-World/Ainvest-Decision-27b-v1.0 -- perplexity-ai/pplx-decider-v1.2 -- autotrust/AutoTrust-JEV-27B-v1.1 -- startlux-models/StartLux-Decision-v1.1-27B-Instant -- PNPM244/oak1.1 -- andyzhang232/ajev-gemma4-31b-lm6q -- LycheeAI/Lychee-27B -- vllm-sr/d3-turbo -- vllm-sr/d3 -- taoszhang/VisionJEV -- bespokelabs/Nimble-v4-max -- morriszjm/AnyJev-Tacitus-31B -- Mapika/decider-31b -- plantwaterer/allec -- PelaAI/KnowLine-35B-A3B-Gen1.5 -- OneShotOrg/OneShot -- scar-ai/vera-large -- sanghwa-na/bobcat-1.3 -- mukaddamzaid/amos-1-large -- Maincode/matilda-decision-v1 -- PelaAI/KnowLine-26B-A4B-Gen2.5 -- frontier-infra/jebadiah-27b-v2-1 -- FINAL-Bench/Darwin-27B-ZTC -- apertixai/cerno-27b -- torchcast-ai/torchcast-decision-27b -- OmniJev/OneJev1.6-27B -- jsaurabh/qwen-decision-27b-noncausal-lora -- chinhnc/Quyet-1.0-Large -- plantwaterer/garum -- baselabs/elephant-1 -- StandardThinking/StandardOne-35B-SH -- FrontiersMind/Lumma-fev-27B -- sthanika-ai/Sieve-27B-v2 -- moganai/mogan-decision-31B-it -- MISA-core/linne-fern-1-0 -- jaredpalmer/kev-27b -- juspay/Xor-26B-A4B -- lihe07/span-02 -- plantwaterer/garas -- shisa-ai/shisa-de-2.1 -- caiovicentino1/Eikos-27B-FP8 -- shgao/rsi-jev-v6.1-vl-27b -- PelaAI/KnowLine-9B-Gen1 -- startlux-models/StartLux-Decision-v1.1-9B-Instant -- vllm-sr/Decision-2.0-Vega-27B -- autotrust/JEV-35B -- autotrust/JEV-27B -- matthewagi/Tern-35B-A3B -- surogate/rune-26b-a4b-GGUF -- SkyPanther/synack-decide-26b-a4b -- PelaAI/KnowLine-4B-Gen5.5 -- PixilabAI/Blink-v0.3-26B-A4B-NVFP4 -- theunnecessarythings/JADE -- org2ai/Wald-26B-A4B -- andyzhang232/ajev-gemma4-26b-a4b-lora1 -- Cloudflare/clef -- startlux-models/StartLux-Decision-v1.1-4B-Instant -- perplexity-ai/pplx-decider-flash-v1.2 -- vllm-sr/d3-flash -- kirp/jpt-35b-a3b -- juspay/xor -- autotrust/GEV-26B-Decide -- OmniJev/OneJev1.6-9B -- nace-ai/drex-v2.0 -- plantwaterer/muria -- tattrongvu/Jeffan-4B-2010 -- victorbennett/Verdin-4B-v1.1 -- bespokelabs/Bespoke-Nimble-9B-v3 -- frontier-infra/jebadiah-9b-v2-1 -- vllm-sr/d3-mini -- andyzhang232/ajev-gemma4-12b-lora7 -- OmniJev/OneJev1.6-4B -- sky7350/Mica-v0.5-4B -- olaverse/PurpleMIST-Flash-1.0 -- apertixai/cerno-9b -- WangSenQWQ/Senova-4B -- sthanika-ai/Sieve-9B-v2 -- Mapika/decider-35b-a3b-nvfp4 -- EldanRing/Winnow-12B -- ironbcc/Seb-9B-v2 -- vllm-sr/Decision-2.0-Lux-9B -- nace-ai/drex-v1.5 -- HopitAI/hopper -- corners-ai/CoCo-Decision-2.4-4B -- Cloudflare/clef-omni -- alibiserikbay/JevK5-9B -- Cloudflare/clef-flash -- shgao/rsi-jev-v6.1-vl-4b -- fdemelo/jegue -- eljiwo/jiwo-4b -- kirp/jpt-9b -- everettjf/ezjev-4b-s2 -- akhilaaa3/Jev-Omni -- DoccyHealth/Solomon -- StandardThinking/StandardOne-8B-SH -- MLukauskas/amber-hybrid-4b -- autotrust/JEV-9B -- bespokelabs/Bespoke-Nimble-9B-v2 -- sthanika-ai/Sieve-9B -- apertixai/cerno-4b -- sthanika-ai/Sieve-4B -- jaredpalmer/kev-9b -- blackmallangcow/nari-decision-4b -- empiriolabsai/aplomb-1 -- vllm-sr/Decision-2.0-Nox-4B -- HopitAI/hopper-g -- startlux-models/StartLux-Decision-v1.1-2B-Instant -- flymy-ai/decision-4b-v1.6 -- StandardThinking/StandardOne-3B-SH -- kirp/jpt-4b -- internlm/Intern-Decision-4B -- shgao/rsi-jev-v5.0-vl-3b -- Valen-Team/Valen-4B -- TokenRhythm/NeoHorse-Jev-4B -- alibiserikbay/JevK5 -- Mapika/decider-4b -- interfaze-ai/lev -- theaviv/Lev1-3.7B -- jaredpalmer/kev-4b -- chaoliangUNSW/Jev-Style-2B-Decision-v4 -- sthanika-ai/Sieve-2B-v3 -- LatentMesh-Labs/latentmesh-blitz-2b-v1 -- felhen-ai/saracura-ptbr-4b -- cbuk10001/Cam1 -- vllm-sr/d3-nano -- kpiya/kas-4b -- togethercomputer/Tev1-4B-experimental -- EldanRing/Winnow-E4B -- Leptos-ai/Aurelis-2B -- AmirBraham/jevstral-8b -- wayfind/metask-jev-4b-policy-mix -- UserMoonlight/intelif-qwen3-4b -- knowledgator/gliclass-v4-edge -- LiquidAI/d1-3B -- pngwn/system-one-qwen3.5-4b-scorer-v2b -- scar-ai/vera-core-2 -- OmniJev/OneJev1.6-0.8B -- Kyunghwan-kim/crystal-0.8b -- vllm-sr/vela3-fast -- tardellirs/crivo-1b -- davidburhans/gevva-e4b -- startlux-models/StartLux-Decision-v1.1-0.8B-Instant -- run-saina/saina-helm-2-omni-0.8b -- vllm-sr/d3-lite -- MLukauskas/amber-0.4b -- eljiwo/jiwo-0.8b -- run-saina/saina-helm-2-0.8b -- flock-io/this-that-model-1.2 -- davidburhans/gevva-e2b -- tardellirs/crivo-400m -- Mapika/decider-2b -- vllm-sr/Decision-2.0-Sol-2B -- AnamK/jdi-ptr14-0.8b -- Valen-Team/Valen-2B -- vllm-sr/vela3-fast-vision -- sthanika-ai/sieve-2b -- teilomillet/wyat-1-800m -- internlm/Intern-Decision-2B -- sthanika-ai/Sieve-0.8B -- StrandsAgents/strands-decider-2B-hobson-v21 -- carrtesy/EXAONE-4.0-1.2B-JEV -- tasksource/tasksource-decider-nano -- tardellirs/crivo-68m -- Elfsong/jev-ettin-150m-v16 -- vllm-sr/Decision-2.0-Eos-0.8B -- tardellirs/crivo-150m -- hotchpotch/bekko-system-one-v0-400m -- codavidgarcia/mola-150m -- vllm-sr/d3-edge -- codavidgarcia/mola-68m -- Elfsong/jev-flan-t5-large-20261017-public-v2 -- decision-tune/decisiontune-1.0 -- Valen-Team/Valen-0.8B -- Hanno-Labs/bosun-v3.1-1.7b -- vllm-sr/Decision-2.0-Kai-0.6B -- Lukitaduarte/dinah-1 -- jaredpalmer/kev-0.8b -- lCREEl/pick-150m -- kirp/jpt-0.8b -- Elfsong/jev-ettin-400m-20261013-v0 -- mohamedlotfy50/sifr-0.8b-v3 -- tardellirs/crivo-32m -- tardellirs/prumo -- cortex-agent-llc/kodiak-v0.2-1b-accuracy -- internlm/Intern-Decision-0.8B -- chaoliangUNSW/Jev-Style-0.8B-Decision-v3 -- togethercomputer/Tev1-0.8B-experimental -- hotchpotch/bekko-system-one-v0-68m -- LiquidAI/d1-omni-600M -- tardellirs/crivo-17m -- wfzyx/von -- Hanno-Labs/bosun-v3.1-0.6b -- scar-ai/vera-spark -- moganai/lavoir -- fastino/GLiNER2.5-Decide -- convaiinnovations/laya -- alibiserikbay/JevK5-Lite -- monotykamary/LFM2.5-2.6B-RLCD -- hotchpotch/bekko-system-one-v0-17m -- Contrastive-LM/CLM-v0.1-8B -- notnotsamuel/LFM2.5-350M-RLCD -- DangerLabs/DM-JEPA -- SupersonicLabs/Julia-1 -- kshitijthakkar/nirnaya-0.4b-0.2a-decision -- heman10x/rlcd-modernbert-151m -- shisa-ai/shisa-de-2 -- OmniJev/OneJev1.5-0.8B -- OmniJev/OneJev1.5-9B -- OmniJev/OneJev1.5-27B -- leinenterprises/Herbert_v0.5 -- tinnel123/jev-d3-46k -- ken-jo/QEV-2B -- ken-jo/QEV-0.8B -- sthanika-ai/Sieve-Vision-4B -- RBDash-Team/RBDash-Decision-v0.2 -- sthanika-ai/Sieve-Vision-27B -- AlexWortega/openjev -- bespokelabs/Bespoke-Nimble-9B -- C-Tianyu/NanoJev -- DavidHatley/system-one-mini -- DeepMostInnovations/sales-conversion-model-reinf-learning -- fastino/gliner2.5-base-v1 -- fastino/gliner2.5-multi-v1 -- fastino/gliner2.5-small-v1 -- flock-io/this-that-model-1.0 -- frontier-infra/jebadiah-27b -- FrontiersMind/Lumma-fev-0.1b -- FrontiersMind/Lumma-fev-0.6b -- google/diffusiongemma-26B-A4B-it -- google/gemma-4-31B -- harshatheg/Qwen-2.5-1B-RLCD -- knowledgator/gliformer-large-v1 -- llm-semantic-router/Decision-1.0-Eos-0.8B -- llm-semantic-router/Decision-1.0-Kai-0.6B -- llm-semantic-router/Decision-1.0-Lex-0.6B -- llm-semantic-router/Decision-1.0-Lux-9B -- llm-semantic-router/Decision-1.0-Nox-4B -- llm-semantic-router/Decision-1.0-Sol-2B -- Manav2op/verdict-small -- Mapika/decider-35b-a3b -- MoLeMo-Lab/mojev -- perplexity-ai/pplx-decider-v1-27b -- pngwn/system-one-qwen3.5-4b-scorer -- Qwen/Qwen2.5-1.5B -- Qwen/Qwen3-4B-Instruct-2507 -- Qwen/Qwen3.5-4B-Base -- Qwen/Qwen3.6-27B -- Qwen/Qwen3.8-27B -- Qwen/Qwen3.8-27B-FP8 -- andyzhang232/ajev-gemma4-12b-lora5 -- Lukitaduarte/dinah-0 -- OmniJev/OneJev-0.8B -- perplexity-ai/pplx-decider-v1.1-27b -- corners-ai/CoCo-Decision-4B -- frontier-infra/jebadiah-9b-v2 -- Maincode/matilda-jev-v1.5 -- sthanika-ai/Sieve-27B -- sthanika-ai/Sieve-2B-v2 -- scar-ai/vera-core -- OmniJev/OneJev1.5-4B +short_description: Open reproductions of TypeSafe's Jev, all in one place +thumbnail: https://huggingface.co/spaces/multimodalart/jev-reproductions-tracker/resolve/main/og.png --- # Jev Reproductions Tracker Who is rebuilding TypeSafe's **Jev** (System One / RLCD) in the open? -This static Space opens on the Decision Index leaderboard; the **News** tab tracks the artifacts in one combined grid, color-coded by kind: +This static Space tracks the artifacts in one combined grid, color-coded by kind: - **Decoding**: parallel constrained decoding on stock models (inference technique, no new weights) - **Diffusion**: text diffusion models run in a "Jev mode" @@ -292,23 +22,21 @@ This static Space opens on the Decision Index leaderboard; the **News** tab trac - **Prior art**: "this already exists" claims - **Explainers**: architecture speculation, explainers, benchmarks and roundups -Cards sort by a **trending** score: ♥ likes on X + 5 × GitHub stars + 8 × Hub likes + views ÷ 500, plus a **recency credit** (up to 8000, halving daily) so a fresh release is not buried under older, louder ones. The credit is scaled by the item's own engagement, so a brand-new entry nobody has reacted to yet cannot climb on its date alone. Cards added within a day of the metrics snapshot get a green **new** badge. Use the category and "has" chips to filter, or switch the sort. +Cards sort by a **trending** score: ♥ likes on X + 5 × GitHub stars + 8 × Hub likes + views ÷ 500. Use the category and "has" chips to filter, or switch the sort. …plus a section on what is still **not** in the open: TypeSafe's weights, the RLCD algorithm, and any open model matching Jev's calibration claims. Hugging Face models and Spaces are first-class artifacts here, not only GitHub repos. -## Decision Index (`index.html`) +## Regenerating the social image -The **Index** tab (switch at the top of every page) is the leaderboard: every open reproduction and Jev, ranked by the Full score: 60% private tests of the same skills and 40% private tasks from new domains, both on the public scale. The public benchmarks are shown next to every model but do not count, so the ranking rests on tests nobody can train on. The switch on the main chart offers Full score, Public only and Your own (set your own part and area weights). Below the chart sit the per-area radars, every benchmark with its public and private scores, and a full page per model (`index.html?model=`; `?model=jev` is Jev's own report). A Vision board ranks the image-capable entrants the same way. - -The public suite is 37 benchmarks in five areas, each chance-corrected so 0 is random guessing and 100 is perfect; an unanswered request counts as wrong. The private tests and the new-domain tasks are never published. The **Methodology** page has the full rules, every benchmark's subset and baseline, and the changelog. +`og.html` is the 1200×630 stage that produces `og.png` (rendered at 2× with headless Chrome). It reuses the page's styles and data, so re-render it after adding artifacts to keep the counts current. ## Contributing -All news data lives in one JS array (`ITEMS`) near the bottom of `news.html` (the News tab; the Index tab is `index.html`). Open a PR on this Space to add or correct an entry. Please include the announcement post on X if there is one, and the Hub / GitHub link. +All data lives in one JS array (`ITEMS`) near the bottom of `index.html`. Open a PR on this Space to add or correct an entry. Please include the announcement post on X if there is one, and the Hub / GitHub link. -Engagement metrics are a snapshot (2026-09-24) from public post metadata and will drift. +Engagement metrics are a snapshot (2026-09-17) from public post metadata and will drift. ## Credits diff --git a/check.js b/check.js deleted file mode 100644 index f069605cdef4dfe564dc374cfff8acc9f9ec4ff7..0000000000000000000000000000000000000000 --- a/check.js +++ /dev/null @@ -1,9 +0,0 @@ -const fs=require('fs'); -const h=fs.readFileSync('news.html','utf8'); -const js=h.slice(h.indexOf('')); -const head=js.slice(0,js.indexOf('/* ---- state ---- */')); -const checks=` -console.log('CATS',CATS.map(c=>c.order+':'+c.id).join(' '),'| ITEMS',ITEMS.length,'| orphans',ITEMS.filter(i=>!CATS.find(c=>c.id===i.cat)).length,'| maxdesc',Math.max(...ITEMS.map(i=>i.desc.length))); -for(const c of CATS){console.log(' ',c.id,'->',sortItems(ITEMS.filter(i=>i.cat===c.id),'trending').slice(0,5).map(i=>i.title.slice(0,26)+'='+trending(i)).join(' | '))} -`; -eval(head+checks); diff --git a/data/index.json b/data/index.json deleted file mode 100644 index e9f64f68e331e4c45f5d559660506729a4203bc7..0000000000000000000000000000000000000000 --- a/data/index.json +++ /dev/null @@ -1 +0,0 @@ -{"generated_utc":"2026-10-28T19:32:17Z","suite":{"requests":110201,"scoreable":109759,"benchmarks":42,"jev_benchmarks":42,"panel":37,"panel_full":37,"categories":5,"dropped":["MiniWoB++","Boxoban","RTFM","ScienceWorld","Hanabi","Codenames"],"hardware":"1 x NVIDIA RTX PRO 6000","jev_version":"jev-1.14.0","panel_id":"decision-index-0.3","calibration_note":"32 benchmarks with a per-field right or wrong answer (ToolRet, BRIGHT and RouterBench excluded), each weighted equally; a 1-in-6 sample of requests; confidence is the probability placed on the chosen option. Rows an entrant trained on count as wrong here too.","formulas":[{"id":"balanced_raw","normalization":"raw","formula":"100*sum_c(0.2*category[c])","label":"Public index, raw accuracy"},{"id":"balanced_skill","normalization":"skill","formula":"0.60*private_same_skills + 0.40*private_new_domains, both mapped onto the public scale with untuned stock models","label":"Full score"},{"id":"breadth_skill","normalization":"skill","shift":0.1,"formula":"100*(product_c((0.1+0.9*category[c])**0.2)-0.1)/0.9","label":"Public index, breadth"}],"edition":"release-v4","label":"Decision Index 0.4.1","corpus_sha256":"b2b56d6fb636837ca469e689087bdbf373dda8de7638aa2da6793e6eda0792d5","base_edition":"release-v2.1","note":null,"areas":{"knowledge":[25,30,31,32,33,43,44,45,57,58,28],"language":[11,12,29,38,39,40,41,42,59],"retrieval":[4,5,36,37,56,61],"tools":[1,2,3,9,62],"arts":[20,21,22,23,50,64]},"not_in_index":[{"id":6,"name":"RouterBench"},{"id":10,"name":"SGD"},{"id":24,"name":"MMLU"},{"id":26,"name":"ARC-Easy"},{"id":27,"name":"ARC-Challenge"}],"lower_rules":[],"headline":"balanced_skill","chance_levels":[{"id":25,"name":"GPQA Diamond","area":"knowledge","chance":0.25,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":30,"name":"GSM8K","area":"knowledge","chance":0.175,"label":"Uniform random choice","method":"per track: 4-choice and 10-choice versions corrected separately, then averaged"},{"id":31,"name":"ChessBench","area":"knowledge","chance":0.0819,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":32,"name":"MuSR","area":"knowledge","chance":0.371,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":33,"name":"SATA-Bench","area":"knowledge","chance":0.0131,"label":"Random whole-case guesses","method":"expected score of uniform random choice over each question's options"},{"id":43,"name":"CRUXEval","area":"knowledge","chance":0.3697,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":44,"name":"CLadder","area":"knowledge","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":45,"name":"HLE","area":"knowledge","chance":0.1641,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":57,"name":"MMLU-Pro","area":"knowledge","chance":0.1109,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":58,"name":"BBH","area":"knowledge","chance":0.3101,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":11,"name":"ContractNLI","area":"language","chance":0.3085,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":12,"name":"ANLI","area":"language","chance":0.3324,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":28,"name":"WinoGrande","area":"language","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":29,"name":"HellaSwag","area":"language","chance":0.25,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":38,"name":"ACOS","area":"language","chance":0.031,"label":"Always yes · per-review F1","method":"per-review F1 of answering yes to every aspect-and-sentiment pair (a fair coin scores lower)"},{"id":39,"name":"FinEntity","area":"language","chance":0.3201,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":40,"name":"iSarcasmEval","area":"language","chance":0.2227,"label":"Random choices · F1","method":"track A English: F1 of random guessing on the sarcastic class"},{"id":41,"name":"VAST","area":"language","chance":0.3333,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":42,"name":"NLI4CT","area":"language","chance":0.486,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":59,"name":"RAGTruth","area":"language","chance":0.5177,"label":"Always \"hallucinated\" · F1","method":"F1 of always answering \"hallucinated\" (a fair coin scores lower)"},{"id":4,"name":"BANKING77","area":"retrieval","chance":0.0127,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":5,"name":"CLINC150","area":"retrieval","chance":0.006,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":36,"name":"BRIGHT","area":"retrieval","chance":0.116,"label":"Random ranking, answerable queries","method":"per query: expected nDCG@10 of a random ranking of that query's candidates, answerable queries only"},{"id":37,"name":"Amazon ESCI","area":"retrieval","chance":0.2027,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":56,"name":"PhishNChips","area":"retrieval","chance":0.5,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":61,"name":"HoVer","area":"retrieval","chance":0.5,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":1,"name":"BFCL","area":"tools","chance":0.2592,"label":"Random whole-case guesses","method":"all fields of a case right by chance"},{"id":2,"name":"ToolRet","area":"tools","chance":0.1341,"label":"Random ranking, answerable queries","method":"per query: expected nDCG@10 of a random ranking of that query's candidates, answerable queries only"},{"id":3,"name":"API-Bank","area":"tools","chance":0.0189,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":9,"name":"Home appliances","area":"tools","chance":0,"label":"Random whole-case guesses","method":"all fields of a case right by chance, on the kept rows"},{"id":62,"name":"When2Call","area":"tools","chance":0.25,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":20,"name":"BPoMP","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":21,"name":"Humicroedit","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":22,"name":"POP909","area":"arts","chance":0.0078,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":23,"name":"cfcolor","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":50,"name":"Habermas","area":"arts","chance":0.311,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":64,"name":"New Yorker","area":"arts","chance":0.2,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"}],"index_note":"The public index averages the 37 benchmarks of the 0.2.1 panel in five areas, with GSM8K rebuilt and ForecastBench retired. Each benchmark is chance-corrected first, (score - chance) / (1 - chance) clipped to 0-1, so 0 means random guessing and 100 means perfect. Every score is coverage-adjusted, so an unanswered or unsupported request counts as wrong. The headline Full score comes from private tests only; this public index is shown alongside it, and benchmarks marked + also show each model's score on the private benchmark of the same competency.","changes":["The main ranking is the Full score: 60% private tests of the same skills and 40% private tasks from new domains, each put on the public scale using untuned stock models. Public benchmarks are still run and shown, but no longer count toward the Full score, so the ranking rests entirely on tests nobody can train on.","GSM8K is rebuilt, ForecastBench is retired from the public index, and WinoGrande now counts under Knowledge & Reasoning."],"area_weights":{"knowledge":0.258494,"language":0.258494,"retrieval":0.200229,"tools":0.182783,"arts":0.1},"area_sizing":"sqrt","gold":{"1":1.2,"3":1.2,"4":1.2,"5":1.2,"12":1.2,"25":1.2,"28":1.2,"29":1.2,"36":1.2,"45":1.2,"48":1.2,"57":1.2,"58":1.2},"index_notes":[{"id":6,"name":"RouterBench","note":"Not in the index: the prompt reveals that GPT-4 is the best route, so the top models all pick GPT-4 and score within a fraction of a point of each other. Scores stay visible."},{"id":10,"name":"SGD","note":"Not in the index: build bug, pending re-run. The builder gave every SGD row the whole dialogue, including the turns after the one being classified. Scores stay visible; SGD returns to the index once re-run with the fixed builder."}]},"categories":[{"id":"knowledge","label":"Knowledge & Reasoning","color":"#5976bd","icon":"M4 4h6a3 3 0 0 1 3 3v14a4 4 0 0 0-4-2H4z M20 4h-4a3 3 0 0 0-3 3v14a4 4 0 0 1 4-2h3z","description":"Academic knowledge, commonsense, mathematics, and code.","panel":[25,30,31,32,33,43,44,45,57,58,28],"panel_full":[25,30,31,32,33,43,44,45,57,58,28],"suite":[57,25,26,27,30,32,33,43,44,45,24,58,31,28],"group":[57,25,26,27,30,32,33,43,44,45,51,52,24,58,28],"priority":[57,25,30,45,28]},{"id":"language","label":"Language Understanding","color":"#407e9e","icon":"M4 4h16v12H9l-5 4z M8 8h8 M8 12h5","description":"Read a text and judge what it means: entailment, sentiment, sarcasm and stance.","panel":[11,12,29,38,39,40,41,42,59],"panel_full":[11,12,29,38,39,40,41,42,59],"suite":[11,12,38,39,40,41,42,29,59],"group":[11,12,38,39,40,41,42,29,59],"priority":[40,42,39,38,11]},{"id":"retrieval","label":"Retrieval & Classification","color":"#3e8e5e","icon":"M11 4a7 7 0 1 0 0 14 7 7 0 0 0 0-14z M20 20l-4.3-4.3","description":"Pick the right item from a large set: rank documents and products by relevance, route a message to one of many intents, sort out spam and phishing.","panel":[4,5,36,37,56,61],"panel_full":[4,5,36,37,56,61],"suite":[36,37,4,5,10,56,61],"group":[36,37,4,5,10,54,55,56,61],"priority":[36,37,4,5,10,55]},{"id":"tools","label":"Tools & Automation","color":"#b77a38","icon":"M5 4v11a4 4 0 0 0 4 4h10 M5 9h9a4 4 0 0 0 4-4 M2 7l3-3 3 3 m8 9 3 3-3 3","description":"Tool choice, model routing, household control, and browser or simulator actions.","panel":[1,2,3,9,62],"panel_full":[1,2,3,9,62],"suite":[1,2,3,6,9,62],"group":[1,2,3,6,9,53,62,7,16],"priority":[1,9,6]},{"id":"arts","label":"Arts & Human Taste","color":"#b6698a","icon":"M12 3a9 9 0 1 0 0 18h2a2 2 0 0 0 0-4h-1a2 2 0 0 1 0-4h4a4 4 0 0 0 4-4c0-4-5-6-9-6 M7 8h.01 M12 6h.01 M17 8h.01 M6 13h.01","description":"Poetry, music, humor, human preferences, and forecasting.","panel":[20,21,22,23,50,64],"panel_full":[20,21,22,23,50,64],"suite":[20,21,22,23,50,64],"group":[20,21,22,23,50,64],"priority":[20,22,21,23]},{"id":"games","label":"Games & Strategy","color":"#ad6f48","icon":"M7 8h10a4 4 0 0 1 4 4l1 5a2 2 0 0 1-3 2l-3-3H8l-3 3a2 2 0 0 1-3-2l1-5a4 4 0 0 1 4-4z M7 10v4 M5 12h4 M16 11h.01 M18 13h.01","description":"Move selection, spatial planning, rules, and decisions against other players.","panel":[],"panel_full":[31],"suite":[],"jev_only":true,"folded_into":"knowledge","group":[31,13,15,17,19],"priority":[31]}],"benchmarks":{"1":{"id":1,"dataset":"BFCL","short":"BFCL","metric":"case exact accuracy","explainer":"Says yes or no for each supplied function: call it for this request? All must be right.","baseline":{"value":0.2592,"label":"Random whole-case guesses"},"lower":false,"range":null,"cases":1694,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2305.15334"},{"kind":"code","url":"https://github.com/gorilla-llm/gorilla"}],"gold":1.2,"private":true},"2":{"id":2,"dataset":"ToolRet","short":"ToolRet","metric":"nDCG@10","explainer":"Rates how useful each of 32 search-hit tools is for a request; ranked by nDCG@10, on requests where at least one of the 32 is relevant.","baseline":{"value":0.1341,"label":"Random ranking, answerable queries","previous":0.0937},"lower":false,"range":null,"cases":685,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2503.01763"},{"kind":"dataset","url":"https://huggingface.co/datasets/mangopy/ToolRet-Queries"}],"private":true},"3":{"id":3,"dataset":"API-Bank","short":"API-Bank","metric":"accuracy","explainer":"Reads an assistant conversation and picks the next API to call from a catalog of 53.","baseline":{"value":0.0189,"label":"Uniform random choice"},"lower":false,"range":null,"cases":508,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2304.08244"},{"kind":"code","url":"https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank"}],"gold":1.2,"private":true},"4":{"id":4,"dataset":"BANKING77","short":"BANKING77","metric":"macro-F1","explainer":"Picks the intent of a bank customer's message, e.g. \"How do I locate my card?\", from 77.","baseline":{"value":0.0127,"label":"Random choices · F1"},"lower":false,"range":null,"cases":3080,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2003.04807"},{"kind":"dataset","url":"https://github.com/PolyAI-LDN/task-specific-datasets"}],"gold":1.2,"private":true},"5":{"id":5,"dataset":"CLINC150+OOS","short":"CLINC150","metric":"macro-F1","explainer":"Reads a short assistant request and picks its intent from 150, or \"out of scope\" when none fits.","baseline":{"value":0.006,"label":"Random choices · F1"},"lower":false,"range":null,"cases":5500,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/clinc/oos-eval"}],"gold":1.2,"private":true},"6":{"id":6,"dataset":"RouterBench","short":"RouterBench","metric":"selected quality (quality objective)","explainer":"Picks which of 11 AI models should answer a prompt; scored by its real answer.","baseline":{"value":0.5735,"label":"Random model selection"},"lower":false,"range":null,"cases":5001,"group":"tools","in_suite":true,"interactive":false,"index_note":"Not in the index: the prompt reveals that GPT-4 is the best route, so the top models all pick GPT-4 and score within a fraction of a point of each other. Scores stay visible.","references":[{"kind":"paper","url":"https://arxiv.org/abs/2403.12031"},{"kind":"dataset","url":"https://huggingface.co/datasets/withmartian/routerbench"}]},"7":{"id":7,"dataset":"MiniWoB++","short":"MiniWoB++","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"tools","in_suite":false,"interactive":true},"9":{"id":9,"dataset":"Home appliance simulator","short":"Home appliances","metric":"case exact accuracy","explainer":"Given a simulated home and a command, answers ~18 questions on handling it; all must be right. Duplicate and dev-set rows removed.","baseline":{"value":0,"label":"Random whole-case guesses"},"lower":false,"range":null,"cases":88,"group":"tools","in_suite":true,"interactive":false,"references":[{"kind":"code","url":"https://github.com/apolinario/decision-index/blob/main/decision_index/suite/build/home_appliance.py"}],"private":true},"10":{"id":10,"dataset":"SGD/SGD-X","short":"SGD","metric":"macro-F1","explainer":"Reads a service dialogue so far and picks the intent the user is pursuing now, or NONE.","baseline":{"value":0.399,"label":"Random choices · F1"},"lower":false,"range":null,"cases":2500,"group":"retrieval","in_suite":true,"interactive":false,"index_note":"Not in the index: build bug, pending re-run. The builder gave every SGD row the whole dialogue, including the turns after the one being classified. Scores stay visible; SGD returns to the index once re-run with the fixed builder.","references":[{"kind":"paper","url":"https://arxiv.org/abs/1909.05855"},{"kind":"dataset","url":"https://github.com/google-research-datasets/dstc8-schema-guided-dialogue"}]},"11":{"id":11,"dataset":"ContractNLI","short":"ContractNLI","metric":"macro-F1","explainer":"Says whether a full NDA supports, contradicts or omits each of 17 fixed statements.","baseline":{"value":0.3085,"label":"Random choices · F1"},"lower":false,"range":null,"cases":123,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2021.findings-emnlp.164"},{"kind":"dataset","url":"https://stanfordnlp.github.io/contract-nli/"}]},"12":{"id":12,"dataset":"ANLI","short":"ANLI","metric":"macro-F1","explainer":"Decides whether a claim follows from, contradicts, or is neutral to a premise written to fool AI models.","baseline":{"value":0.3324,"label":"Random choices · F1"},"lower":false,"range":null,"cases":3200,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/1910.14599"},{"kind":"dataset","url":"https://huggingface.co/datasets/facebook/anli"}],"gold":1.2,"private":true},"13":{"id":13,"dataset":"Boxoban","short":"Boxoban","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"15":{"id":15,"dataset":"RTFM","short":"RTFM","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"16":{"id":16,"dataset":"ScienceWorld","short":"ScienceWorld","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"tools","in_suite":false,"interactive":true},"17":{"id":17,"dataset":"Hanabi","short":"Hanabi","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"19":{"id":19,"dataset":"Codenames","short":"Codenames","metric":"episode success","explainer":"Interactive environment. No reproduction has run it, so it sits outside this edition of the index and appears only here.","baseline":{"value":null,"label":null},"lower":false,"cases":null,"group":"games","in_suite":false,"interactive":true},"20":{"id":20,"dataset":"BPoMP","short":"BPoMP","metric":"accuracy","explainer":"Given two versions of a limerick, one with its rhyme, meter or sense broken, picks the original.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":811,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://zenodo.org/records/7299879"}],"private":true},"21":{"id":21,"dataset":"Humicroedit","short":"Humicroedit","metric":"accuracy","explainer":"Given two one-word edits of a news headline, picks the one people rated funnier.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":2628,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/1906.00274"},{"kind":"dataset","url":"https://cs.rochester.edu/u/nhossain/humicroedit.html"}],"private":true},"22":{"id":22,"dataset":"POP909-CL","short":"POP909","metric":"accuracy","explainer":"Given the notes around a beat in a pop song (not audio), picks the chord playing from 129.","baseline":{"value":0.0078,"label":"Uniform random choice"},"lower":false,"range":null,"cases":2000,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2510.06528"},{"kind":"dataset","url":"https://github.com/AndyWeasley2004/POP909-CL-Dataset"}],"private":true},"23":{"id":23,"dataset":"cfcolor","short":"cfcolor","metric":"accuracy","explainer":"Given a person's earlier ratings of color palettes, predicts which of two new palettes they rated higher.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":5000,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://www.dgp.toronto.edu/~donovan/cfcolor/"}]},"24":{"id":24,"dataset":"MMLU","short":"MMLU","metric":"accuracy","explainer":"Four-option questions across 57 school and professional subjects, from law to algebra.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":14042,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/cais/mmlu"}]},"25":{"id":25,"dataset":"GPQA Diamond","short":"GPQA Diamond","metric":"accuracy","explainer":"198 hard four-option science questions written by PhD experts in physics, chemistry and biology.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":198,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/idavidrein/gpqa"}],"gold":1.2,"private":true},"26":{"id":26,"dataset":"ARC-Easy","short":"ARC-Easy","metric":"accuracy","explainer":"Grade-school science multiple choice, the easier set.","baseline":{"value":0.2502,"label":"Uniform random choice"},"lower":false,"range":null,"cases":2376,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/ai2_arc"}]},"27":{"id":27,"dataset":"ARC-Challenge","short":"ARC-Challenge","metric":"accuracy","explainer":"Grade-school science multiple choice, the harder set that simple retrieval gets wrong.","baseline":{"value":0.2502,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1172,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/ai2_arc"}]},"28":{"id":28,"dataset":"WinoGrande","short":"WinoGrande","metric":"accuracy","explainer":"Fills a blank with one of two options using common sense, e.g. who \"got the easier cases\".","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1267,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/winogrande"}],"gold":1.2,"private":true},"29":{"id":29,"dataset":"HellaSwag","short":"HellaSwag","metric":"accuracy","explainer":"Picks the most sensible of four continuations of an everyday scene.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":10042,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/Rowan/hellaswag"}],"gold":1.2,"private":true},"30":{"id":30,"dataset":"GSM8K","short":"GSM8K","metric":"accuracy","explainer":"Picks a math word problem's answer from 4, and separately 10, numbers, with no working. Rebuilt for v0.3.","baseline":{"value":0.175,"label":"Uniform random choice"},"lower":false,"range":null,"cases":2638,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/openai/gsm8k"}],"private":true},"31":{"id":31,"dataset":"ChessBench","short":"ChessBench","metric":"accuracy","explainer":"Given a chess position and every legal move, picks the strongest one; tied best moves count.","baseline":{"value":0.0819,"label":"Uniform random choice"},"lower":false,"range":null,"cases":5000,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2402.04494"},{"kind":"code","url":"https://github.com/google-deepmind/searchless_chess"}],"private":true},"32":{"id":32,"dataset":"MuSR","short":"MuSR","metric":"accuracy","explainer":"Reads a long story and answers one question needing clues from across it, e.g. who the murderer is.","baseline":{"value":0.371,"label":"Uniform random choice"},"lower":false,"range":null,"cases":752,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2310.16049"},{"kind":"dataset","url":"https://github.com/Zayne-sprague/MuSR"}],"private":true},"33":{"id":33,"dataset":"SATA-Bench","short":"SATA-Bench","metric":"case exact accuracy","explainer":"Given a passage, a question and candidate answers, says yes or no to each; all must be right.","baseline":{"value":0.0131,"label":"Random whole-case guesses"},"lower":false,"range":null,"cases":1650,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/sata-bench/sata-bench"}],"private":true},"36":{"id":36,"dataset":"BRIGHT","short":"BRIGHT","metric":"nDCG@10","explainer":"Rates how helpful each of 32 search-hit documents is for a hard question; ranked by nDCG@10, on questions where at least one of the 32 is relevant.","baseline":{"value":0.116,"label":"Random ranking, answerable queries","previous":0.0448},"lower":false,"range":null,"cases":220,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2407.12883"},{"kind":"dataset","url":"https://huggingface.co/datasets/xlangai/BRIGHT"}],"gold":1.2,"private":true},"37":{"id":37,"dataset":"Amazon ESCI","short":"Amazon ESCI","metric":"macro-F1","explainer":"Labels a product as an exact match, substitute, complement or irrelevant to a shopping query.","baseline":{"value":0.2027,"label":"Random choices · F1"},"lower":false,"range":null,"cases":5000,"group":"retrieval","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/amazon-science/esci-data"}],"private":true},"38":{"id":38,"dataset":"ACOS","short":"ACOS","metric":"per-review F1","explainer":"Reads a restaurant or laptop review and says yes/no for each aspect-and-sentiment pair; scored by F1 per review, averaged over reviews.","baseline":{"value":0.031,"label":"Always yes · per-review F1","previous":0},"lower":false,"range":null,"cases":400,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2021.acl-long.29"},{"kind":"dataset","url":"https://github.com/NUSTM/ACOS"}],"private":true},"39":{"id":39,"dataset":"FinEntity","short":"FinEntity","metric":"macro-F1","explainer":"Says whether a financial news snippet is positive, negative or neutral about a named firm.","baseline":{"value":0.3201,"label":"Random choices · F1"},"lower":false,"range":null,"cases":979,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2023.emnlp-main.956"},{"kind":"dataset","url":"https://github.com/yixuantt/FinEntity"}],"private":true},"40":{"id":40,"dataset":"iSarcasmEval","short":"iSarcasmEval","metric":"Sarcasm F1 · track A, English","explainer":"Says yes or no: is this tweet sarcastic? Scored with F1 on sarcastic English tweets (track A); other tracks listed under each result.","baseline":{"value":0.2227,"label":"Random choices · F1"},"lower":false,"range":null,"cases":4600,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://aclanthology.org/2022.semeval-1.111"},{"kind":"dataset","url":"https://github.com/iabufarha/iSarcasmEval"}],"private":true},"41":{"id":41,"dataset":"VAST","short":"VAST","metric":"macro-F1","explainer":"Says whether a comment is for, against, or neutral on a given topic.","baseline":{"value":0.3333,"label":"Random choices · F1"},"lower":false,"range":null,"cases":3006,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/emilyallaway/zero-shot-stance"}],"private":true},"42":{"id":42,"dataset":"NLI4CT","short":"NLI4CT","metric":"macro-F1","explainer":"Says whether clinical-trial report sections support or contradict a statement, some deliberately altered.","baseline":{"value":0.486,"label":"Random choices · F1"},"lower":false,"range":null,"cases":5500,"group":"language","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/ai-systems/Task-2-SemEval-2024"}],"private":true},"43":{"id":43,"dataset":"CRUXEval","short":"CRUXEval","metric":"accuracy","explainer":"Reads a short Python function and an input and picks its output from 2 to 10 candidates, without running it.","baseline":{"value":0.3697,"label":"Uniform random choice"},"lower":false,"range":null,"cases":570,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2401.03065"},{"kind":"dataset","url":"https://github.com/facebookresearch/cruxeval"}],"private":true},"44":{"id":44,"dataset":"CLadder","short":"CLadder","metric":"accuracy","explainer":"Answers a yes/no question about cause and effect in a small made-up world with stated probabilities.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":5000,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2312.04350"},{"kind":"dataset","url":"https://github.com/causalNLP/cladder"}],"private":true},"45":{"id":45,"dataset":"HLE","short":"HLE","metric":"accuracy","explainer":"Expert-level multiple choice from Humanity's Last Exam, text-only questions, 5 to 26 options.","baseline":{"value":0.1641,"label":"Uniform random choice"},"lower":false,"range":null,"cases":501,"group":"knowledge","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/cais/hle"}],"gold":1.2},"50":{"id":50,"dataset":"Habermas Machine","short":"Habermas","metric":"accuracy","explainer":"Given a group's opinions on a policy question, predicts which candidate statement the group ranked highest.","baseline":{"value":0.311,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1676,"group":"arts","in_suite":true,"interactive":false,"references":[{"kind":"dataset","url":"https://github.com/google-deepmind/habermas_machine"}],"private":true},"51":{"id":51,"dataset":"OpenBookQA","short":"OpenBookQA","metric":"accuracy","explainer":"Answer elementary-science questions from four original choices, without the supporting fact.","baseline":{"value":0.25,"label":"Uniform random choice"},"lower":false,"range":null,"cases":500,"group":"knowledge","in_suite":false,"interactive":false},"52":{"id":52,"dataset":"CommonsenseQA","short":"CommonsenseQA","metric":"accuracy","explainer":"Answer everyday commonsense questions from five original choices.","baseline":{"value":0.2,"label":"Uniform random choice"},"lower":false,"range":null,"cases":1221,"group":"knowledge","in_suite":false,"interactive":false},"53":{"id":53,"dataset":"Support-ticket calibration","short":"Support tickets","metric":"accuracy","explainer":"Judge a support ticket for queue, anger and priority, against deliberately noisy labels.","baseline":{"value":null,"label":null},"lower":false,"range":null,"cases":300,"group":"tools","in_suite":false,"interactive":false},"54":{"id":54,"dataset":"Phishing difficulty gradient","short":"Phishing gradient","metric":"accuracy","explainer":"Judge generated emails as phishing across four difficulty tiers, from obvious to misleading.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":800,"group":"retrieval","in_suite":false,"interactive":false},"55":{"id":55,"dataset":"Email spam classification","short":"Email spam","metric":"accuracy","explainer":"Detect spam in real emails, scored on the structured-focus question wording.","baseline":{"value":0.5,"label":"Uniform random choice"},"lower":false,"range":null,"cases":3000,"group":"retrieval","in_suite":false,"interactive":false},"56":{"id":56,"dataset":"PhishNChips phishing decisions","short":"PhishNChips","metric":"accuracy","explainer":"Says whether an email and its link are phishing or legitimate.","baseline":{"value":0.5,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":2000,"group":"retrieval","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"dataset","url":"https://huggingface.co/datasets/AreLit/PhishNChips"},{"kind":"code","url":"https://github.com/anisselbd/jev-phishing-bench"}],"private":true},"57":{"id":57,"dataset":"MMLU-Pro","short":"MMLU-Pro","metric":"accuracy","explainer":"Harder MMLU successor: questions across 14 subjects with up to ten options.","baseline":{"value":0.1109,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":12032,"group":"knowledge","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2406.01574"},{"kind":"dataset","url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro"}],"gold":1.2,"private":true},"58":{"id":58,"dataset":"BBH","short":"BBH","metric":"accuracy","explainer":"23 tricky reasoning tasks from BIG-Bench (logic, dates, tracking objects, fallacies), answered with no working.","baseline":{"value":0.3101,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":5507,"group":"knowledge","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2210.09261"},{"kind":"dataset","url":"https://github.com/suzgunmirac/BIG-Bench-Hard"}],"gold":1.2,"private":true},"59":{"id":59,"dataset":"RAGTruth","short":"RAGTruth","metric":"F1 on hallucinated class","explainer":"Says yes or no: does an AI-written answer contain anything its source text doesn't support? Scored with F1.","baseline":{"value":0.5177,"label":"Always \"hallucinated\" · F1"},"lower":false,"range":null,"cases":2700,"group":"language","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"dataset","url":"https://github.com/ParticleMedia/RAGTruth"}],"private":true},"61":{"id":61,"dataset":"HoVer","short":"HoVer","metric":"accuracy","explainer":"Decides whether a claim linking two to four Wikipedia articles is supported by their introductions.","baseline":{"value":0.5,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":4000,"group":"retrieval","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2011.03088"},{"kind":"dataset","url":"https://github.com/hover-nlp/hover"}],"private":true},"62":{"id":62,"dataset":"When2Call","short":"When2Call","metric":"accuracy","explainer":"Given a question and an assistant's tools, picks the right move: answer, call a tool, ask for details, or decline.","baseline":{"value":0.25,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":3652,"group":"tools","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://aclanthology.org/2025.naacl-long.174"},{"kind":"dataset","url":"https://huggingface.co/datasets/nvidia/When2Call"}],"private":true},"64":{"id":64,"dataset":"New Yorker","short":"New Yorker","metric":"accuracy","explainer":"Given a text description of a New Yorker cartoon, picks the caption written for it from five.","baseline":{"value":0.2,"label":"uniform random choice, mean of 1/options"},"lower":false,"range":null,"cases":528,"group":"arts","in_suite":true,"interactive":false,"edition_added":"v0.2","references":[{"kind":"paper","url":"https://arxiv.org/abs/2209.06293"},{"kind":"dataset","url":"https://huggingface.co/datasets/jmhessel/newyorker_caption_contest"}],"private":true}},"jev":{"engine":"jev","name":"Jev 1.14","version":"jev-1.14.0","edition":"Lab run, 2026-10-14","results":{"1":{"score":0.964,"metric":"case exact accuracy","cases":1694,"median_ms":922.1,"p95_ms":null,"usd":null},"2":{"score":0.6505,"metric":"nDCG@10","cases":685,"median_ms":1012.5,"p95_ms":null,"usd":null},"3":{"score":0.8661,"metric":"accuracy","cases":508,"median_ms":992.8,"p95_ms":null,"usd":null},"4":{"score":0.7994,"metric":"macro-F1","cases":3080,"median_ms":837.1,"p95_ms":null,"usd":null},"5":{"score":0.9028,"metric":"macro-F1","cases":5500,"median_ms":556.4,"p95_ms":null,"usd":null},"6":{"score":0.7988,"metric":"selected quality (quality objective)","cases":10000,"median_ms":508.5,"p95_ms":null,"usd":null},"9":{"score":0.4545,"metric":"case exact accuracy","cases":88,"median_ms":883.7,"p95_ms":null,"usd":null},"10":{"score":0.0903,"metric":"macro-F1","cases":2500,"median_ms":1008.8,"p95_ms":null,"usd":null},"11":{"score":0.7076,"metric":"macro-F1","cases":123,"median_ms":853.8,"p95_ms":null,"usd":null},"12":{"score":0.7427,"metric":"macro-F1","cases":3200,"median_ms":828.8,"p95_ms":null,"usd":null},"20":{"score":0.9006,"metric":"accuracy","cases":5000,"median_ms":585.4,"p95_ms":null,"usd":null},"21":{"score":0.6176,"metric":"accuracy","cases":2628,"median_ms":1001.1,"p95_ms":null,"usd":null},"22":{"score":0.1645,"metric":"accuracy","cases":2000,"median_ms":924.4,"p95_ms":null,"usd":null},"23":{"score":0.6634,"metric":"accuracy","cases":5000,"median_ms":610.2,"p95_ms":null,"usd":null},"24":{"score":0.9167,"metric":"accuracy","cases":14033,"median_ms":525.1,"p95_ms":null,"usd":null},"25":{"score":0.7551,"metric":"accuracy","cases":196,"median_ms":998.8,"p95_ms":null,"usd":null},"26":{"score":0.9933,"metric":"accuracy","cases":2376,"median_ms":919.6,"p95_ms":null,"usd":null},"27":{"score":0.9761,"metric":"accuracy","cases":1172,"median_ms":934.3,"p95_ms":null,"usd":null},"28":{"score":0.9108,"metric":"accuracy","cases":1267,"median_ms":913.4,"p95_ms":null,"usd":null},"29":{"score":0.9434,"metric":"accuracy","cases":10042,"median_ms":480.4,"p95_ms":null,"usd":null},"30":{"score":0.7301,"metric":"accuracy (4- and 10-choice)","cases":2638,"median_ms":null,"p95_ms":null,"usd":null},"31":{"score":0.1688,"metric":"accuracy","cases":5000,"median_ms":587.9,"p95_ms":null,"usd":null},"32":{"score":0.7261,"metric":"accuracy","cases":752,"median_ms":1002.5,"p95_ms":null,"usd":null},"33":{"score":0.2788,"metric":"case exact accuracy","cases":1650,"median_ms":874.9,"p95_ms":null,"usd":null},"36":{"score":0.4705,"metric":"nDCG@10","cases":220,"median_ms":809.3,"p95_ms":null,"usd":null},"37":{"score":0.5718,"metric":"macro-F1","cases":5000,"median_ms":616.6,"p95_ms":null,"usd":null},"38":{"score":0.3314,"metric":"per-review F1","cases":1565,"median_ms":1015.9,"p95_ms":null,"usd":null},"39":{"score":0.8768,"metric":"macro-F1","cases":979,"median_ms":920.9,"p95_ms":null,"usd":null},"40":{"score":0.5327,"metric":"Sarcasm F1 · track A, English","cases":4600,"median_ms":627.3,"p95_ms":null,"usd":null},"41":{"score":0.7365,"metric":"macro-F1","cases":3006,"median_ms":835.8,"p95_ms":null,"usd":null},"42":{"score":0.8512,"metric":"macro-F1","cases":5500,"median_ms":575,"p95_ms":null,"usd":null},"43":{"score":0.7474,"metric":"accuracy","cases":570,"median_ms":1044,"p95_ms":null,"usd":null},"44":{"score":0.8858,"metric":"accuracy","cases":5000,"median_ms":629.8,"p95_ms":null,"usd":null},"45":{"score":0.1936,"metric":"accuracy","cases":501,"median_ms":923.6,"p95_ms":null,"usd":null},"50":{"score":0.4457,"metric":"accuracy","cases":1676,"median_ms":937.3,"p95_ms":null,"usd":null},"56":{"score":0.8375,"metric":"accuracy","cases":2000,"median_ms":1582.9,"p95_ms":null,"usd":null},"57":{"score":0.8213,"metric":"accuracy","cases":12032,"median_ms":517.5,"p95_ms":null,"usd":null},"58":{"score":0.9319,"metric":"accuracy","cases":5507,"median_ms":503.5,"p95_ms":null,"usd":null},"59":{"score":0.7512,"metric":"F1 on hallucinated class","cases":2700,"median_ms":514.2,"p95_ms":null,"usd":null},"61":{"score":0.7675,"metric":"accuracy","cases":4000,"median_ms":510.7,"p95_ms":null,"usd":null},"62":{"score":0.8165,"metric":"accuracy","cases":3652,"median_ms":520,"p95_ms":null,"usd":null},"64":{"score":0.7254,"metric":"accuracy","cases":528,"median_ms":507.8,"p95_ms":null,"usd":null}},"latency":{"median":196,"p95":296.8,"note":"Hosted API, network round-trip from our lab: one sequential HTTPS client on the latency-v1 sample. Not comparable to the on-card single-process figures."},"interactive":[],"references":[{"catalog_id":25,"model":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)","score":0.9374,"metric":"Accuracy"},{"catalog_id":45,"model":"Claude Fable 5.1 (Adaptive Reasoning, Max Effort, Default Fallback)","score":0.5913,"metric":"Accuracy"},{"catalog_id":25,"model":"GPT-6 Astra (max)","score":0.9606,"metric":"Accuracy"},{"catalog_id":45,"model":"GPT-6 Astra (max)","score":0.5468,"metric":"Accuracy"},{"catalog_id":25,"model":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","score":0.9323,"metric":"Accuracy"},{"catalog_id":45,"model":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","score":0.5487,"metric":"Accuracy"},{"catalog_id":25,"model":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","score":0.9263,"metric":"Accuracy"},{"catalog_id":45,"model":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","score":0.5547,"metric":"Accuracy"},{"catalog_id":25,"model":"Muse Spark 1.3 (max)","score":0.9354,"metric":"Accuracy"},{"catalog_id":45,"model":"Muse Spark 1.3 (max)","score":0.487,"metric":"Accuracy"},{"catalog_id":25,"model":"GPT-5.6 Sol (max)","score":0.9414,"metric":"Accuracy"},{"catalog_id":45,"model":"GPT-5.6 Sol (max)","score":0.4949,"metric":"Accuracy"},{"catalog_id":25,"model":"Qwen3.8 Max (0902)","score":0.9283,"metric":"Accuracy"},{"catalog_id":45,"model":"Qwen3.8 Max (0902)","score":0.431,"metric":"Accuracy"},{"catalog_id":25,"model":"GLM-5.3 (max)","score":0.9172,"metric":"Accuracy"},{"catalog_id":45,"model":"GLM-5.3 (max)","score":0.4226,"metric":"Accuracy"},{"catalog_id":25,"model":"Grok 4.6 (high)","score":0.9495,"metric":"Accuracy"},{"catalog_id":45,"model":"Grok 4.6 (high)","score":0.4291,"metric":"Accuracy"},{"catalog_id":25,"model":"Kimi K3 (max)","score":0.9354,"metric":"Accuracy"},{"catalog_id":45,"model":"Kimi K3 (max)","score":0.469,"metric":"Accuracy"},{"catalog_id":45,"model":"Step 5 Preview","score":0.4648,"metric":"Accuracy"},{"catalog_id":25,"model":"GPT-5.6 Terra (max)","score":0.9253,"metric":"Accuracy"},{"catalog_id":45,"model":"GPT-5.6 Terra (max)","score":0.4291,"metric":"Accuracy"},{"catalog_id":25,"model":"GLM 5.3 Flash","score":0.9121,"metric":"Accuracy"},{"catalog_id":45,"model":"GLM 5.3 Flash","score":0.3985,"metric":"Accuracy"},{"catalog_id":25,"model":"Gemini 3.8 Flash (high)","score":0.9525,"metric":"Accuracy"},{"catalog_id":45,"model":"Gemini 3.8 Flash (high)","score":0.4782,"metric":"Accuracy"},{"catalog_id":45,"model":"DeepSeek V4.1 Flash (Reasoning, Max Effort)","score":0.3925,"metric":"Accuracy"},{"catalog_id":34,"model":"Claude Fable 5","score":0.819,"metric":"AVG@5 accuracy"},{"catalog_id":34,"model":"Claude Opus 5","score":0.806,"metric":"AVG@5 accuracy"}],"summary_notes":["Run by the lab through the TypeSafe API (model id jev-1.14.0) with the same clients, rows and scoring as the Jev 1.13 run; only the model id differs.","Metrics exclude failed/incomplete cases; coverage is explicit. No silent replacement of failures.","Latency includes HTTPS and service scheduling; not isolated model inference.","The interactive games and the extra benchmarks of the Jev 1.13 release run were not run for 1.14."],"calibration":{"acc":0.7431,"conf":0.8165,"ece":0.0799,"brier":0.3567,"over95":0.0265,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0005,"conf":0.0843,"acc":0.1429},{"lo":0.1,"w":0.0225,"conf":0.1574,"acc":0.1142},{"lo":0.2,"w":0.0307,"conf":0.2411,"acc":0.1246},{"lo":0.3,"w":0.0276,"conf":0.3466,"acc":0.4647},{"lo":0.4,"w":0.0324,"conf":0.4529,"acc":0.4043},{"lo":0.5,"w":0.0689,"conf":0.5476,"acc":0.5088},{"lo":0.6,"w":0.0743,"conf":0.6442,"acc":0.5259},{"lo":0.7,"w":0.0787,"conf":0.7453,"acc":0.6231},{"lo":0.8,"w":0.1066,"conf":0.848,"acc":0.7007},{"lo":0.9,"w":0.5577,"conf":0.98,"acc":0.9196}],"areas":{"knowledge":{"ece":0.0658,"acc":0.7286,"conf":0.7741},"language":{"ece":0.0424,"acc":0.84,"conf":0.8792},"retrieval":{"ece":0.2445,"acc":0.6207,"conf":0.8646},"tools":{"ece":0.0275,"acc":0.9414,"conf":0.9643},"arts":{"ece":0.0854,"acc":0.5768,"conf":0.6611}}},"scores":{"balanced_raw":69.98,"balanced_skill":60.24,"breadth_skill":60.07,"public_skill":60.7},"frozen_scores":{"balanced_raw":69.98,"balanced_skill":60.7,"breadth_skill":60.07},"categories":[{"id":"knowledge","raw":0.656,"skill":0.5712,"combined":0.4655,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7245,"skill":0.6118,"combined":0.58,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7249,"skill":0.6353,"combined":0.648,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7625,"skill":0.7378,"combined":0.7443,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5838,"skill":0.3916,"combined":0.4772,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.964,"skill":0.9514,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8444,"private_eq":0.8853},"2":{"raw":0.6505,"skill":0.5964,"coverage":1,"random":0.1341,"tracks":[],"combined":0.653,"private_eq":0.7021},"3":{"raw":0.8661,"skill":0.8635,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8855,"private_eq":0.888},"4":{"raw":0.7994,"skill":0.7968,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8698,"private_eq":0.8719},"5":{"raw":0.9028,"skill":0.9022,"coverage":1,"random":0.006,"tracks":[],"combined":0.736,"private_eq":0.7396},"6":{"raw":0.7988,"skill":0.5283,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4545,"skill":0.4545,"coverage":1,"random":0,"tracks":[],"combined":0.5838,"private_eq":0.5875},"10":{"raw":0.0903,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7076,"skill":0.5771,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7427,"skill":0.6146,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4933,"private_eq":0.6649},"20":{"raw":0.9041,"skill":0.8082,"coverage":1,"random":0.5,"tracks":[],"combined":0.8637,"private_eq":0.9321},"21":{"raw":0.6176,"skill":0.2352,"coverage":1,"random":0.5,"tracks":[],"combined":0.3069,"private_eq":0.6569},"22":{"raw":0.1596,"skill":0.153,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1888,"private_eq":0.2033},"23":{"raw":0.6504,"skill":0.3007,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.9167,"skill":0.8889,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.7551,"skill":0.6735,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5532,"combined":0.3984},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9761,"skill":0.9681,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9108,"skill":0.8216,"coverage":1,"random":0.5,"tracks":[],"combined":0.6632,"private_eq":0.833},"29":{"raw":0.9434,"skill":0.9245,"coverage":1,"random":0.25,"tracks":[],"combined":0.9681,"private_eq":0.9756},"30":{"raw":0.7301,"skill":0.6784,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8044,"random":0.25},{"track":"10-choice","raw":0.6558,"random":0.1}],"combined":0.7605,"private_eq":0.8038},"31":{"raw":0.1688,"skill":0.0947,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.7261,"skill":0.5645,"coverage":1,"random":0.371,"tracks":[],"combined":0.3201,"private_eq":0.5766},"33":{"raw":0.2788,"skill":0.2692,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2046,"private_eq":0.223},"36":{"raw":0.4705,"skill":0.401,"coverage":1,"random":0.116,"tracks":[],"combined":0.3887,"private_eq":0.4649},"37":{"raw":0.5718,"skill":0.463,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6048,"private_eq":0.6877},"38":{"raw":0.3314,"skill":0.31,"coverage":1,"random":0.031,"tracks":[],"combined":0.4874,"private_eq":0.508},"39":{"raw":0.8768,"skill":0.8188,"coverage":1,"random":0.3201,"tracks":[],"combined":0.565,"private_eq":0.7069},"40":{"raw":0.5327,"skill":0.3989,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5145,"headline":false},{"track":"A · English","score":0.5327,"headline":true},{"track":"C · Arabic pairs","score":0.795,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"combined":0.3662,"private_eq":0.5122},"41":{"raw":0.7365,"skill":0.6047,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5031,"private_eq":0.6718},"42":{"raw":0.8512,"skill":0.7105,"coverage":1,"random":0.486,"tracks":[],"combined":0.8068,"private_eq":0.9013},"43":{"raw":0.7474,"skill":0.5992,"coverage":1,"random":0.3697,"tracks":[],"combined":0.583,"private_eq":0.7395},"44":{"raw":0.8858,"skill":0.7716,"coverage":1,"random":0.5,"tracks":[],"combined":0.7226,"private_eq":0.8624},"45":{"raw":0.1936,"skill":0.0353,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4457,"skill":0.1955,"coverage":1,"random":0.311,"tracks":[],"combined":0.3264,"private_eq":0.5405},"56":{"raw":0.8375,"skill":0.675,"coverage":1,"random":0.5,"tracks":[],"combined":0.6462,"private_eq":0.8246},"57":{"raw":0.8213,"skill":0.799,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6484,"combined":0.601},"58":{"raw":0.9319,"skill":0.9013,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5156,"combined":0.2908},"59":{"raw":0.7512,"skill":0.4841,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3897,"private_eq":0.7085},"61":{"raw":0.7675,"skill":0.535,"coverage":1,"random":0.5,"tracks":[],"combined":0.6322,"private_eq":0.8177},"62":{"raw":0.8165,"skill":0.7553,"coverage":1,"random":0.25,"tracks":[],"combined":0.7067,"private_eq":0.7818},"64":{"raw":0.7254,"skill":0.6568,"coverage":1,"random":0.2,"tracks":[],"combined":0.7003,"private_eq":0.7622}},"coverage":1,"pending":0,"v03":{"engine":"jev","pub":60.7,"same_skills":59.28,"new_domains":48.26,"v":60.24,"lab":false,"jev":true,"rank_v":48,"rank_pub":57,"name":"Jev 1.14","eq":{"S":58.91,"O":62.24},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.34,"new_domains":24.9}},"meta":{"org":{"id":"typesafe-ai","name":"typesafe.ai","logo":"data/orgs/typesafe-ai.png"}}},"models":[{"engine":"decider-chat-gemma4-31b","name":"Decider chat · Gemma-4-31B","long_name":"Decider chat · Gemma-4-31B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":32682372656,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/Mapika/decider","base_model":"google/gemma-4-31B","base_url":"https://huggingface.co/google/gemma-4-31B","weights_repo":null,"served_checkpoint":"google/gemma-4-31B-it"},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.724,"conf":0.7711,"ece":0.0471,"brier":0.3928,"over95":0.0049,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0007,"conf":0.0908,"acc":0.0511},{"lo":0.1,"w":0.0074,"conf":0.1552,"acc":0.1275},{"lo":0.2,"w":0.0164,"conf":0.2571,"acc":0.1647},{"lo":0.3,"w":0.0302,"conf":0.3557,"acc":0.2335},{"lo":0.4,"w":0.0488,"conf":0.4538,"acc":0.3905},{"lo":0.5,"w":0.0689,"conf":0.5529,"acc":0.5073},{"lo":0.6,"w":0.0853,"conf":0.651,"acc":0.5212},{"lo":0.7,"w":0.1592,"conf":0.7577,"acc":0.6701},{"lo":0.8,"w":0.3135,"conf":0.848,"acc":0.8283},{"lo":0.9,"w":0.2696,"conf":0.9374,"acc":0.9216}],"areas":{"knowledge":{"ece":0.0515,"acc":0.6931,"conf":0.6804},"language":{"ece":0.0254,"acc":0.8154,"conf":0.8204},"retrieval":{"ece":0.2449,"acc":0.6096,"conf":0.8545},"tools":{"ece":0.0472,"acc":0.9386,"conf":0.8952},"arts":{"ece":0.1508,"acc":0.59,"conf":0.7408}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":108.5,"p95":1114.7,"mean":358.6},"results":{"1":{"score":0.9799,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":328.5},"2":{"score":0.6359,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":5679.8},"3":{"score":0.8504,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7907,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":294.3},"5":{"score":0.9116,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":502.5},"6":{"score":0.8,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":741.6},"9":{"score":0.625,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.031,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":503.4},"11":{"score":0.7356,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7316,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":79.4},"20":{"score":0.9026,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.3},"21":{"score":0.6381,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":71.8},"22":{"score":0.285,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2664.9},"23":{"score":0.6408,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":145.4},"24":{"score":0.863,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":72.3},"25":{"score":0.4898,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9882,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":71.8},"27":{"score":0.9744,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":71.8},"28":{"score":0.8366,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":71.8},"29":{"score":0.9202,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":100.1},"30":{"score":0.7104,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2234,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":241.3},"32":{"score":0.6423,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2933,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":877.4},"36":{"score":0.4359,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5311,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":108.5},"38":{"score":0.1883,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2773},"39":{"score":0.9087,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5107,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":71.8,"tracks":[{"track":"A · Arabic","score":0.5099,"headline":false},{"track":"A · English","score":0.5107,"headline":true},{"track":"C · Arabic pairs","score":0.92,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.732,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":80.1},"42":{"score":0.8285,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":207.7},"43":{"score":0.793,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.746,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.5},"45":{"score":0.1477,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4791,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":241.8},"56":{"score":0.875,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1112.6,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.6964,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":82.6,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.7621,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":79.6,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7694,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":208.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.7642,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":125.1,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7689,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":240.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.7386,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":68.01,"balanced_skill":61.63,"breadth_skill":56.86,"public_skill":57.79},"frozen_scores":{"balanced_raw":68.01,"balanced_skill":57.79,"breadth_skill":56.86},"categories":[{"id":"knowledge","raw":0.5773,"skill":0.4546,"combined":0.4479,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.708,"skill":0.5949,"combined":0.5552,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7176,"skill":0.6308,"combined":0.6343,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7826,"skill":0.7557,"combined":0.7051,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6111,"skill":0.4218,"combined":0.4434,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9799,"skill":0.9729,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9329,"private_eq":0.9501},"2":{"raw":0.6359,"skill":0.5796,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6846,"private_eq":0.7291},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9276,"private_eq":0.9288},"4":{"raw":0.7907,"skill":0.788,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8581,"private_eq":0.8605},"5":{"raw":0.9116,"skill":0.9111,"coverage":1,"random":0.006,"tracks":[],"combined":0.7483,"private_eq":0.7517},"6":{"raw":0.8,"skill":0.5285,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7864,"headline":false},{"track":"RouterBench-5shot","score":0.8136,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.625,"skill":0.625,"coverage":1,"random":0,"tracks":[],"combined":0.2929,"private_eq":0.3},"11":{"raw":0.7356,"skill":0.6176,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7316,"skill":0.598,"coverage":1,"random":0.3324,"tracks":[],"combined":0.466,"private_eq":0.6469},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.9059,"skill":0.8117,"coverage":1,"random":0.5,"tracks":[],"combined":0.8566,"private_eq":0.9286},"21":{"raw":0.6381,"skill":0.2763,"coverage":1,"random":0.5,"tracks":[],"combined":0.2264,"private_eq":0.6171},"22":{"raw":0.2785,"skill":0.2729,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2476,"private_eq":0.261},"23":{"raw":0.6263,"skill":0.2525,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8651,"skill":0.8201,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4898,"skill":0.3197,"coverage":1,"random":0.25,"tracks":[],"combined":0.3466,"private_eq":0.5148},"28":{"raw":0.8366,"skill":0.6732,"coverage":1,"random":0.5,"tracks":[],"combined":0.6278,"private_eq":0.8155},"29":{"raw":0.9202,"skill":0.8936,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7104,"skill":0.6539,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7809,"random":0.25},{"track":"10-choice","raw":0.6399,"random":0.1}],"combined":0.7499,"private_eq":0.7952},"31":{"raw":0.2234,"skill":0.1542,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1005,"private_eq":0.1827},"32":{"raw":0.6423,"skill":0.4313,"coverage":1,"random":0.371,"tracks":[],"combined":0.3899,"private_eq":0.62},"33":{"raw":0.2933,"skill":0.2839,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4307,"private_eq":0.4436},"36":{"raw":0.4359,"skill":0.3619,"coverage":1,"random":0.116,"tracks":[],"combined":0.3828,"private_eq":0.4597},"37":{"raw":0.5311,"skill":0.4118,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5434,"private_eq":0.6393},"38":{"raw":0.1883,"skill":0.1623,"coverage":1,"random":0.031,"tracks":[],"combined":0.302,"private_eq":0.3304},"39":{"raw":0.9087,"skill":0.8657,"coverage":1,"random":0.3201,"tracks":[],"combined":0.599,"private_eq":0.7298},"40":{"raw":0.5107,"skill":0.3705,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5099,"headline":false},{"track":"A · English","score":0.5107,"headline":true},{"track":"C · Arabic pairs","score":0.92,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.4205,"private_eq":0.5539},"41":{"raw":0.732,"skill":0.5981,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4403,"private_eq":0.6304},"42":{"raw":0.8285,"skill":0.6663,"coverage":1,"random":0.486,"tracks":[],"combined":0.8072,"private_eq":0.9015},"43":{"raw":0.793,"skill":0.6715,"coverage":1,"random":0.3697,"tracks":[],"combined":0.583,"private_eq":0.7395},"44":{"raw":0.746,"skill":0.492,"coverage":1,"random":0.5,"tracks":[],"combined":0.4174,"private_eq":0.7115},"45":{"raw":0.1477,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4791,"skill":0.244,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.875,"skill":0.75,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.6462,"private_eq":0.8246},"57":{"raw":0.6964,"skill":0.6585,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.5011,"private_eq":0.5606},"58":{"raw":0.7621,"skill":0.6552,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.3301,"private_eq":0.5424},"59":{"raw":0.7694,"skill":0.5219,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.3339,"private_eq":0.6819},"61":{"raw":0.7642,"skill":0.5284,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.6096,"private_eq":0.8065},"62":{"raw":0.7689,"skill":0.6919,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5972,"private_eq":0.7006},"64":{"raw":0.7386,"skill":0.6732,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.6615,"private_eq":0.7315}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"decider2.serve_chat_adaptive over serve_chat_lowmem as board (graph server serve_chat OOMs on 96 GB); one sequential HTTP client; card venv torch 2.14/transformers 5.17 instead of the job's pytorch 2.10 image","v03":{"engine":"decider-chat-gemma4-31b","pub":57.79,"same_skills":56.88,"new_domains":54.01,"v":61.63,"lab":false,"jev":false,"rank_v":41,"rank_pub":81,"name":"Decider chat · Gemma-4-31B","eq":{"S":56.48,"O":69.35},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":33.89,"new_domains":27.74}}},{"engine":"rune-26b-a4b-v3","name":"Surogate Rune 26B-A4B v3","long_name":"Surogate Rune 26B-A4B v3","variant":"bf16","meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_estimated":false,"params_from_name":false,"trained_bytes":51612008580,"model_url":"https://huggingface.co/surogate/rune-26b-a4b-GGUF","code_url":"https://github.com/invergent-ai/surogate","base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":"surogate/rune-26b-a4b-GGUF","served_checkpoint":"bd4a7cbbed66af1a95dfbac12cabf52e6e677411","org":{"id":"surogate","name":"Surogate","logo":"data/orgs/surogate.png"}},"completed":109758,"counts":{"ok":109758,"error":1},"lifted":false,"calibration":{"acc":0.7476,"conf":0.8672,"ece":0.1196,"brier":0.3704,"over95":0.0482,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0001,"conf":0.0918,"acc":0},{"lo":0.1,"w":0.0063,"conf":0.1638,"acc":0.0665},{"lo":0.2,"w":0.0109,"conf":0.2533,"acc":0.1317},{"lo":0.3,"w":0.0224,"conf":0.3525,"acc":0.2547},{"lo":0.4,"w":0.0307,"conf":0.454,"acc":0.3172},{"lo":0.5,"w":0.059,"conf":0.5486,"acc":0.422},{"lo":0.6,"w":0.055,"conf":0.6485,"acc":0.4686},{"lo":0.7,"w":0.0684,"conf":0.7551,"acc":0.5358},{"lo":0.8,"w":0.0918,"conf":0.8543,"acc":0.6461},{"lo":0.9,"w":0.6553,"conf":0.9819,"acc":0.8906}],"areas":{"knowledge":{"ece":0.1302,"acc":0.673,"conf":0.8032},"language":{"ece":0.0787,"acc":0.8334,"conf":0.9121},"retrieval":{"ece":0.1446,"acc":0.7601,"conf":0.9018},"tools":{"ece":0.0574,"acc":0.915,"conf":0.9603},"arts":{"ece":0.1981,"acc":0.6464,"conf":0.8434}}},"gaps":{"share":0,"reasons":[{"reason":"runtime error","share":0}]},"latency":{"median":120.5,"p95":682.8,"mean":267.2},"results":{"1":{"score":0.9481,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":250.8},"2":{"score":0.6439,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1890.7},"3":{"score":0.8327,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8375,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":289.4},"5":{"score":0.874,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":384},"6":{"score":0.7987,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":572.4},"9":{"score":0.4659,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.664,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":366.1},"11":{"score":0.7697,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7352,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":83.9},"20":{"score":0.8956,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.2},"21":{"score":0.6221,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":56.9},"22":{"score":0.668,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1981.2},"23":{"score":0.6314,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":172.7},"24":{"score":0.8391,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":85.3},"25":{"score":0.4694,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9903,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":85.1},"27":{"score":0.9684,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":85},"28":{"score":0.8595,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":53.3},"29":{"score":0.924,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":115.3},"30":{"score":0.771,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1874,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":243.8},"32":{"score":0.6489,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3485,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":315.5},"36":{"score":0.4632,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5529,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":118.4},"38":{"score":0.2676,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1802.5},"39":{"score":0.8843,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6038,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":84.6,"tracks":[{"track":"A · Arabic","score":0.5258,"headline":false},{"track":"A · English","score":0.6038,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.7662,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":85.8},"42":{"score":0.8044,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":177.3},"43":{"score":0.7439,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7274,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84},"45":{"score":0.106,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":0,"errors":1,"median_ms":null},"50":{"score":0.4224,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":273.1},"56":{"score":0.809,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":314.3,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.672,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":86.7,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.8115,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":90.5,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.768,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":241.8,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.8067,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":148.8,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7601,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":229.4,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.7405,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":68.45,"balanced_skill":56.42,"breadth_skill":57.59,"public_skill":58.28},"frozen_scores":{"balanced_raw":68.45,"balanced_skill":58.28,"breadth_skill":57.59},"categories":[{"id":"knowledge","raw":0.5774,"skill":0.4595,"combined":0.4198,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.7293,"skill":0.6193,"combined":0.5863,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.724,"skill":0.6352,"combined":0.6231,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.742,"skill":0.7122,"combined":0.6902,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6618,"skill":0.465,"combined":0.5197,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9481,"skill":0.9299,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8921,"private_eq":0.9202},"2":{"raw":0.6439,"skill":0.5887,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6665,"private_eq":0.7136},"3":{"raw":0.8327,"skill":0.8295,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8278,"private_eq":0.832},"4":{"raw":0.8375,"skill":0.8354,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8721,"private_eq":0.8742},"5":{"raw":0.874,"skill":0.8732,"coverage":1,"random":0.006,"tracks":[],"combined":0.7345,"private_eq":0.7381},"6":{"raw":0.7987,"skill":0.5251,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7864,"headline":false},{"track":"RouterBench-5shot","score":0.8111,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.4659,"skill":0.4659,"coverage":1,"random":0,"tracks":[],"combined":0.427,"private_eq":0.4325},"11":{"raw":0.7697,"skill":0.667,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7352,"skill":0.6034,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5571,"private_eq":0.707},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8996,"skill":0.7991,"coverage":1,"random":0.5,"tracks":[],"combined":0.8204,"private_eq":0.9107},"21":{"raw":0.6221,"skill":0.2443,"coverage":1,"random":0.5,"tracks":[],"combined":0.3371,"private_eq":0.6718},"22":{"raw":0.6607,"skill":0.6581,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4821,"private_eq":0.491},"23":{"raw":0.6256,"skill":0.2513,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8447,"skill":0.793,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4694,"skill":0.2925,"coverage":1,"random":0.25,"tracks":[],"combined":0.3241,"private_eq":0.4981},"28":{"raw":0.8595,"skill":0.719,"coverage":1,"random":0.5,"tracks":[],"combined":0.5369,"private_eq":0.7706},"29":{"raw":0.924,"skill":0.8987,"coverage":1,"random":0.25,"tracks":[],"combined":0.9584,"private_eq":0.9684},"30":{"raw":0.771,"skill":0.7261,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8249,"random":0.25},{"track":"10-choice","raw":0.7172,"random":0.1}],"combined":0.7768,"private_eq":0.8171},"31":{"raw":0.1874,"skill":0.115,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0766,"private_eq":0.161},"32":{"raw":0.6489,"skill":0.4418,"coverage":1,"random":0.371,"tracks":[],"combined":0.297,"private_eq":0.5622},"33":{"raw":0.3485,"skill":0.3399,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4565,"private_eq":0.4687},"36":{"raw":0.4632,"skill":0.3928,"coverage":1,"random":0.116,"tracks":[],"combined":0.3454,"private_eq":0.427},"37":{"raw":0.5529,"skill":0.4393,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4999,"private_eq":0.605},"38":{"raw":0.2676,"skill":0.2442,"coverage":1,"random":0.031,"tracks":[],"combined":0.4379,"private_eq":0.4606},"39":{"raw":0.8843,"skill":0.8298,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5569,"private_eq":0.7015},"40":{"raw":0.6038,"skill":0.4903,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5258,"headline":false},{"track":"A · English","score":0.6038,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.4899,"private_eq":0.6072},"41":{"raw":0.7662,"skill":0.6493,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4937,"private_eq":0.6656},"42":{"raw":0.8044,"skill":0.6195,"coverage":1,"random":0.486,"tracks":[],"combined":0.7855,"private_eq":0.8905},"43":{"raw":0.7439,"skill":0.5936,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5244,"private_eq":0.703},"44":{"raw":0.7274,"skill":0.4548,"coverage":1,"random":0.5,"tracks":[],"combined":0.4977,"private_eq":0.7512},"45":{"raw":0.1058,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.4224,"skill":0.1618,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.809,"skill":0.618,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.6723,"private_eq":0.8375},"57":{"raw":0.672,"skill":0.6311,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.4689,"private_eq":0.5323},"58":{"raw":0.8115,"skill":0.7268,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.2571,"private_eq":0.4926},"59":{"raw":0.768,"skill":0.519,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.3427,"private_eq":0.6861},"61":{"raw":0.8067,"skill":0.6134,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5982,"private_eq":0.8009},"62":{"raw":0.7601,"skill":0.6801,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5698,"private_eq":0.6803},"64":{"raw":0.7405,"skill":0.6756,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.7489,"private_eq":0.8006}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"surogate v1.5.3 release wheel (pinned rune-v3-audit setup: uv py3.12.14, torch 2.11.0+cu130, noble glibc shim) serving the rune-v3 bf16 safetensors, surogate serve --max-model-len 262144 --kv-capacity 262144 --max-num-seqs 8, POST /v1/decisions with the whole row in one request (shared state prefill), one sequential client via run.py (latency_run_extra.py wrapper, engine latency_rune.RuneV3Latency)","v03":{"engine":"rune-26b-a4b-v3","pub":58.28,"same_skills":57.09,"new_domains":43.2,"v":56.42,"lab":false,"jev":false,"rank_v":70,"rank_pub":73,"name":"Surogate Rune 26B-A4B v3","eq":{"S":56.7,"O":56},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":34.02,"new_domains":22.4}}},{"engine":"decision-2.0-vega-27b","name":"vLLM-SR Decision 2.0 Vega 27B","long_name":"vLLM-SR Decision 2.0 Vega 27B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":27800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/Decision-2.0-Vega-27B","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7315,"conf":0.6454,"ece":0.0861,"brier":0.3758,"over95":0.0004,"n":34833,"benchmarks":32,"rel":[{"lo":0,"w":0.0569,"conf":0.0544,"acc":0.3176},{"lo":0.1,"w":0.0173,"conf":0.144,"acc":0.3124},{"lo":0.2,"w":0.0251,"conf":0.26,"acc":0.3476},{"lo":0.3,"w":0.0601,"conf":0.3546,"acc":0.3869},{"lo":0.4,"w":0.0856,"conf":0.4512,"acc":0.483},{"lo":0.5,"w":0.1475,"conf":0.5507,"acc":0.6159},{"lo":0.6,"w":0.1338,"conf":0.6499,"acc":0.7772},{"lo":0.7,"w":0.1396,"conf":0.7504,"acc":0.8711},{"lo":0.8,"w":0.164,"conf":0.8515,"acc":0.9203},{"lo":0.9,"w":0.17,"conf":0.9453,"acc":0.984}],"areas":{"knowledge":{"ece":0.0596,"acc":0.6658,"conf":0.6334},"language":{"ece":0.102,"acc":0.8155,"conf":0.7135},"retrieval":{"ece":0.2108,"acc":0.7344,"conf":0.5263},"tools":{"ece":0.0605,"acc":0.9418,"conf":0.8813},"arts":{"ece":0.1105,"acc":0.5961,"conf":0.5027}}},"gaps":null,"latency":{"median":119.6,"p95":null,"mean":239.2},"results":{"1":{"score":0.9563,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":265.1},"2":{"score":0.6824,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3076.4},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":955.7},"4":{"score":0.8806,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":312.7},"5":{"score":0.9356,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":509.3},"6":{"score":0.7962,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":656.9},"9":{"score":0.6591,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2451.9},"10":{"score":0.5817,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":205.2},"11":{"score":0.7611,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":4660.4},"12":{"score":0.7367,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":95.3},"20":{"score":0.911,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":97.2},"21":{"score":0.6202,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":95.6},"22":{"score":0.3655,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":962.4},"23":{"score":0.6324,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":127.2},"24":{"score":0.8226,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":93.3},"25":{"score":0.4235,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":100},"26":{"score":0.9891,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":92},"27":{"score":0.9642,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":91.5},"28":{"score":0.8548,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":91.6},"29":{"score":0.9414,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":101.7},"30":{"score":0.7638,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1752,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":217},"32":{"score":0.6569,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":204},"33":{"score":0.317,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":528.8},"36":{"score":0.4875,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5824.3},"37":{"score":0.5673,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":118.2},"38":{"score":0.2246,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1476.9},"39":{"score":0.7983,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":116.3},"40":{"score":0.5621,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":108.2},"41":{"score":0.5985,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":104.8},"42":{"score":0.8506,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":133.3},"43":{"score":0.8246,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":101.7},"44":{"score":0.7472,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":102.2},"45":{"score":0.1018,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":111.4},"50":{"score":0.4111,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":189.5},"56":{"score":0.614,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":386.7},"57":{"score":0.6484,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":109.3},"58":{"score":0.8066,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":104.5},"59":{"score":0.8144,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":179.1},"61":{"score":0.8257,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":127.6},"62":{"score":0.6457,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":159.3},"64":{"score":0.7311,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":100.3}},"benchmarks":{"1":{"raw":0.9563,"skill":0.941,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.6824,"skill":0.6332,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7089,"private_eq":0.7499},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8804,"private_eq":0.883},"4":{"raw":0.8806,"skill":0.8791,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8692,"private_eq":0.8713},"5":{"raw":0.9356,"skill":0.9352,"coverage":1,"random":0.006,"tracks":[],"combined":0.8015,"private_eq":0.8039},"6":{"raw":0.7962,"skill":0.5222,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6591,"skill":0.6591,"coverage":1,"random":0,"tracks":[],"combined":0.6445,"private_eq":0.6475},"10":{"raw":0.5817,"skill":0.304,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7611,"skill":0.6544,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7367,"skill":0.6056,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4722,"private_eq":0.651},"20":{"raw":0.9131,"skill":0.8263,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.6202,"skill":0.2405,"coverage":1,"random":0.5,"tracks":[],"combined":0.2466,"private_eq":0.6271},"22":{"raw":0.372,"skill":0.3671,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2464,"private_eq":0.2598},"23":{"raw":0.6394,"skill":0.2788,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8226,"skill":0.7635,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4235,"skill":0.2313,"coverage":1,"random":0.25,"tracks":[],"combined":0.2751,"private_eq":0.4618},"26":{"raw":0.9891,"skill":0.9855,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9642,"skill":0.9523,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8548,"skill":0.7096,"coverage":1,"random":0.5,"tracks":[],"combined":0.6025,"private_eq":0.803},"29":{"raw":0.9414,"skill":0.9219,"coverage":1,"random":0.25,"tracks":[],"combined":0.9286,"private_eq":0.9463},"30":{"raw":0.7638,"skill":0.7166,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8112,"random":0.25},{"track":"10-choice","raw":0.7165,"random":0.1}],"combined":0.7336,"private_eq":0.7819},"31":{"raw":0.1752,"skill":0.1017,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0582,"private_eq":0.1443},"32":{"raw":0.6569,"skill":0.4545,"coverage":1,"random":0.371,"tracks":[],"combined":0.3356,"private_eq":0.5862},"33":{"raw":0.317,"skill":0.3079,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2406,"private_eq":0.2581},"36":{"raw":0.4875,"skill":0.4202,"coverage":1,"random":0.116,"tracks":[],"combined":0.4044,"private_eq":0.4786},"37":{"raw":0.5673,"skill":0.4573,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5548,"private_eq":0.6483},"38":{"raw":0.2246,"skill":0.1998,"coverage":1,"random":0.031,"tracks":[],"combined":0.3607,"private_eq":0.3866},"39":{"raw":0.7983,"skill":0.7033,"coverage":1,"random":0.3201,"tracks":[],"combined":0.476,"private_eq":0.6471},"40":{"raw":0.5621,"skill":0.4367,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5232,"headline":false},{"track":"A · English","score":0.5621,"headline":true},{"track":"C · Arabic pairs","score":0.93,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.2719,"private_eq":0.4397},"41":{"raw":0.5985,"skill":0.3978,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4871,"private_eq":0.6613},"42":{"raw":0.8506,"skill":0.7093,"coverage":1,"random":0.486,"tracks":[],"combined":0.9211,"private_eq":0.9594},"43":{"raw":0.8246,"skill":0.7217,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5619,"private_eq":0.7264},"44":{"raw":0.7472,"skill":0.4944,"coverage":1,"random":0.5,"tracks":[],"combined":0.4174,"private_eq":0.7115},"45":{"raw":0.1018,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4111,"skill":0.1453,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.614,"skill":0.228,"coverage":1,"random":0.5,"tracks":[],"combined":0.4386,"private_eq":0.722},"57":{"raw":0.6484,"skill":0.6045,"coverage":1,"random":0.1109,"tracks":[],"combined":0.3488,"private_eq":0.4267},"58":{"raw":0.8066,"skill":0.7197,"coverage":1,"random":0.3101,"tracks":[],"combined":0.2178,"private_eq":0.4658},"59":{"raw":0.8144,"skill":0.6152,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3303,"private_eq":0.6802},"61":{"raw":0.8257,"skill":0.6514,"coverage":1,"random":0.5,"tracks":[],"combined":0.547,"private_eq":0.7756},"62":{"raw":0.6457,"skill":0.5276,"coverage":1,"random":0.25,"tracks":[],"combined":0.6384,"private_eq":0.7311},"64":{"raw":0.7311,"skill":0.6639,"coverage":1,"random":0.2,"tracks":[],"combined":0.6615,"private_eq":0.7315}},"coverage":1,"pending":0,"scores":{"balanced_raw":67.75,"balanced_skill":58.01,"breadth_skill":56.18,"public_skill":56.97},"frozen_scores":{"balanced_raw":67.75,"balanced_skill":56.97,"breadth_skill":56.18},"categories":[{"id":"knowledge","raw":0.5739,"skill":0.4596,"combined":0.3778,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7046,"skill":0.5904,"combined":0.5391,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7229,"skill":0.6088,"combined":0.6107,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7704,"skill":0.7354,"combined":0.7748,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6145,"skill":0.4203,"combined":0.4558,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"decision-2.0-vega-27b","pub":56.97,"same_skills":55.71,"new_domains":48.12,"v":58.01,"lab":false,"jev":false,"rank_v":65,"rank_pub":91,"name":"vLLM-SR Decision 2.0 Vega 27B","eq":{"S":55.3,"O":62.07},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":33.18,"new_domains":24.83}}},{"engine":"kev-27b","name":"Kev 27B","long_name":"Kev 27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/jaredpalmer/kev-27b","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7322,"conf":0.7281,"ece":0.0218,"brier":0.3591,"over95":0.007,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0338,"conf":0.072,"acc":0.1569},{"lo":0.1,"w":0.023,"conf":0.1374,"acc":0.3039},{"lo":0.2,"w":0.0166,"conf":0.2576,"acc":0.2574},{"lo":0.3,"w":0.0416,"conf":0.3517,"acc":0.4192},{"lo":0.4,"w":0.06,"conf":0.4534,"acc":0.428},{"lo":0.5,"w":0.1133,"conf":0.5475,"acc":0.5649},{"lo":0.6,"w":0.1112,"conf":0.6492,"acc":0.6158},{"lo":0.7,"w":0.1116,"conf":0.7498,"acc":0.7567},{"lo":0.8,"w":0.1298,"conf":0.8514,"acc":0.8568},{"lo":0.9,"w":0.3591,"conf":0.969,"acc":0.9589}],"areas":{"knowledge":{"ece":0.0444,"acc":0.6762,"conf":0.702},"language":{"ece":0.0432,"acc":0.8482,"conf":0.8067},"retrieval":{"ece":0.0857,"acc":0.6521,"conf":0.6697},"tools":{"ece":0.0432,"acc":0.9367,"conf":0.9558},"arts":{"ece":0.0606,"acc":0.5878,"conf":0.5542}}},"gaps":null,"latency":{"median":135,"p95":null,"mean":227.8},"results":{"1":{"score":0.9782,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6836,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8307,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8606,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.8656,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7981,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.6364,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.2385,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7784,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7094,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9514,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6244,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.3565,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6346,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8401,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4745,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9684,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8832,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.944,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.7127,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1828,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6463,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2782,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4848,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5736,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.2328,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9402,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6133,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.6829,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8312,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.7719,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7416,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1158,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4051,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.624,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6728,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7986,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8061,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.6987,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.6208,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7027,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9782,"skill":0.9705,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6836,"skill":0.6346,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6937,"private_eq":0.7369},"3":{"raw":0.8307,"skill":0.8274,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8961,"private_eq":0.8982},"4":{"raw":0.8606,"skill":0.8588,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8541,"private_eq":0.8566},"5":{"raw":0.8656,"skill":0.8648,"coverage":1,"random":0.006,"tracks":[],"combined":0.7385,"private_eq":0.742},"6":{"raw":0.7981,"skill":0.5266,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6364,"skill":0.6364,"coverage":1,"random":0,"tracks":[],"combined":0.6293,"private_eq":0.6325},"10":{"raw":0.2385,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7784,"skill":0.6796,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7094,"skill":0.5647,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4677,"private_eq":0.648},"20":{"raw":0.9528,"skill":0.9056,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.6244,"skill":0.2489,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.3572,"skill":0.3522,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3179,"private_eq":0.3299},"23":{"raw":0.6441,"skill":0.2882,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8401,"skill":0.7868,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4745,"skill":0.2993,"coverage":1,"random":0.25,"tracks":[],"combined":0.2528,"private_eq":0.4452},"26":{"raw":0.9899,"skill":0.9865,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9684,"skill":0.9579,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8832,"skill":0.7664,"coverage":1,"random":0.5,"tracks":[],"combined":0.5774,"private_eq":0.7906},"29":{"raw":0.944,"skill":0.9253,"coverage":1,"random":0.25,"tracks":[],"combined":0.9831,"private_eq":0.9867},"30":{"raw":0.7127,"skill":0.6554,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7718,"random":0.25},{"track":"10-choice","raw":0.6535,"random":0.1}],"combined":0.7145,"private_eq":0.7663},"31":{"raw":0.1828,"skill":0.1099,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0702,"private_eq":0.1552},"32":{"raw":0.6463,"skill":0.4377,"coverage":1,"random":0.371,"tracks":[],"combined":0.328,"private_eq":0.5815},"33":{"raw":0.2782,"skill":0.2686,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1943,"private_eq":0.2129},"36":{"raw":0.4848,"skill":0.4172,"coverage":1,"random":0.116,"tracks":[],"combined":0.3835,"private_eq":0.4603},"37":{"raw":0.5736,"skill":0.4651,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5702,"private_eq":0.6604},"38":{"raw":0.2328,"skill":0.2083,"coverage":1,"random":0.031,"tracks":[],"combined":0.3703,"private_eq":0.3958},"39":{"raw":0.9402,"skill":0.912,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5111,"private_eq":0.6707},"40":{"raw":0.6133,"skill":0.5025,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2353,"headline":false},{"track":"A · English","score":0.6133,"headline":true},{"track":"C · Arabic pairs","score":0.64,"headline":false},{"track":"C · English pairs","score":0.94,"headline":false}],"combined":0.5146,"private_eq":0.6262},"41":{"raw":0.6829,"skill":0.5243,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4683,"private_eq":0.6489},"42":{"raw":0.8312,"skill":0.6716,"coverage":1,"random":0.486,"tracks":[],"combined":0.8776,"private_eq":0.9373},"43":{"raw":0.7719,"skill":0.6381,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5579,"private_eq":0.7239},"44":{"raw":0.7416,"skill":0.4832,"coverage":1,"random":0.5,"tracks":[],"combined":0.4014,"private_eq":0.7036},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4051,"skill":0.1366,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.624,"skill":0.248,"coverage":1,"random":0.5,"tracks":[],"combined":0.4125,"private_eq":0.7091},"57":{"raw":0.6728,"skill":0.632,"coverage":1,"random":0.1109,"tracks":[],"combined":0.4467,"private_eq":0.5128},"58":{"raw":0.7986,"skill":0.7081,"coverage":1,"random":0.3101,"tracks":[],"combined":0.2821,"private_eq":0.5097},"59":{"raw":0.8061,"skill":0.598,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3427,"private_eq":0.6861},"61":{"raw":0.6987,"skill":0.3974,"coverage":1,"random":0.5,"tracks":[],"combined":0.3146,"private_eq":0.6607},"62":{"raw":0.6208,"skill":0.4944,"coverage":1,"random":0.25,"tracks":[],"combined":0.6451,"private_eq":0.7361},"64":{"raw":0.7027,"skill":0.6284,"coverage":1,"random":0.2,"tracks":[],"combined":0.6228,"private_eq":0.7009}},"coverage":1,"pending":0,"scores":{"balanced_raw":67.55,"balanced_skill":59.56,"breadth_skill":55.91,"public_skill":56.69},"frozen_scores":{"balanced_raw":67.55,"balanced_skill":56.69,"breadth_skill":55.91},"categories":[{"id":"knowledge","raw":0.5723,"skill":0.4567,"combined":0.383,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7307,"skill":0.626,"combined":0.5745,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6893,"skill":0.5575,"combined":0.5558,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7614,"skill":0.7265,"combined":0.7754,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6144,"skill":0.4266,"combined":0.4505,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"kev-27b","pub":56.69,"same_skills":55.51,"new_domains":51.51,"v":59.56,"lab":false,"jev":false,"rank_v":54,"rank_pub":92,"name":"Kev 27B","eq":{"S":55.1,"O":66.26},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":33.06,"new_domains":26.5}},"notes":["Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."]},{"engine":"blink-v0.3-26b-a4b-nvfp4","name":"Blink v0.3 26B-A4B (NVFP4)","long_name":"Blink v0.3 26B-A4B (NVFP4)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/PixilabAI/Blink-v0.3-26B-A4B-NVFP4","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"PixilabAI","name":"Pixilab.ai","logo":"data/orgs/PixilabAI.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7522,"conf":0.7925,"ece":0.0404,"brier":0.3391,"over95":0.0194,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0011,"conf":0.0899,"acc":0.1066},{"lo":0.1,"w":0.0144,"conf":0.1549,"acc":0.0993},{"lo":0.2,"w":0.0225,"conf":0.261,"acc":0.1873},{"lo":0.3,"w":0.0311,"conf":0.3496,"acc":0.287},{"lo":0.4,"w":0.0484,"conf":0.4538,"acc":0.3836},{"lo":0.5,"w":0.0989,"conf":0.5499,"acc":0.5257},{"lo":0.6,"w":0.087,"conf":0.6487,"acc":0.643},{"lo":0.7,"w":0.1029,"conf":0.751,"acc":0.6875},{"lo":0.8,"w":0.1214,"conf":0.855,"acc":0.815},{"lo":0.9,"w":0.4722,"conf":0.9731,"acc":0.9344}],"areas":{"knowledge":{"ece":0.0567,"acc":0.6703,"conf":0.7269},"language":{"ece":0.0359,"acc":0.8498,"conf":0.8139},"retrieval":{"ece":0.1027,"acc":0.7631,"conf":0.8655},"tools":{"ece":0.0654,"acc":0.9209,"conf":0.9598},"arts":{"ece":0.0947,"acc":0.6465,"conf":0.7398}}},"gaps":null,"latency":{"median":25.1,"p95":null,"mean":48.4},"results":{"1":{"score":0.9599,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6613,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8228,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8353,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.8857,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.799,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.4886,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6792,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7699,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7204,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.8978,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6244,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.6585,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6282,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8308,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4541,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9878,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9693,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8485,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9153,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.754,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1778,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6636,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3412,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4715,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5516,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.282,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9063,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6807,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7712,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.7883,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.7386,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7178,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4535,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.7955,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6639,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.8141,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8025,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.7933,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.675,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7367,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9599,"skill":0.9458,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8989,"private_eq":0.9252},"2":{"raw":0.6613,"skill":0.6088,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6594,"private_eq":0.7076},"3":{"raw":0.8228,"skill":0.8194,"coverage":1,"random":0.0189,"tracks":[],"combined":0.854,"private_eq":0.8574},"4":{"raw":0.8353,"skill":0.8332,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8618,"private_eq":0.8641},"5":{"raw":0.8857,"skill":0.885,"coverage":1,"random":0.006,"tracks":[],"combined":0.7177,"private_eq":0.7216},"6":{"raw":0.799,"skill":0.5287,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4886,"skill":0.4886,"coverage":1,"random":0,"tracks":[],"combined":0.4472,"private_eq":0.4525},"10":{"raw":0.6792,"skill":0.4662,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7699,"skill":0.6673,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7204,"skill":0.5812,"coverage":1,"random":0.3324,"tracks":[],"combined":0.478,"private_eq":0.6548},"20":{"raw":0.9013,"skill":0.8026,"coverage":1,"random":0.5,"tracks":[],"combined":0.7769,"private_eq":0.8892},"21":{"raw":0.6244,"skill":0.2489,"coverage":1,"random":0.5,"tracks":[],"combined":0.317,"private_eq":0.6619},"22":{"raw":0.6538,"skill":0.6511,"coverage":1,"random":0.0078,"tracks":[],"combined":0.526,"private_eq":0.5341},"23":{"raw":0.6191,"skill":0.2382,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8308,"skill":0.7744,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4541,"skill":0.2721,"coverage":1,"random":0.25,"tracks":[],"combined":0.282,"private_eq":0.4669},"26":{"raw":0.9878,"skill":0.9837,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9693,"skill":0.9591,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8485,"skill":0.697,"coverage":1,"random":0.5,"tracks":[],"combined":0.5774,"private_eq":0.7906},"29":{"raw":0.9153,"skill":0.8871,"coverage":1,"random":0.25,"tracks":[],"combined":0.9038,"private_eq":0.9279},"30":{"raw":0.754,"skill":0.7059,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8135,"random":0.25},{"track":"10-choice","raw":0.6945,"random":0.1}],"combined":0.7818,"private_eq":0.8212},"31":{"raw":0.1778,"skill":0.1045,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0658,"private_eq":0.1512},"32":{"raw":0.6636,"skill":0.4652,"coverage":1,"random":0.371,"tracks":[],"combined":0.2582,"private_eq":0.5381},"33":{"raw":0.3412,"skill":0.3325,"coverage":1,"random":0.0131,"tracks":[],"combined":0.369,"private_eq":0.3834},"36":{"raw":0.4715,"skill":0.4021,"coverage":1,"random":0.116,"tracks":[],"combined":0.3591,"private_eq":0.439},"37":{"raw":0.5516,"skill":0.4376,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5348,"private_eq":0.6325},"38":{"raw":0.282,"skill":0.259,"coverage":1,"random":0.031,"tracks":[],"combined":0.4391,"private_eq":0.4617},"39":{"raw":0.9063,"skill":0.8622,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6071,"private_eq":0.7352},"40":{"raw":0.6807,"skill":0.5892,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.481,"headline":false},{"track":"A · English","score":0.6807,"headline":true},{"track":"C · Arabic pairs","score":0.87,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"combined":0.5817,"private_eq":0.6778},"41":{"raw":0.7712,"skill":0.6567,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4852,"private_eq":0.66},"42":{"raw":0.7883,"skill":0.5881,"coverage":1,"random":0.486,"tracks":[],"combined":0.9499,"private_eq":0.974},"43":{"raw":0.7386,"skill":0.5853,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5746,"private_eq":0.7343},"44":{"raw":0.7178,"skill":0.4356,"coverage":1,"random":0.5,"tracks":[],"combined":0.4441,"private_eq":0.7247},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4535,"skill":0.2068,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.7955,"skill":0.591,"coverage":1,"random":0.5,"tracks":[],"combined":0.6549,"private_eq":0.8289},"57":{"raw":0.6639,"skill":0.622,"coverage":1,"random":0.1109,"tracks":[],"combined":0.4811,"private_eq":0.543},"58":{"raw":0.8141,"skill":0.7305,"coverage":1,"random":0.3101,"tracks":[],"combined":0.248,"private_eq":0.4864},"59":{"raw":0.8025,"skill":0.5905,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2554,"private_eq":0.6445},"61":{"raw":0.7933,"skill":0.5866,"coverage":1,"random":0.5,"tracks":[],"combined":0.6492,"private_eq":0.8261},"62":{"raw":0.675,"skill":0.5667,"coverage":1,"random":0.25,"tracks":[],"combined":0.5356,"private_eq":0.6549},"64":{"raw":0.7367,"skill":0.6709,"coverage":1,"random":0.2,"tracks":[],"combined":0.7489,"private_eq":0.8006}},"coverage":1,"pending":0,"scores":{"balanced_raw":68.49,"balanced_skill":55.92,"breadth_skill":57.5,"public_skill":58.21},"frozen_scores":{"balanced_raw":68.49,"balanced_skill":58.21,"breadth_skill":57.5},"categories":[{"id":"knowledge","raw":0.5726,"skill":0.4512,"combined":0.4074,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7408,"skill":0.6356,"combined":0.5925,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7229,"skill":0.6302,"combined":0.6311,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7341,"skill":0.7004,"combined":0.6937,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6648,"skill":0.4698,"combined":0.5231,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"blink-v0.3-26b-a4b-nvfp4","pub":58.21,"same_skills":57.21,"new_domains":42.05,"v":55.92,"lab":false,"jev":false,"rank_v":73,"rank_pub":74,"name":"Blink v0.3 26B-A4B (NVFP4)","eq":{"S":56.81,"O":54.57},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":34.09,"new_domains":21.83}},"notes":["Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."]},{"engine":"simple-jev-qwen3.8-27b","name":"simple-jev · Qwen3.8-27B (featherless)","long_name":"simple-jev · Qwen3.8-27B (featherless)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/featherless-ai/simple-jev","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B"},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.72,"conf":0.8322,"ece":0.113,"brier":0.4001,"over95":0.0385,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0045,"conf":0.0736,"acc":0.0638},{"lo":0.1,"w":0.0112,"conf":0.1491,"acc":0.1851},{"lo":0.2,"w":0.0127,"conf":0.2513,"acc":0.2158},{"lo":0.3,"w":0.0275,"conf":0.3543,"acc":0.3127},{"lo":0.4,"w":0.0461,"conf":0.4542,"acc":0.3266},{"lo":0.5,"w":0.0789,"conf":0.5525,"acc":0.446},{"lo":0.6,"w":0.0686,"conf":0.65,"acc":0.4865},{"lo":0.7,"w":0.0789,"conf":0.7502,"acc":0.5498},{"lo":0.8,"w":0.0913,"conf":0.8558,"acc":0.6403},{"lo":0.9,"w":0.5804,"conf":0.9836,"acc":0.8975}],"areas":{"knowledge":{"ece":0.1302,"acc":0.6437,"conf":0.7739},"language":{"ece":0.066,"acc":0.8335,"conf":0.8995},"retrieval":{"ece":0.1885,"acc":0.6781,"conf":0.8652},"tools":{"ece":0.0431,"acc":0.9373,"conf":0.9803},"arts":{"ece":0.1477,"acc":0.5866,"conf":0.7241}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":373,"p95":2796.8,"mean":930.6},"results":{"1":{"score":0.9681,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":494},"2":{"score":0.6798,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":21436.8},"3":{"score":0.8543,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7946,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1165.1},"5":{"score":0.8808,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":2429.1},"6":{"score":0.7978,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":992.3},"9":{"score":0.6477,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4189,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":608.5},"11":{"score":0.7753,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6923,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":347.3},"20":{"score":0.9612,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":320.7},"21":{"score":0.613,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":282.4},"22":{"score":0.2895,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":4623.1},"23":{"score":0.6362,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":409.8},"24":{"score":0.8401,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":313.4},"25":{"score":0.4847,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9924,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":308.2},"27":{"score":0.9684,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":310.1},"28":{"score":0.8445,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":280.1},"29":{"score":0.931,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":367.5},"30":{"score":0.362,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1736,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":698.6},"32":{"score":0.6316,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3206,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":637.4},"36":{"score":0.4974,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5735,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":351},"38":{"score":0.3558,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":6283.9},"39":{"score":0.8848,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6035,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":282.2,"tracks":[{"track":"A · Arabic","score":0.3493,"headline":false},{"track":"A · English","score":0.6035,"headline":true},{"track":"C · Arabic pairs","score":0.805,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.7185,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":349.6},"42":{"score":0.8374,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":392.5},"43":{"score":0.6228,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7102,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":334.6},"45":{"score":0.1238,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4105,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":537.1},"56":{"score":0.8465,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3020.8,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.5975,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":360.5,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.708,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":311.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7233,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":392.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.7462,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":359.2,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7503,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":485.6,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":66.93,"balanced_skill":56.99,"breadth_skill":54.4,"public_skill":56.05},"frozen_scores":{"balanced_raw":66.93,"balanced_skill":56.05,"breadth_skill":54.4},"categories":[{"id":"knowledge","raw":0.5109,"skill":0.3692,"combined":0.3392,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7284,"skill":0.6122,"combined":0.5613,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7233,"skill":0.6325,"combined":0.6351,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7898,"skill":0.7621,"combined":0.7575,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5983,"skill":0.4082,"combined":0.4533,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9681,"skill":0.957,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6798,"skill":0.6303,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7093,"private_eq":0.7503},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8804,"private_eq":0.883},"4":{"raw":0.7946,"skill":0.792,"coverage":1,"random":0.0127,"tracks":[],"combined":0.881,"private_eq":0.8829},"5":{"raw":0.8808,"skill":0.8801,"coverage":1,"random":0.006,"tracks":[],"combined":0.7329,"private_eq":0.7365},"6":{"raw":0.7979,"skill":0.5234,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7843,"headline":false},{"track":"RouterBench-5shot","score":0.8114,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.6477,"skill":0.6477,"coverage":1,"random":0,"tracks":[],"combined":0.5559,"private_eq":0.56},"11":{"raw":0.7753,"skill":0.675,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6923,"skill":0.5391,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4975,"private_eq":0.6677},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.9627,"skill":0.9253,"coverage":1,"random":0.5,"tracks":[],"combined":0.9361,"private_eq":0.9679},"21":{"raw":0.613,"skill":0.226,"coverage":1,"random":0.5,"tracks":[],"combined":0.1962,"private_eq":0.6022},"22":{"raw":0.2826,"skill":0.277,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2209,"private_eq":0.2348},"23":{"raw":0.6334,"skill":0.2668,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8425,"skill":0.7899,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4847,"skill":0.3129,"coverage":1,"random":0.25,"tracks":[],"combined":0.2386,"private_eq":0.4347},"28":{"raw":0.8445,"skill":0.689,"coverage":1,"random":0.5,"tracks":[],"combined":0.542,"private_eq":0.7731},"29":{"raw":0.931,"skill":0.908,"coverage":1,"random":0.25,"tracks":[],"combined":0.9731,"private_eq":0.9793},"30":{"raw":0.362,"skill":0.2311,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4594,"random":0.25},{"track":"10-choice","raw":0.2646,"random":0.1}],"combined":0.4335,"private_eq":0.5371},"31":{"raw":0.1736,"skill":0.0999,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0918,"private_eq":0.1748},"32":{"raw":0.6316,"skill":0.4143,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.3206,"skill":0.3116,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2457,"private_eq":0.2631},"36":{"raw":0.4974,"skill":0.4314,"coverage":1,"random":0.116,"tracks":[],"combined":0.3528,"private_eq":0.4335},"37":{"raw":0.5735,"skill":0.4651,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5232,"private_eq":0.6234},"38":{"raw":0.3558,"skill":0.3352,"coverage":1,"random":0.031,"tracks":[],"combined":0.5241,"private_eq":0.5431},"39":{"raw":0.8848,"skill":0.8306,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6132,"private_eq":0.7393},"40":{"raw":0.6035,"skill":0.4899,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3493,"headline":false},{"track":"A · English","score":0.6035,"headline":true},{"track":"C · Arabic pairs","score":0.805,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.3949,"private_eq":0.5342},"41":{"raw":0.7185,"skill":0.5778,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5716,"private_eq":0.717},"42":{"raw":0.8374,"skill":0.6837,"coverage":1,"random":0.486,"tracks":[],"combined":0.585,"private_eq":0.7886},"43":{"raw":0.6228,"skill":0.4015,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4366,"private_eq":0.6483},"44":{"raw":0.7102,"skill":0.4204,"coverage":1,"random":0.5,"tracks":[],"combined":0.3799,"private_eq":0.693},"45":{"raw":0.1238,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4105,"skill":0.1444,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.8465,"skill":0.693,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.7328,"private_eq":0.8674},"57":{"raw":0.5975,"skill":0.5473,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.4277,"private_eq":0.4961},"58":{"raw":0.708,"skill":0.5768,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.2036,"private_eq":0.4561},"59":{"raw":0.7233,"skill":0.4263,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.2617,"private_eq":0.6475},"61":{"raw":0.7462,"skill":0.4924,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5756,"private_eq":0.7897},"62":{"raw":0.7503,"skill":0.6671,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.6246,"private_eq":0.7209},"64":{"raw":0.6875,"skill":0.6094,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.6519,"private_eq":0.7239}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author hf-server/hf_server.py (bf16, fla kernels, examples_binary policy, max-model-len 65536, batch tokens 65536), one sequential client (JEV_ATTACH); 203 of 760 rows ran out of GPU memory at the server default batch and were answered in-process by the same hf_server service with max_batch_size 32 (simplejev_ladder.py, as on the board for heavy rows); causal-conv1d absent (torch conv fallback, as on the board)","v03":{"engine":"simple-jev-qwen3.8-27b","pub":56.05,"same_skills":55.46,"new_domains":46.36,"v":56.99,"lab":false,"jev":false,"rank_v":68,"rank_pub":96,"name":"simple-jev · Qwen3.8-27B (featherless)","eq":{"S":55.05,"O":59.9},"rank_tie":14,"tied":true,"parts":{"public":0,"same_skills":33.03,"new_domains":23.96}}},{"engine":"jev-27b","name":"JEV-27B","long_name":"JEV-27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/autotrust/JEV-27B","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7118,"conf":0.7877,"ece":0.0759,"brier":0.3826,"over95":0.0169,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0002,"conf":0.0891,"acc":0.1528},{"lo":0.1,"w":0.0173,"conf":0.1634,"acc":0.1473},{"lo":0.2,"w":0.0242,"conf":0.2554,"acc":0.1487},{"lo":0.3,"w":0.0372,"conf":0.353,"acc":0.2341},{"lo":0.4,"w":0.0503,"conf":0.4516,"acc":0.3132},{"lo":0.5,"w":0.0989,"conf":0.5479,"acc":0.4845},{"lo":0.6,"w":0.0933,"conf":0.6476,"acc":0.551},{"lo":0.7,"w":0.0958,"conf":0.7522,"acc":0.6469},{"lo":0.8,"w":0.115,"conf":0.8516,"acc":0.7337},{"lo":0.9,"w":0.4677,"conf":0.9796,"acc":0.9311}],"areas":{"knowledge":{"ece":0.0712,"acc":0.6524,"conf":0.7234},"language":{"ece":0.0595,"acc":0.7897,"conf":0.8492},"retrieval":{"ece":0.1238,"acc":0.6809,"conf":0.8046},"tools":{"ece":0.0423,"acc":0.9359,"conf":0.9753},"arts":{"ece":0.1,"acc":0.5926,"conf":0.6921}}},"gaps":null,"latency":{"median":102.1,"p95":null,"mean":584.6},"results":{"1":{"score":0.9599,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":263.3},"2":{"score":0.6573,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2279.4},"3":{"score":0.8287,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2465.6},"4":{"score":0.742,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":99.7},"5":{"score":0.8462,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":176.3},"6":{"score":0.7986,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":311},"9":{"score":0.5227,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1334.7},"10":{"score":0.6122,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":96},"11":{"score":0.7504,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3031.3},"12":{"score":0.772,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":13.6},"20":{"score":0.9262,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.6},"21":{"score":0.6115,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":6.3},"22":{"score":0.403,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1842.1},"23":{"score":0.63,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.3},"24":{"score":0.8347,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13},"25":{"score":0.449,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":51.6},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":10},"27":{"score":0.9667,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":11},"28":{"score":0.8256,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":5.4},"29":{"score":0.9249,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":21.3},"30":{"score":0.616,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.165,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":131.4},"32":{"score":0.6237,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":83},"33":{"score":0.3364,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":267.4},"36":{"score":0.4844,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2240.4},"37":{"score":0.5456,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":65.5},"38":{"score":0.1948,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":624.3},"39":{"score":0.8335,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":30.3},"40":{"score":0.4198,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":7.5},"41":{"score":0.5476,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":18.9},"42":{"score":0.8427,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":78},"43":{"score":0.7368,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":23.9},"44":{"score":0.705,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.1},"45":{"score":0.0699,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":78.7},"50":{"score":0.4039,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":75.7},"56":{"score":0.619,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":173.3},"57":{"score":0.6153,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":35.9},"58":{"score":0.7894,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":17.6},"59":{"score":0.8088,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":79.5},"61":{"score":0.739,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":48.8},"62":{"score":0.7853,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":88.6},"64":{"score":0.6856,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":17.1}},"benchmarks":{"1":{"raw":0.9599,"skill":0.9458,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8717,"private_eq":0.9053},"2":{"raw":0.6573,"skill":0.6042,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6884,"private_eq":0.7324},"3":{"raw":0.8287,"skill":0.8254,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9066,"private_eq":0.9084},"4":{"raw":0.742,"skill":0.7387,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8019,"private_eq":0.8056},"5":{"raw":0.8462,"skill":0.8453,"coverage":1,"random":0.006,"tracks":[],"combined":0.7509,"private_eq":0.7542},"6":{"raw":0.7986,"skill":0.5278,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5227,"skill":0.5227,"coverage":1,"random":0,"tracks":[],"combined":0.5914,"private_eq":0.595},"10":{"raw":0.6122,"skill":0.3547,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7504,"skill":0.639,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.772,"skill":0.6585,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5181,"private_eq":0.6813},"20":{"raw":0.9294,"skill":0.8589,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.6115,"skill":0.223,"coverage":1,"random":0.5,"tracks":[],"combined":0.156,"private_eq":0.5823},"22":{"raw":0.3945,"skill":0.3898,"coverage":1,"random":0.0078,"tracks":[],"combined":0.44,"private_eq":0.4497},"23":{"raw":0.6414,"skill":0.2827,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8347,"skill":0.7796,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.449,"skill":0.2653,"coverage":1,"random":0.25,"tracks":[],"combined":0.2443,"private_eq":0.4389},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9667,"skill":0.9556,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8256,"skill":0.6512,"coverage":1,"random":0.5,"tracks":[],"combined":0.6126,"private_eq":0.808},"29":{"raw":0.9249,"skill":0.8999,"coverage":1,"random":0.25,"tracks":[],"combined":0.9038,"private_eq":0.9279},"30":{"raw":0.616,"skill":0.5422,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7202,"random":0.25},{"track":"10-choice","raw":0.5118,"random":0.1}],"combined":0.6347,"private_eq":0.7012},"31":{"raw":0.165,"skill":0.0906,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0886,"private_eq":0.1719},"32":{"raw":0.6237,"skill":0.4017,"coverage":1,"random":0.371,"tracks":[],"combined":0.2815,"private_eq":0.5526},"33":{"raw":0.3364,"skill":0.3276,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3742,"private_eq":0.3884},"36":{"raw":0.4844,"skill":0.4167,"coverage":1,"random":0.116,"tracks":[],"combined":0.378,"private_eq":0.4555},"37":{"raw":0.5456,"skill":0.4301,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5366,"private_eq":0.6339},"38":{"raw":0.1948,"skill":0.169,"coverage":1,"random":0.031,"tracks":[],"combined":0.3219,"private_eq":0.3494},"39":{"raw":0.8335,"skill":0.7551,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5923,"private_eq":0.7253},"40":{"raw":0.4198,"skill":0.2535,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4572,"headline":false},{"track":"A · English","score":0.4198,"headline":true},{"track":"C · Arabic pairs","score":0.73,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.2531,"private_eq":0.4253},"41":{"raw":0.5476,"skill":0.3215,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3459,"private_eq":0.5682},"42":{"raw":0.8427,"skill":0.694,"coverage":1,"random":0.486,"tracks":[],"combined":0.7267,"private_eq":0.8606},"43":{"raw":0.7368,"skill":0.5825,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5663,"private_eq":0.7291},"44":{"raw":0.705,"skill":0.41,"coverage":1,"random":0.5,"tracks":[],"combined":0.4121,"private_eq":0.7089},"45":{"raw":0.0699,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4039,"skill":0.1349,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.619,"skill":0.238,"coverage":1,"random":0.5,"tracks":[],"combined":0.5165,"private_eq":0.7605},"57":{"raw":0.6153,"skill":0.5673,"coverage":1,"random":0.1109,"tracks":[],"combined":0.4644,"private_eq":0.5283},"58":{"raw":0.7894,"skill":0.6947,"coverage":1,"random":0.3101,"tracks":[],"combined":0.228,"private_eq":0.4728},"59":{"raw":0.8088,"skill":0.6036,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3509,"private_eq":0.69},"61":{"raw":0.739,"skill":0.478,"coverage":1,"random":0.5,"tracks":[],"combined":0.4904,"private_eq":0.7476},"62":{"raw":0.7853,"skill":0.7137,"coverage":1,"random":0.25,"tracks":[],"combined":0.5767,"private_eq":0.6854},"64":{"raw":0.6856,"skill":0.607,"coverage":1,"random":0.2,"tracks":[],"combined":0.6326,"private_eq":0.7086}},"coverage":1,"pending":0,"scores":{"balanced_raw":65,"balanced_skill":57.35,"breadth_skill":52.7,"public_skill":53.64},"frozen_scores":{"balanced_raw":65,"balanced_skill":53.64,"breadth_skill":52.7},"categories":[{"id":"knowledge","raw":0.5402,"skill":0.4141,"combined":0.3904,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6845,"skill":0.5644,"combined":0.5116,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6653,"skill":0.5374,"combined":0.5849,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7614,"skill":0.7345,"combined":0.7389,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.611,"skill":0.416,"combined":0.4692,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-27b","pub":53.64,"same_skills":54.22,"new_domains":48.61,"v":57.35,"lab":false,"jev":false,"rank_v":67,"rank_pub":103,"name":"JEV-27B","eq":{"S":53.79,"O":62.67},"rank_tie":14,"tied":true,"parts":{"public":0,"same_skills":32.28,"new_domains":25.07}}},{"engine":"eikos-27b-fp8","name":"Eikos-27B","long_name":"Eikos-27B-FP8 Qwen3.8-27B LoRA","variant":"FP8","meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":31220328864,"model_url":"https://huggingface.co/caiovicentino1/Eikos-27B-FP8","code_url":"https://github.com/caiovicentino/eikos","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"caiovicentino1/Eikos-27B-FP8","served_checkpoint":null},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.7053,"conf":0.7428,"ece":0.0574,"brier":0.4009,"over95":0.0005,"n":35093,"benchmarks":32,"rel":[{"lo":0,"w":0.002,"conf":0.0848,"acc":0.0195},{"lo":0.1,"w":0.0225,"conf":0.1546,"acc":0.1296},{"lo":0.2,"w":0.0215,"conf":0.2515,"acc":0.1859},{"lo":0.3,"w":0.035,"conf":0.3548,"acc":0.2158},{"lo":0.4,"w":0.0572,"conf":0.4538,"acc":0.363},{"lo":0.5,"w":0.095,"conf":0.5503,"acc":0.4945},{"lo":0.6,"w":0.1088,"conf":0.6514,"acc":0.6061},{"lo":0.7,"w":0.1394,"conf":0.7541,"acc":0.6852},{"lo":0.8,"w":0.2426,"conf":0.8541,"acc":0.7904},{"lo":0.9,"w":0.2762,"conf":0.9411,"acc":0.9773}],"areas":{"knowledge":{"ece":0.0824,"acc":0.6532,"conf":0.7185},"language":{"ece":0.0796,"acc":0.7434,"conf":0.794},"retrieval":{"ece":0.0263,"acc":0.7369,"conf":0.7158},"tools":{"ece":0.0521,"acc":0.936,"conf":0.9},"arts":{"ece":0.0548,"acc":0.5878,"conf":0.6311}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":129.7,"p95":509.8,"mean":213.2},"results":{"1":{"score":0.9628,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":229.9},"2":{"score":0.6675,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1149.7},"3":{"score":0.8602,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7902,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":229.9},"5":{"score":0.8769,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":339.9},"6":{"score":0.7971,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":479.8},"9":{"score":0.5114,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6686,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":200},"11":{"score":0.804,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7559,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":119.8},"20":{"score":0.9374,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.9},"21":{"score":0.61,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":109.9},"22":{"score":0.2865,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":764.7},"23":{"score":0.6518,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":139.8},"24":{"score":0.8416,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":119.9},"25":{"score":0.5051,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9907,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":119.8},"27":{"score":0.9693,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":119.9},"28":{"score":0.8406,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":109.9},"29":{"score":0.9296,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":119.9},"30":{"score":0.4314,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1716,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":209.8},"32":{"score":0.609,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.283,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":359.7},"36":{"score":0.4832,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5545,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.9},"38":{"score":0.224,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1200},"39":{"score":0,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null,"wrong_rows":979,"contamination":"The whole FinEntity benchmark counts as wrong: its full test set is in the declared training data"},"40":{"score":0.5744,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":109.9,"tracks":[{"track":"A · Arabic","score":0.4169,"headline":false},{"track":"A · English","score":0.5744,"headline":true},{"track":"C · Arabic pairs","score":0.755,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.7134,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":119.9},"42":{"score":0.8409,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":139.5},"43":{"score":0.6807,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7512,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.8},"45":{"score":0.0978,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4362,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":199.6},"56":{"score":0.6385,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":319.8,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.6095,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":119.9,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.7894,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":119.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.741,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":164.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.737,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":129.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7924,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":159.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.7102,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":63.73,"balanced_skill":58.79,"breadth_skill":51.8,"public_skill":52.85},"frozen_scores":{"balanced_raw":63.73,"balanced_skill":52.85,"breadth_skill":51.8},"categories":[{"id":"knowledge","raw":0.5281,"skill":0.3998,"combined":0.3757,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6298,"skill":0.5256,"combined":0.5514,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6834,"skill":0.5587,"combined":0.5932,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7702,"skill":0.7443,"combined":0.785,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.604,"skill":0.4142,"combined":0.4711,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9628,"skill":0.9498,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6675,"skill":0.6161,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6925,"private_eq":0.7359},"3":{"raw":0.8602,"skill":0.8575,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9066,"private_eq":0.9084},"4":{"raw":0.7902,"skill":0.7875,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8753,"private_eq":0.8773},"5":{"raw":0.8769,"skill":0.8762,"coverage":1,"random":0.006,"tracks":[],"combined":0.6936,"private_eq":0.6979},"6":{"raw":0.7971,"skill":0.5211,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.785,"headline":false},{"track":"RouterBench-5shot","score":0.8092,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.5114,"skill":0.5114,"coverage":1,"random":0,"tracks":[],"combined":0.6242,"private_eq":0.6275},"11":{"raw":0.804,"skill":0.7165,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7559,"skill":0.6344,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5437,"private_eq":0.6982},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.94,"skill":0.8799,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.61,"skill":0.2199,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.2845,"skill":0.2789,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3146,"private_eq":0.3267},"23":{"raw":0.6434,"skill":0.2869,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8473,"skill":0.7965,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"combined":0.2765,"private_eq":0.4628},"28":{"raw":0.8406,"skill":0.6812,"coverage":1,"random":0.5,"tracks":[],"combined":0.5673,"private_eq":0.7856},"29":{"raw":0.9296,"skill":0.9061,"coverage":1,"random":0.25,"tracks":[],"combined":0.9584,"private_eq":0.9684},"30":{"raw":0.4314,"skill":0.3199,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5648,"random":0.25},{"track":"10-choice","raw":0.298,"random":0.1}],"combined":0.5094,"private_eq":0.599},"31":{"raw":0.1716,"skill":0.0977,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0745,"private_eq":0.1591},"32":{"raw":0.609,"skill":0.3784,"coverage":1,"random":0.371,"tracks":[],"combined":0.3201,"private_eq":0.5766},"33":{"raw":0.283,"skill":0.2735,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2251,"private_eq":0.243},"36":{"raw":0.4832,"skill":0.4154,"coverage":1,"random":0.116,"tracks":[],"combined":0.3876,"private_eq":0.4639},"37":{"raw":0.5545,"skill":0.4412,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5666,"private_eq":0.6576},"38":{"raw":0.224,"skill":0.1992,"coverage":1,"random":0.031,"tracks":[],"combined":0.3875,"private_eq":0.4123},"39":{"raw":0,"skill":0,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4321,"private_eq":0.6176},"40":{"raw":0.5744,"skill":0.4525,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4169,"headline":false},{"track":"A · English","score":0.5744,"headline":true},{"track":"C · Arabic pairs","score":0.755,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.5026,"private_eq":0.617},"41":{"raw":0.7134,"skill":0.5701,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3796,"private_eq":0.5904},"42":{"raw":0.8409,"skill":0.6905,"coverage":1,"random":0.486,"tracks":[],"combined":0.8873,"private_eq":0.9422},"43":{"raw":0.6807,"skill":0.4934,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5452,"private_eq":0.716},"44":{"raw":0.7512,"skill":0.5024,"coverage":1,"random":0.5,"tracks":[],"combined":0.5031,"private_eq":0.7539},"45":{"raw":0.0978,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4362,"skill":0.1817,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.6385,"skill":0.277,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4991,"private_eq":0.7519},"57":{"raw":0.6095,"skill":0.5608,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.5077,"private_eq":0.5664},"58":{"raw":0.7894,"skill":0.6947,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.2154,"private_eq":0.4642},"59":{"raw":0.741,"skill":0.463,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.2403,"private_eq":0.6373},"61":{"raw":0.737,"skill":0.474,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5017,"private_eq":0.7532},"62":{"raw":0.7924,"skill":0.7232,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.7067,"private_eq":0.7818},"64":{"raw":0.7102,"skill":0.6377,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.6422,"private_eq":0.7162}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"model repo serve.py 1.2 (caiovicentino1/Eikos-27B-FP8@e40a92f, letter readout, calib.json T=1, max_one_pass 160) over vLLM 0.30.0 with the submission's launch flags (serve_vllm.sh flags, --max-model-len 65536, --enable-prefix-caching --mamba-cache-mode align, --max-num-seqs 64, gpu util 0.85); one sequential client; one RTX PRO 6000 Space; venv torch 2.13.0+cu130, transformers 5.17.0 (fleet/eikos-27b-fp8-freeze.txt)","contamination":[{"catalog_id":39,"benchmark":"FinEntity","wrong":979,"flag":"The whole FinEntity benchmark counts as wrong: its full test set is in the declared training data"}],"v03":{"engine":"eikos-27b-fp8","pub":52.85,"same_skills":55.92,"new_domains":49.43,"v":58.79,"lab":false,"jev":false,"rank_v":60,"rank_pub":107,"name":"Eikos-27B-FP8 Qwen3.8-27B LoRA","eq":{"S":55.52,"O":63.69},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":33.31,"new_domains":25.48}}},{"engine":"jade","name":"JADE","long_name":"JADE","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/theunnecessarythings/JADE","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110179,"unsupported":22,"error":0},"lifted":false,"calibration":{"acc":0.7136,"conf":0.9515,"ece":0.2379,"brier":0.5098,"over95":0.1729,"n":34834,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0,"conf":null,"acc":null},{"lo":0.2,"w":0.0003,"conf":0.2723,"acc":0.0478},{"lo":0.3,"w":0.0027,"conf":0.3647,"acc":0.1588},{"lo":0.4,"w":0.0127,"conf":0.4657,"acc":0.1183},{"lo":0.5,"w":0.0189,"conf":0.5393,"acc":0.2908},{"lo":0.6,"w":0.0281,"conf":0.6296,"acc":0.3088},{"lo":0.7,"w":0.0417,"conf":0.7463,"acc":0.3704},{"lo":0.8,"w":0.0475,"conf":0.8664,"acc":0.4162},{"lo":0.9,"w":0.8482,"conf":0.9955,"acc":0.7808}],"areas":{"knowledge":{"ece":0.2712,"acc":0.6624,"conf":0.9335},"language":{"ece":0.1655,"acc":0.8151,"conf":0.9806},"retrieval":{"ece":0.3189,"acc":0.6461,"conf":0.9617},"tools":{"ece":0.0597,"acc":0.9361,"conf":0.9896},"arts":{"ece":0.3432,"acc":0.5644,"conf":0.9077}}},"gaps":null,"latency":{"median":133.3,"p95":null,"mean":277.7},"results":{"1":{"score":0.9699,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":294.1},"2":{"score":0.6462,"jev":null,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":2358.7},"3":{"score":0.834,"jev":null,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":1494.3},"4":{"score":0.7854,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":264.2},"5":{"score":0.8759,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":398.3},"6":{"score":0.7985,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":629.4},"9":{"score":0.5455,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1221.3},"10":{"score":0.3485,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":286.6},"11":{"score":0.7464,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1920.9},"12":{"score":0.6421,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":74.5},"20":{"score":0.9174,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.5},"21":{"score":0.6202,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":68.3},"22":{"score":0.198,"jev":null,"metric":"accuracy","requests":2000,"answered":1975,"unsupported":25,"errors":0,"median_ms":1349.7},"23":{"score":0.629,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":139.1},"24":{"score":0.8353,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":72.2},"25":{"score":0.4592,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":76.8},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":68.6},"27":{"score":0.9659,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":71.4},"28":{"score":0.8335,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":68.4},"29":{"score":0.9451,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":94.7},"30":{"score":0.6513,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1652,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":260.8},"32":{"score":0.629,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":239.3},"33":{"score":0.2309,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":622.5},"36":{"score":0.4778,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3563.9},"37":{"score":0.5581,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111},"38":{"score":0.2684,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2339},"39":{"score":0.8794,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":150.5},"40":{"score":0.6085,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":69.7},"41":{"score":0.6257,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":76.6},"42":{"score":0.8389,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":116.3},"43":{"score":0.693,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":75.9},"44":{"score":0.7226,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":76.9},"45":{"score":0.096,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":96.5},"50":{"score":0.4493,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":258.7},"56":{"score":0.5645,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":514.8},"57":{"score":0.6419,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":79.1},"58":{"score":0.7904,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":75.5},"59":{"score":0.67,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":165.2},"61":{"score":0.7675,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":118.1},"62":{"score":0.7544,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":146.4},"64":{"score":0.7027,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":76.6}},"benchmarks":{"1":{"raw":0.9699,"skill":0.9594,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8921,"private_eq":0.9202},"2":{"raw":0.6443,"skill":0.5893,"coverage":0.9971,"random":0.1341,"tracks":[],"combined":0.6861,"private_eq":0.7304},"3":{"raw":0.8307,"skill":0.8275,"coverage":0.9961,"random":0.0189,"tracks":[],"combined":0.8855,"private_eq":0.888},"4":{"raw":0.7854,"skill":0.7826,"coverage":1,"random":0.0127,"tracks":[],"combined":0.872,"private_eq":0.8741},"5":{"raw":0.8759,"skill":0.8752,"coverage":1,"random":0.006,"tracks":[],"combined":0.7131,"private_eq":0.7171},"6":{"raw":0.7985,"skill":0.5275,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5455,"skill":0.5455,"coverage":1,"random":0,"tracks":[],"combined":0.5787,"private_eq":0.5825},"10":{"raw":0.3485,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7464,"skill":0.6332,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6421,"skill":0.4639,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5416,"private_eq":0.6968},"20":{"raw":0.9198,"skill":0.8397,"coverage":1,"random":0.5,"tracks":[],"combined":0.907,"private_eq":0.9535},"21":{"raw":0.6202,"skill":0.2405,"coverage":1,"random":0.5,"tracks":[],"combined":0.2365,"private_eq":0.6221},"22":{"raw":0.1952,"skill":0.1889,"coverage":0.9875,"random":0.0078,"tracks":[],"combined":0.2506,"private_eq":0.2639},"23":{"raw":0.62,"skill":0.2399,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8353,"skill":0.7804,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4592,"skill":0.2789,"coverage":1,"random":0.25,"tracks":[],"combined":0.2513,"private_eq":0.4441},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9659,"skill":0.9545,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8335,"skill":0.667,"coverage":1,"random":0.5,"tracks":[],"combined":0.547,"private_eq":0.7756},"29":{"raw":0.9451,"skill":0.9268,"coverage":1,"random":0.25,"tracks":[],"combined":0.9781,"private_eq":0.983},"30":{"raw":0.6513,"skill":0.5824,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7293,"random":0.25},{"track":"10-choice","raw":0.5732,"random":0.1}],"combined":0.6532,"private_eq":0.7163},"31":{"raw":0.1652,"skill":0.0908,"coverage":1,"random":0.0819,"tracks":[],"combined":0.079,"private_eq":0.1631},"32":{"raw":0.629,"skill":0.4102,"coverage":1,"random":0.371,"tracks":[],"combined":0.3201,"private_eq":0.5766},"33":{"raw":0.2309,"skill":0.2207,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1018,"private_eq":0.1227},"36":{"raw":0.4778,"skill":0.4093,"coverage":1,"random":0.116,"tracks":[],"combined":0.3772,"private_eq":0.4548},"37":{"raw":0.5581,"skill":0.4458,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5103,"private_eq":0.6132},"38":{"raw":0.2684,"skill":0.245,"coverage":1,"random":0.031,"tracks":[],"combined":0.3115,"private_eq":0.3395},"39":{"raw":0.8794,"skill":0.8226,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6077,"private_eq":0.7356},"40":{"raw":0.6085,"skill":0.4963,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3648,"headline":false},{"track":"A · English","score":0.6085,"headline":true},{"track":"C · Arabic pairs","score":0.655,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"combined":0.4041,"private_eq":0.5413},"41":{"raw":0.6257,"skill":0.4386,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4715,"private_eq":0.651},"42":{"raw":0.8389,"skill":0.6866,"coverage":1,"random":0.486,"tracks":[],"combined":0.963,"private_eq":0.9807},"43":{"raw":0.693,"skill":0.5129,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5119,"private_eq":0.6952},"44":{"raw":0.7226,"skill":0.4452,"coverage":1,"random":0.5,"tracks":[],"combined":0.4762,"private_eq":0.7406},"45":{"raw":0.0958,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.4493,"skill":0.2007,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.5645,"skill":0.129,"coverage":1,"random":0.5,"tracks":[],"combined":0.3694,"private_eq":0.6878},"57":{"raw":0.6419,"skill":0.5972,"coverage":1,"random":0.1109,"tracks":[],"combined":0.5333,"private_eq":0.5889},"58":{"raw":0.7904,"skill":0.6962,"coverage":1,"random":0.3101,"tracks":[],"combined":0.2069,"private_eq":0.4584},"59":{"raw":0.67,"skill":0.3158,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1682,"private_eq":0.6029},"61":{"raw":0.7675,"skill":0.535,"coverage":1,"random":0.5,"tracks":[],"combined":0.4621,"private_eq":0.7336},"62":{"raw":0.7544,"skill":0.6725,"coverage":1,"random":0.25,"tracks":[],"combined":0.5767,"private_eq":0.6854},"64":{"raw":0.7027,"skill":0.6284,"coverage":1,"random":0.2,"tracks":[],"combined":0.6713,"private_eq":0.7392}},"coverage":0.9993,"pending":0.0007,"scores":{"balanced_raw":65.2,"balanced_skill":55.57,"breadth_skill":52.42,"public_skill":53.42},"frozen_scores":{"balanced_raw":65.2,"balanced_skill":53.42,"breadth_skill":52.42},"categories":[{"id":"knowledge","raw":0.5397,"skill":0.4124,"combined":0.3693,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.696,"skill":0.5646,"combined":0.5655,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6753,"skill":0.544,"combined":0.5601,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7602,"skill":0.7318,"combined":0.736,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.5845,"skill":0.3897,"combined":0.4523,"coverage":0.9979,"pending":0,"n":6}],"v03":{"engine":"jade","pub":53.42,"same_skills":54.29,"new_domains":44.92,"v":55.57,"lab":false,"jev":false,"rank_v":74,"rank_pub":105,"name":"JADE","eq":{"S":53.86,"O":58.12},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":32.32,"new_domains":23.25}}},{"engine":"jev-gemma4-26b-a4b","name":"GEV-26B-Decide","long_name":"GEV-26B-Decide","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":25800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/autotrust/GEV-26B-Decide","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7374,"conf":0.8095,"ece":0.0721,"brier":0.3527,"over95":0.0118,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0074,"conf":0.1678,"acc":0.102},{"lo":0.2,"w":0.0206,"conf":0.2591,"acc":0.1851},{"lo":0.3,"w":0.0357,"conf":0.3556,"acc":0.3141},{"lo":0.4,"w":0.0413,"conf":0.4534,"acc":0.3189},{"lo":0.5,"w":0.0962,"conf":0.5484,"acc":0.4348},{"lo":0.6,"w":0.0863,"conf":0.6523,"acc":0.5295},{"lo":0.7,"w":0.0893,"conf":0.7502,"acc":0.6184},{"lo":0.8,"w":0.1143,"conf":0.8533,"acc":0.7236},{"lo":0.9,"w":0.5089,"conf":0.9783,"acc":0.949}],"areas":{"knowledge":{"ece":0.0977,"acc":0.6848,"conf":0.7825},"language":{"ece":0.0285,"acc":0.8367,"conf":0.8637},"retrieval":{"ece":0.1346,"acc":0.6992,"conf":0.8334},"tools":{"ece":0.0365,"acc":0.9211,"conf":0.9425},"arts":{"ece":0.0776,"acc":0.5946,"conf":0.6721}}},"gaps":null,"latency":{"median":128.5,"p95":null,"mean":228.3},"results":{"1":{"score":0.9463,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":138.8},"2":{"score":0.6707,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1291.5},"3":{"score":0.8425,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1822.7},"4":{"score":0.8799,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.5},"5":{"score":0.9298,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":67},"6":{"score":0.7994,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":168.4},"9":{"score":0.2159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":569.6},"10":{"score":0.3202,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":48.4},"11":{"score":0.8306,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1297.6},"12":{"score":0.7034,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":6},"20":{"score":0.9026,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.8},"21":{"score":0.6305,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":3},"22":{"score":0.1965,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1044},"23":{"score":0.6316,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.9},"24":{"score":0.8352,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":5.9},"25":{"score":0.4388,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":22},"26":{"score":0.9861,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":4.5},"27":{"score":0.9659,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":5},"28":{"score":0.91,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":2.5},"29":{"score":0.9643,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":9.6},"30":{"score":0.6505,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2398,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.6},"32":{"score":0.6609,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":38.6},"33":{"score":0.3442,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":111.2},"36":{"score":0.4599,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1153.5},"37":{"score":0.6082,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.5},"38":{"score":0.2454,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":239.4},"39":{"score":0.8894,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":12.9},"40":{"score":0.6027,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":3.4},"41":{"score":0.6086,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":8.3},"42":{"score":0.7735,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":37.6},"43":{"score":0.6719,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":11.8},"44":{"score":0.716,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.7},"45":{"score":0.0858,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":38.4},"50":{"score":0.608,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":36.7},"56":{"score":0.789,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":71.1},"57":{"score":0.6518,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":15.2},"58":{"score":0.7518,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":7.8},"59":{"score":0.8414,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":38},"61":{"score":0.8788,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":20.4},"62":{"score":0.8568,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":44.6},"64":{"score":0.8201,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":7.6}},"benchmarks":{"1":{"raw":0.9463,"skill":0.9275,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8921,"private_eq":0.9202},"2":{"raw":0.6707,"skill":0.6198,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6876,"private_eq":0.7317},"3":{"raw":0.8425,"skill":0.8395,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8278,"private_eq":0.832},"4":{"raw":0.8799,"skill":0.8784,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8998,"private_eq":0.9012},"5":{"raw":0.9298,"skill":0.9294,"coverage":1,"random":0.006,"tracks":[],"combined":0.7243,"private_eq":0.7281},"6":{"raw":0.7994,"skill":0.5297,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2159,"skill":0.2159,"coverage":1,"random":0,"tracks":[],"combined":0.2247,"private_eq":0.2325},"10":{"raw":0.3202,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8306,"skill":0.7551,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7034,"skill":0.5557,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5393,"private_eq":0.6953},"20":{"raw":0.9043,"skill":0.8085,"coverage":1,"random":0.5,"tracks":[],"combined":0.8058,"private_eq":0.9035},"21":{"raw":0.6305,"skill":0.261,"coverage":1,"random":0.5,"tracks":[],"combined":0.3371,"private_eq":0.6718},"22":{"raw":0.1902,"skill":0.1839,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2546,"private_eq":0.2678},"23":{"raw":0.6369,"skill":0.2739,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8352,"skill":0.7803,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4388,"skill":0.2517,"coverage":1,"random":0.25,"tracks":[],"combined":0.2317,"private_eq":0.4296},"26":{"raw":0.9861,"skill":0.9815,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9659,"skill":0.9545,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.91,"skill":0.82,"coverage":1,"random":0.5,"tracks":[],"combined":0.5268,"private_eq":0.7656},"29":{"raw":0.9643,"skill":0.9524,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6505,"skill":0.5841,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7521,"random":0.25},{"track":"10-choice","raw":0.5489,"random":0.1}],"combined":0.6756,"private_eq":0.7346},"31":{"raw":0.2398,"skill":0.172,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1223,"private_eq":0.2024},"32":{"raw":0.6609,"skill":0.4609,"coverage":1,"random":0.371,"tracks":[],"combined":0.1423,"private_eq":0.466},"33":{"raw":0.3442,"skill":0.3355,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3845,"private_eq":0.3985},"36":{"raw":0.4599,"skill":0.389,"coverage":1,"random":0.116,"tracks":[],"combined":0.3741,"private_eq":0.4521},"37":{"raw":0.6082,"skill":0.5086,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5591,"private_eq":0.6517},"38":{"raw":0.2454,"skill":0.2213,"coverage":1,"random":0.031,"tracks":[],"combined":0.3872,"private_eq":0.412},"39":{"raw":0.8894,"skill":0.8373,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5786,"private_eq":0.7161},"40":{"raw":0.6027,"skill":0.4888,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3986,"headline":false},{"track":"A · English","score":0.6027,"headline":true},{"track":"C · Arabic pairs","score":0.74,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"combined":0.4083,"private_eq":0.5445},"41":{"raw":0.6086,"skill":0.4129,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4642,"private_eq":0.6462},"42":{"raw":0.7735,"skill":0.5593,"coverage":1,"random":0.486,"tracks":[],"combined":0.7413,"private_eq":0.868},"43":{"raw":0.6719,"skill":0.4795,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5619,"private_eq":0.7264},"44":{"raw":0.716,"skill":0.432,"coverage":1,"random":0.5,"tracks":[],"combined":0.4603,"private_eq":0.7327},"45":{"raw":0.0858,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.608,"skill":0.4311,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.789,"skill":0.578,"coverage":1,"random":0.5,"tracks":[],"combined":0.577,"private_eq":0.7904},"57":{"raw":0.6518,"skill":0.6084,"coverage":1,"random":0.1109,"tracks":[],"combined":0.5077,"private_eq":0.5664},"58":{"raw":0.7518,"skill":0.6402,"coverage":1,"random":0.3101,"tracks":[],"combined":0.2634,"private_eq":0.4969},"59":{"raw":0.8414,"skill":0.6712,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2653,"private_eq":0.6492},"61":{"raw":0.8788,"skill":0.7576,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.8568,"skill":0.8091,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.8201,"skill":0.7751,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"scores":{"balanced_raw":67.74,"balanced_skill":53.5,"breadth_skill":57.15,"public_skill":58.03},"frozen_scores":{"balanced_raw":67.74,"balanced_skill":58.03,"breadth_skill":57.15},"categories":[{"id":"knowledge","raw":0.5574,"skill":0.4374,"combined":0.3872,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7226,"skill":0.6123,"combined":0.5587,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7575,"skill":0.6788,"combined":0.6329,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7204,"skill":0.6973,"combined":0.663,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6317,"skill":0.4556,"combined":0.4859,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-gemma4-26b-a4b","pub":58.03,"same_skills":54.99,"new_domains":39.86,"v":53.5,"lab":false,"jev":false,"rank_v":84,"rank_pub":79,"name":"GEV-26B-Decide","eq":{"S":54.58,"O":51.88},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":32.75,"new_domains":20.75}}},{"engine":"xor-1.2","name":"Xor 1.2","long_name":"Xor 1.2","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":35950000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/juspay/xor","code_url":null,"base_model":"Qwen/Qwen3.6-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","weights_repo":null,"served_checkpoint":null,"org":{"id":"juspay","name":"Juspay Technologies","logo":"data/orgs/juspay.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7101,"conf":0.7289,"ece":0.0236,"brier":0.3829,"over95":0.0075,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.018,"conf":0.0784,"acc":0.1388},{"lo":0.1,"w":0.0278,"conf":0.1415,"acc":0.1861},{"lo":0.2,"w":0.0171,"conf":0.2493,"acc":0.2533},{"lo":0.3,"w":0.0415,"conf":0.3567,"acc":0.3514},{"lo":0.4,"w":0.0563,"conf":0.4527,"acc":0.3745},{"lo":0.5,"w":0.128,"conf":0.5501,"acc":0.5105},{"lo":0.6,"w":0.1135,"conf":0.6503,"acc":0.641},{"lo":0.7,"w":0.1205,"conf":0.7509,"acc":0.7076},{"lo":0.8,"w":0.1435,"conf":0.8509,"acc":0.8451},{"lo":0.9,"w":0.3339,"conf":0.9652,"acc":0.9519}],"areas":{"knowledge":{"ece":0.0492,"acc":0.6358,"conf":0.6845},"language":{"ece":0.0308,"acc":0.8148,"conf":0.7885},"retrieval":{"ece":0.1151,"acc":0.6478,"conf":0.7608},"tools":{"ece":0.0289,"acc":0.93,"conf":0.9057},"arts":{"ece":0.0301,"acc":0.6028,"conf":0.5876}}},"gaps":null,"latency":{"median":149.9,"p95":null,"mean":186.5},"results":{"1":{"score":0.9622,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":514.8},"2":{"score":0.6308,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2695.4},"3":{"score":0.8406,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1269.6},"4":{"score":0.7831,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":129.9},"5":{"score":0.8574,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":153.8},"6":{"score":0.8,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":522.6},"9":{"score":0.4773,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":454.8},"10":{"score":0.3503,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":250.8},"11":{"score":0.7487,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":596.5},"12":{"score":0.6994,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":152.9},"20":{"score":0.9672,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":118.9},"21":{"score":0.6381,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":154.9},"22":{"score":0.3195,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":387.1},"23":{"score":0.6466,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":164.9},"24":{"score":0.8461,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":117.8},"25":{"score":0.5,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":116.6},"26":{"score":0.9916,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":116.9},"27":{"score":0.9642,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":117.9},"28":{"score":0.8137,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":117.9},"29":{"score":0.942,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":117.8},"30":{"score":0.3988,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1544,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":171.8},"32":{"score":0.6064,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":309.7},"33":{"score":0.2933,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":357.8},"36":{"score":0.4352,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":592.9},"37":{"score":0.5182,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":340.8},"38":{"score":0.1691,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":441.6},"39":{"score":0.8906,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":248.5},"40":{"score":0.5461,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":164.9},"41":{"score":0.7003,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":146.9},"42":{"score":0.791,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":148.9},"43":{"score":0.5982,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":134.7},"44":{"score":0.6504,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":142.9},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":154.8},"50":{"score":0.4553,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":154.8},"56":{"score":0.6675,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":191.8},"57":{"score":0.6376,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":120.5},"58":{"score":0.7209,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":122.9},"59":{"score":0.7106,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":168.8},"61":{"score":0.733,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":161.9},"62":{"score":0.7371,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":155.8},"64":{"score":0.7292,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":153.9}},"benchmarks":{"1":{"raw":0.9622,"skill":0.949,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6308,"skill":0.5737,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6746,"private_eq":0.7206},"3":{"raw":0.8406,"skill":0.8375,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8383,"private_eq":0.8422},"4":{"raw":0.7831,"skill":0.7803,"coverage":1,"random":0.0127,"tracks":[],"combined":0.865,"private_eq":0.8672},"5":{"raw":0.8574,"skill":0.8565,"coverage":1,"random":0.006,"tracks":[],"combined":0.6777,"private_eq":0.6823},"6":{"raw":0.8,"skill":0.5311,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4773,"skill":0.4773,"coverage":1,"random":0,"tracks":[],"combined":0.427,"private_eq":0.4325},"10":{"raw":0.3503,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7487,"skill":0.6366,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6994,"skill":0.5497,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4924,"private_eq":0.6643},"20":{"raw":0.967,"skill":0.934,"coverage":1,"random":0.5,"tracks":[],"combined":0.9215,"private_eq":0.9607},"21":{"raw":0.6381,"skill":0.2763,"coverage":1,"random":0.5,"tracks":[],"combined":0.3472,"private_eq":0.6768},"22":{"raw":0.3092,"skill":0.3038,"coverage":1,"random":0.0078,"tracks":[],"combined":0.234,"private_eq":0.2476},"23":{"raw":0.6564,"skill":0.3127,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8461,"skill":0.7948,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5,"skill":0.3333,"coverage":1,"random":0.25,"tracks":[],"combined":0.3199,"private_eq":0.495},"26":{"raw":0.9916,"skill":0.9888,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9642,"skill":0.9523,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8137,"skill":0.6274,"coverage":1,"random":0.5,"tracks":[],"combined":0.5622,"private_eq":0.7831},"29":{"raw":0.942,"skill":0.9227,"coverage":1,"random":0.25,"tracks":[],"combined":0.9584,"private_eq":0.9684},"30":{"raw":0.3988,"skill":0.2779,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5133,"random":0.25},{"track":"10-choice","raw":0.2843,"random":0.1}],"combined":0.4964,"private_eq":0.5884},"31":{"raw":0.1544,"skill":0.079,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0691,"private_eq":0.1542},"32":{"raw":0.6064,"skill":0.3742,"coverage":1,"random":0.371,"tracks":[],"combined":0.2736,"private_eq":0.5477},"33":{"raw":0.2933,"skill":0.2839,"coverage":1,"random":0.0131,"tracks":[],"combined":0.364,"private_eq":0.3785},"36":{"raw":0.4352,"skill":0.3611,"coverage":1,"random":0.116,"tracks":[],"combined":0.3703,"private_eq":0.4488},"37":{"raw":0.5182,"skill":0.3957,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5256,"private_eq":0.6253},"38":{"raw":0.1691,"skill":0.1425,"coverage":1,"random":0.031,"tracks":[],"combined":0.3093,"private_eq":0.3374},"39":{"raw":0.8906,"skill":0.8391,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5626,"private_eq":0.7053},"40":{"raw":0.5461,"skill":0.4161,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3324,"headline":false},{"track":"A · English","score":0.5461,"headline":true},{"track":"C · Arabic pairs","score":0.72,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.4098,"private_eq":0.5457},"41":{"raw":0.7003,"skill":0.5505,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4515,"private_eq":0.6378},"42":{"raw":0.791,"skill":0.5934,"coverage":1,"random":0.486,"tracks":[],"combined":0.77,"private_eq":0.8826},"43":{"raw":0.5982,"skill":0.3626,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4616,"private_eq":0.6639},"44":{"raw":0.6504,"skill":0.3008,"coverage":1,"random":0.5,"tracks":[],"combined":0.246,"private_eq":0.6268},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4553,"skill":0.2094,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.6675,"skill":0.335,"coverage":1,"random":0.5,"tracks":[],"combined":0.4558,"private_eq":0.7305},"57":{"raw":0.6376,"skill":0.5924,"coverage":1,"random":0.1109,"tracks":[],"combined":0.5343,"private_eq":0.5898},"58":{"raw":0.7209,"skill":0.5954,"coverage":1,"random":0.3101,"tracks":[],"combined":0.2104,"private_eq":0.4608},"59":{"raw":0.7106,"skill":0.4,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2821,"private_eq":0.6572},"61":{"raw":0.733,"skill":0.466,"coverage":1,"random":0.5,"tracks":[],"combined":0.5414,"private_eq":0.7728},"62":{"raw":0.7371,"skill":0.6495,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.7292,"skill":0.6615,"coverage":1,"random":0.2,"tracks":[],"combined":0.6907,"private_eq":0.7546}},"coverage":1,"pending":0,"scores":{"balanced_raw":64.21,"balanced_skill":53.62,"breadth_skill":51.06,"public_skill":52.28},"frozen_scores":{"balanced_raw":64.21,"balanced_skill":52.28,"breadth_skill":51.06},"categories":[{"id":"knowledge","raw":0.5051,"skill":0.3547,"combined":0.3577,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6943,"skill":0.5686,"combined":0.5388,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6681,"skill":0.5446,"combined":0.5786,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7423,"skill":0.7119,"combined":0.7161,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6259,"skill":0.4496,"combined":0.4808,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"xor-1.2","pub":52.28,"same_skills":53.6,"new_domains":41.82,"v":53.62,"lab":false,"jev":false,"rank_v":83,"rank_pub":109,"name":"Xor 1.2","eq":{"S":53.17,"O":54.29},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":31.9,"new_domains":21.71}}},{"engine":"decider-chat-qwen3.6-27b","name":"Decider chat · Qwen3.6-27B","long_name":"Decider chat · Qwen3.6-27B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/Mapika/decider","base_model":"Qwen/Qwen3.6-27B","base_url":"https://huggingface.co/Qwen/Qwen3.6-27B","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.6-27B"},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.6968,"conf":0.6848,"ece":0.0209,"brier":0.3998,"over95":0.001,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0178,"conf":0.0629,"acc":0.1578},{"lo":0.1,"w":0.0226,"conf":0.1531,"acc":0.1685},{"lo":0.2,"w":0.0328,"conf":0.2495,"acc":0.2453},{"lo":0.3,"w":0.05,"conf":0.3558,"acc":0.3247},{"lo":0.4,"w":0.0754,"conf":0.4503,"acc":0.4396},{"lo":0.5,"w":0.1335,"conf":0.5491,"acc":0.5344},{"lo":0.6,"w":0.133,"conf":0.6521,"acc":0.6622},{"lo":0.7,"w":0.1458,"conf":0.7518,"acc":0.7561},{"lo":0.8,"w":0.1809,"conf":0.8513,"acc":0.8858},{"lo":0.9,"w":0.2083,"conf":0.9442,"acc":0.9741}],"areas":{"knowledge":{"ece":0.0329,"acc":0.6483,"conf":0.6508},"language":{"ece":0.0457,"acc":0.8102,"conf":0.7664},"retrieval":{"ece":0.0804,"acc":0.5994,"conf":0.6328},"tools":{"ece":0.0662,"acc":0.9059,"conf":0.8434},"arts":{"ece":0.0559,"acc":0.5517,"conf":0.5591}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":83.6,"p95":965.6,"mean":284},"results":{"1":{"score":0.9752,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":272.7},"2":{"score":0.6438,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2975.6},"3":{"score":0.7972,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.751,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":234.8},"5":{"score":0.8506,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":372.5},"6":{"score":0.7991,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":505.3},"9":{"score":0.4432,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1759,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":373.5},"11":{"score":0.74,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7459,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":63.4},"20":{"score":0.8764,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":63.3},"21":{"score":0.6084,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":57},"22":{"score":0.185,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1814.2},"23":{"score":0.6332,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":122.9},"24":{"score":0.8521,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":57.9},"25":{"score":0.5459,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9882,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":57.3},"27":{"score":0.9727,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":57},"28":{"score":0.8319,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":56.9},"29":{"score":0.9542,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":68},"30":{"score":0.4155,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1762,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":186.2},"32":{"score":0.6263,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3479,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":899.2},"36":{"score":0.4665,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4767,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.7},"38":{"score":0.2444,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2353.8},"39":{"score":0.8183,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.458,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":56.8,"tracks":[{"track":"A · Arabic","score":0.4615,"headline":false},{"track":"A · English","score":0.458,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5839,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":63.5},"42":{"score":0.8203,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":185.8},"43":{"score":0.6404,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6634,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":63.6},"45":{"score":0.1158,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4087,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":185.9},"56":{"score":0.6755,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":964.7,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.6287,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":66.5,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.7273,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":64.5,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7503,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":165.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.68,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":100.8,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.785,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":184.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.7273,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":63.65,"balanced_skill":54.67,"breadth_skill":50.44,"public_skill":51.57},"frozen_scores":{"balanced_raw":63.65,"balanced_skill":51.57,"breadth_skill":50.44},"categories":[{"id":"knowledge","raw":0.5241,"skill":0.383,"combined":0.3799,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6867,"skill":0.5594,"combined":0.5257,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6536,"skill":0.5223,"combined":0.5376,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7405,"skill":0.7142,"combined":0.6983,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.573,"skill":0.3695,"combined":0.4286,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9752,"skill":0.9665,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9056,"private_eq":0.9301},"2":{"raw":0.6438,"skill":0.5886,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6906,"private_eq":0.7343},"3":{"raw":0.7972,"skill":0.7933,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8068,"private_eq":0.8116},"4":{"raw":0.751,"skill":0.7478,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8127,"private_eq":0.8162},"5":{"raw":0.8506,"skill":0.8497,"coverage":1,"random":0.006,"tracks":[],"combined":0.6229,"private_eq":0.6284},"6":{"raw":0.7991,"skill":0.5262,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.786,"headline":false},{"track":"RouterBench-5shot","score":0.8122,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.4432,"skill":0.4432,"coverage":1,"random":0,"tracks":[],"combined":0.4143,"private_eq":0.42},"11":{"raw":0.74,"skill":0.624,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7459,"skill":0.6194,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5034,"private_eq":0.6716},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8784,"skill":0.7569,"coverage":1,"random":0.5,"tracks":[],"combined":0.7479,"private_eq":0.8749},"21":{"raw":0.6084,"skill":0.2169,"coverage":1,"random":0.5,"tracks":[],"combined":0.2264,"private_eq":0.6171},"22":{"raw":0.1812,"skill":0.1748,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2144,"private_eq":0.2284},"23":{"raw":0.6339,"skill":0.2678,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8592,"skill":0.8123,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5459,"skill":0.3946,"coverage":1,"random":0.25,"tracks":[],"combined":0.2778,"private_eq":0.4638},"28":{"raw":0.8319,"skill":0.6638,"coverage":1,"random":0.5,"tracks":[],"combined":0.6531,"private_eq":0.828},"29":{"raw":0.9542,"skill":0.9389,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.4155,"skill":0.2978,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5254,"random":0.25},{"track":"10-choice","raw":0.3055,"random":0.1}],"combined":0.5105,"private_eq":0.5999},"31":{"raw":0.1762,"skill":0.1028,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0918,"private_eq":0.1748},"32":{"raw":0.6263,"skill":0.4059,"coverage":1,"random":0.371,"tracks":[],"combined":0.3124,"private_eq":0.5718},"33":{"raw":0.3479,"skill":0.3392,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4359,"private_eq":0.4486},"36":{"raw":0.4665,"skill":0.3964,"coverage":1,"random":0.116,"tracks":[],"combined":0.3708,"private_eq":0.4492},"37":{"raw":0.4767,"skill":0.3436,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4783,"private_eq":0.588},"38":{"raw":0.2444,"skill":0.2202,"coverage":1,"random":0.031,"tracks":[],"combined":0.3831,"private_eq":0.4081},"39":{"raw":0.8183,"skill":0.7328,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5464,"private_eq":0.6944},"40":{"raw":0.458,"skill":0.3027,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4615,"headline":false},{"track":"A · English","score":0.458,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.2798,"private_eq":0.4458},"41":{"raw":0.5839,"skill":0.3759,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4342,"private_eq":0.6264},"42":{"raw":0.8203,"skill":0.6504,"coverage":1,"random":0.486,"tracks":[],"combined":0.6615,"private_eq":0.8275},"43":{"raw":0.6404,"skill":0.4294,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4827,"private_eq":0.677},"44":{"raw":0.6634,"skill":0.3268,"coverage":1,"random":0.5,"tracks":[],"combined":0.4226,"private_eq":0.7141},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4087,"skill":0.1418,"coverage":1,"random":0.311,"tracks":[],"combined":0.2539,"private_eq":0.4911},"56":{"raw":0.6755,"skill":0.351,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5424,"private_eq":0.7733},"57":{"raw":0.6287,"skill":0.5824,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3967,"private_eq":0.4688},"58":{"raw":0.7273,"skill":0.6047,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.2121,"private_eq":0.4619},"59":{"raw":0.7503,"skill":0.4823,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.3049,"private_eq":0.6681},"61":{"raw":0.68,"skill":0.36,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3597,"private_eq":0.683},"62":{"raw":0.785,"skill":0.7133,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.6108,"private_eq":0.7107},"64":{"raw":0.7273,"skill":0.6591,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.7003,"private_eq":0.7622}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author decider.serve v1.2.1 chat layout (bf16, graphs), raised admission limits as board; one sequential HTTP client (engine-launched server, not attached)","v03":{"engine":"decider-chat-qwen3.6-27b","pub":51.57,"same_skills":52.15,"new_domains":45.72,"v":54.67,"lab":false,"jev":false,"rank_v":79,"rank_pub":110,"name":"Decider chat · Qwen3.6-27B","eq":{"S":51.7,"O":59.11},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":31.02,"new_domains":23.65}}},{"engine":"clef","name":"Cloudflare clef","long_name":"Cloudflare clef","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Cloudflare/clef","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"Cloudflare","name":"Cloudflare","logo":"data/orgs/Cloudflare.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7783,"conf":0.8096,"ece":0.0395,"brier":0.306,"over95":0.0233,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0311,"conf":0.0637,"acc":0.1167},{"lo":0.1,"w":0.0208,"conf":0.1418,"acc":0.241},{"lo":0.2,"w":0.0088,"conf":0.2576,"acc":0.2976},{"lo":0.3,"w":0.0214,"conf":0.3559,"acc":0.3172},{"lo":0.4,"w":0.0379,"conf":0.4593,"acc":0.3576},{"lo":0.5,"w":0.065,"conf":0.5505,"acc":0.5087},{"lo":0.6,"w":0.0611,"conf":0.6486,"acc":0.5971},{"lo":0.7,"w":0.0742,"conf":0.7508,"acc":0.6736},{"lo":0.8,"w":0.1089,"conf":0.8545,"acc":0.813},{"lo":0.9,"w":0.5709,"conf":0.9691,"acc":0.9435}],"areas":{"knowledge":{"ece":0.0417,"acc":0.7354,"conf":0.7721},"language":{"ece":0.0222,"acc":0.8652,"conf":0.8874},"retrieval":{"ece":0.1393,"acc":0.7431,"conf":0.8765},"tools":{"ece":0.0263,"acc":0.9648,"conf":0.9487},"arts":{"ece":0.0464,"acc":0.6323,"conf":0.6151}}},"gaps":null,"latency":{"median":102.1,"p95":null,"mean":245.5},"results":{"1":{"score":0.9847,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":156.1},"2":{"score":0.6906,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1801.7},"3":{"score":0.9193,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1215.4},"4":{"score":0.9408,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":325.8},"5":{"score":0.9741,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":592.7},"6":{"score":0.7966,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":451.6},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":571.2},"10":{"score":0.4381,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":194.9},"11":{"score":0.8135,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":756.5},"12":{"score":0.6997,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":99.2},"20":{"score":0.9698,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":98.5},"21":{"score":0.6655,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":98.4},"22":{"score":0.159,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1251},"23":{"score":0.6594,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.1},"24":{"score":0.9022,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":98.8},"25":{"score":0.4847,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":99.7},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":98.4},"27":{"score":0.9778,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":98.5},"28":{"score":0.9353,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":98.4},"29":{"score":0.9818,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":99.7},"30":{"score":0.5993,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.245,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":222.1},"32":{"score":0.8378,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":195.8},"33":{"score":0.3345,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":201.8},"36":{"score":0.4746,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2429.3},"37":{"score":0.5737,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":103.7},"38":{"score":0.3317,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1762.2},"39":{"score":0.9609,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":103.7},"40":{"score":0.4639,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":99.4},"41":{"score":0.5934,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":99.4},"42":{"score":0.8298,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":120},"43":{"score":0.8649,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":99.2},"44":{"score":0.9408,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":98.6},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":102.1},"50":{"score":0.6868,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":175.4},"56":{"score":0.7935,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":201},"57":{"score":0.6597,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":99.7},"58":{"score":0.7363,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":98},"59":{"score":0.7954,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":157.8},"61":{"score":0.6558,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":118},"62":{"score":0.7245,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":154.2},"64":{"score":0.7027,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":98.6}},"benchmarks":{"1":{"raw":0.9847,"skill":0.9793,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.6906,"skill":0.6427,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7021,"private_eq":0.7441},"3":{"raw":0.9193,"skill":0.9177,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8908,"private_eq":0.8931},"4":{"raw":0.9408,"skill":0.94,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8747,"private_eq":0.8767},"5":{"raw":0.9741,"skill":0.9739,"coverage":1,"random":0.006,"tracks":[],"combined":0.6627,"private_eq":0.6675},"6":{"raw":0.7966,"skill":0.5231,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.69,"private_eq":0.6925},"10":{"raw":0.4381,"skill":0.0651,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8135,"skill":0.7303,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6997,"skill":0.5502,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5371,"private_eq":0.6938},"20":{"raw":0.9703,"skill":0.9406,"coverage":1,"random":0.5,"tracks":[],"combined":0.9288,"private_eq":0.9643},"21":{"raw":0.6655,"skill":0.3311,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.1547,"skill":0.148,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1894,"private_eq":0.2039},"23":{"raw":0.6698,"skill":0.3396,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.9022,"skill":0.8696,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4847,"skill":0.3129,"coverage":1,"random":0.25,"tracks":[],"combined":0.2904,"private_eq":0.4731},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9778,"skill":0.9704,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9353,"skill":0.8706,"coverage":1,"random":0.5,"tracks":[],"combined":0.3907,"private_eq":0.6983},"29":{"raw":0.9818,"skill":0.9757,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.5993,"skill":0.5214,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6998,"random":0.25},{"track":"10-choice","raw":0.4989,"random":0.1}],"combined":0.5868,"private_eq":0.6622},"31":{"raw":0.245,"skill":0.1777,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0766,"private_eq":0.161},"32":{"raw":0.8378,"skill":0.7421,"coverage":1,"random":0.371,"tracks":[],"combined":0.2273,"private_eq":0.5189},"33":{"raw":0.3345,"skill":0.3257,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2868,"private_eq":0.3032},"36":{"raw":0.4746,"skill":0.4057,"coverage":1,"random":0.116,"tracks":[],"combined":0.3916,"private_eq":0.4674},"37":{"raw":0.5737,"skill":0.4653,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5581,"private_eq":0.6509},"38":{"raw":0.3317,"skill":0.3103,"coverage":1,"random":0.031,"tracks":[],"combined":0.4168,"private_eq":0.4403},"39":{"raw":0.9609,"skill":0.9425,"coverage":1,"random":0.3201,"tracks":[],"combined":0.574,"private_eq":0.713},"40":{"raw":0.4639,"skill":0.3103,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.7007,"headline":false},{"track":"A · English","score":0.4639,"headline":true},{"track":"C · Arabic pairs","score":0.94,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.5934,"skill":0.3901,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3346},"42":{"raw":0.8298,"skill":0.6689,"coverage":1,"random":0.486,"tracks":[],"combined":0.8235,"private_eq":0.9098},"43":{"raw":0.8649,"skill":0.7857,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4116,"private_eq":0.6327},"44":{"raw":0.9408,"skill":0.8816,"coverage":1,"random":0.5,"tracks":[],"combined":0.7763,"private_eq":0.8889},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6868,"skill":0.5454,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.7935,"skill":0.587,"coverage":1,"random":0.5,"tracks":[],"combined":0.7069,"private_eq":0.8546},"57":{"raw":0.6597,"skill":0.6173,"coverage":1,"random":0.1109,"tracks":[],"combined":0.3977,"private_eq":0.4697},"58":{"raw":0.7363,"skill":0.6178,"coverage":1,"random":0.3101,"tracks":[],"combined":0.2166,"private_eq":0.465},"59":{"raw":0.7954,"skill":0.5758,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3735,"private_eq":0.7008},"61":{"raw":0.6558,"skill":0.3116,"coverage":1,"random":0.5,"tracks":[],"combined":0.3656,"private_eq":0.6859},"62":{"raw":0.7245,"skill":0.6327,"coverage":1,"random":0.25,"tracks":[],"combined":0.4808,"private_eq":0.6143},"64":{"raw":0.7027,"skill":0.6284,"coverage":1,"random":0.2,"tracks":[],"combined":0.6422,"private_eq":0.7162}},"coverage":1,"pending":0,"scores":{"balanced_raw":71.07,"balanced_skill":54.92,"breadth_skill":61.04,"public_skill":61.71},"frozen_scores":{"balanced_raw":71.07,"balanced_skill":61.71,"breadth_skill":61.04},"categories":[{"id":"knowledge","raw":0.6129,"skill":0.528,"combined":0.3629,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7241,"skill":0.6127,"combined":0.4779,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.741,"skill":0.6284,"combined":0.5977,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8346,"skill":0.8071,"combined":0.7551,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6416,"skill":0.4888,"combined":0.4471,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"clef","pub":61.71,"same_skills":53.11,"new_domains":45.06,"v":54.92,"lab":false,"jev":false,"rank_v":77,"rank_pub":50,"name":"Cloudflare clef","eq":{"S":52.67,"O":58.3},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":31.6,"new_domains":23.32}}},{"engine":"joshua-diffusion-full","name":"JoshuaSP diffusiongemma (open-jev)","long_name":"JoshuaSP diffgemma","variant":null,"meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/JoshuaSP/open-jev","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it"},"completed":109886,"counts":{"ok":109886},"lifted":false,"calibration":{"acc":0.6805,"conf":0.896,"ece":0.2155,"brier":0.5241,"over95":0.1366,"n":72589,"benchmarks":32,"rel":[{"lo":0,"w":0.002,"conf":0.0894,"acc":0},{"lo":0.1,"w":0.0148,"conf":0.1478,"acc":0},{"lo":0.2,"w":0.0121,"conf":0.2486,"acc":0.0881},{"lo":0.3,"w":0.0154,"conf":0.3542,"acc":0.1417},{"lo":0.4,"w":0.0191,"conf":0.4495,"acc":0.1841},{"lo":0.5,"w":0.037,"conf":0.5496,"acc":0.4329},{"lo":0.6,"w":0.0384,"conf":0.6459,"acc":0.4039},{"lo":0.7,"w":0.0435,"conf":0.7526,"acc":0.4219},{"lo":0.8,"w":0.067,"conf":0.8528,"acc":0.5294},{"lo":0.9,"w":0.7507,"conf":0.9878,"acc":0.7837}],"areas":{"knowledge":{"ece":0.2513,"acc":0.6139,"conf":0.8651},"language":{"ece":0.1913,"acc":0.7806,"conf":0.9719},"retrieval":{"ece":0.2289,"acc":0.6582,"conf":0.886},"tools":{"ece":0.0448,"acc":0.9328,"conf":0.9743},"arts":{"ece":0.2749,"acc":0.513,"conf":0.788}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":260.4,"p95":537.3,"mean":337.8},"results":{"1":{"score":0.9652,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":272.9},"2":{"score":0.6121,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1603.1},"3":{"score":0.8366,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7428,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":316.5},"5":{"score":0.8349,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":465.5},"6":{"score":0.79,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":488.9},"9":{"score":0.4205,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4065,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":293.7},"11":{"score":0.7596,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6637,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":255.1},"20":{"score":0.8694,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":256.6},"21":{"score":0.6298,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":249.9},"22":{"score":0.025,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1205.3},"23":{"score":0.5824,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":259.3},"24":{"score":0.7926,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":250.8},"25":{"score":0.449,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9823,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":248.2},"27":{"score":0.9445,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":249},"28":{"score":0.7364,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":249.1},"29":{"score":0.8328,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":259.4},"30":{"score":0.3313,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1416,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":277.6},"32":{"score":0.6117,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2745,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":291.8},"36":{"score":0.4294,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5337,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":259.4},"38":{"score":0.2449,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1667.8},"39":{"score":0.8904,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.66,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":246.5,"tracks":[{"track":"A · Arabic","score":0.49,"headline":false},{"track":"A · English","score":0.66,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5569,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":256.6},"42":{"score":0.7843,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":267.1},"43":{"score":0.6474,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6778,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":254.6},"45":{"score":0.0978,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4499,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":273},"56":{"score":0.8535,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":283.3,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.5686,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":252,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.7067,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":247.5,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7043,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":271.5,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.7085,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":261.8,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7544,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":269.6,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6364,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":62.33,"balanced_skill":49.53,"breadth_skill":48.38,"public_skill":50.14},"frozen_scores":{"balanced_raw":62.33,"balanced_skill":50.14,"breadth_skill":48.38},"categories":[{"id":"knowledge","raw":0.4795,"skill":0.3246,"combined":0.3266,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6804,"skill":0.5431,"combined":0.5191,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6825,"skill":0.5837,"combined":0.5417,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7313,"skill":0.7016,"combined":0.6689,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5315,"skill":0.3198,"combined":0.4094,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9652,"skill":0.953,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.6121,"skill":0.5521,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6705,"private_eq":0.7171},"3":{"raw":0.8366,"skill":0.8335,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8068,"private_eq":0.8116},"4":{"raw":0.7428,"skill":0.7395,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8469,"private_eq":0.8496},"5":{"raw":0.8349,"skill":0.8339,"coverage":1,"random":0.006,"tracks":[],"combined":0.6516,"private_eq":0.6566},"6":{"raw":0.79,"skill":0.5041,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.779,"headline":false},{"track":"RouterBench-5shot","score":0.8011,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.4205,"skill":0.4205,"coverage":1,"random":0,"tracks":[],"combined":0.3334,"private_eq":0.34},"11":{"raw":0.7596,"skill":0.6523,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6637,"skill":0.4963,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4951,"private_eq":0.6661},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8729,"skill":0.7459,"coverage":1,"random":0.5,"tracks":[],"combined":0.8276,"private_eq":0.9143},"21":{"raw":0.6298,"skill":0.2595,"coverage":1,"random":0.5,"tracks":[],"combined":0.2768,"private_eq":0.642},"22":{"raw":0.0263,"skill":0.0187,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0028,"private_eq":0.0209},"23":{"raw":0.5739,"skill":0.1477,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7954,"skill":0.7272,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.449,"skill":0.2653,"coverage":1,"random":0.25,"tracks":[],"combined":0.1938,"private_eq":0.4015},"28":{"raw":0.7364,"skill":0.4728,"coverage":1,"random":0.5,"tracks":[],"combined":0.542,"private_eq":0.7731},"29":{"raw":0.8328,"skill":0.7771,"coverage":1,"random":0.25,"tracks":[],"combined":0.7059,"private_eq":0.7812},"30":{"raw":0.3313,"skill":0.1935,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4284,"random":0.25},{"track":"10-choice","raw":0.2343,"random":0.1}],"combined":0.3891,"private_eq":0.5009},"31":{"raw":0.1416,"skill":0.0651,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0734,"private_eq":0.1581},"32":{"raw":0.6117,"skill":0.3827,"coverage":1,"random":0.371,"tracks":[],"combined":0.1731,"private_eq":0.4852},"33":{"raw":0.2745,"skill":0.2649,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4154,"private_eq":0.4286},"36":{"raw":0.4294,"skill":0.3545,"coverage":1,"random":0.116,"tracks":[],"combined":0.3459,"private_eq":0.4275},"37":{"raw":0.5337,"skill":0.4152,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4447,"private_eq":0.5615},"38":{"raw":0.2449,"skill":0.2207,"coverage":1,"random":0.031,"tracks":[],"combined":0.4196,"private_eq":0.443},"39":{"raw":0.8904,"skill":0.8388,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5944,"private_eq":0.7267},"40":{"raw":0.66,"skill":0.5626,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.49,"headline":false},{"track":"A · English","score":0.66,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.5711,"private_eq":0.6696},"41":{"raw":0.5569,"skill":0.3353,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4559,"private_eq":0.6407},"42":{"raw":0.7843,"skill":0.5804,"coverage":1,"random":0.486,"tracks":[],"combined":0.8013,"private_eq":0.8985},"43":{"raw":0.6474,"skill":0.4405,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3907,"private_eq":0.6197},"44":{"raw":0.6778,"skill":0.3556,"coverage":1,"random":0.5,"tracks":[],"combined":0.4174,"private_eq":0.7115},"45":{"raw":0.0978,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4499,"skill":0.2016,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.8535,"skill":0.707,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4212,"private_eq":0.7134},"57":{"raw":0.5686,"skill":0.5148,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.4034,"private_eq":0.4747},"58":{"raw":0.7067,"skill":0.5749,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.2508,"private_eq":0.4883},"59":{"raw":0.7043,"skill":0.3869,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.0767,"private_eq":0.5593},"61":{"raw":0.7085,"skill":0.417,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4961,"private_eq":0.7504},"62":{"raw":0.7544,"skill":0.6725,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5287,"private_eq":0.6498},"64":{"raw":0.6364,"skill":0.5455,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.6713,"private_eq":0.7392}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"open-jev[inference] diffusiongemma engine, bf16, chunked-expert path as board; venv-joshua","v03":{"engine":"joshua-diffusion-full","pub":50.14,"same_skills":50.02,"new_domains":37.94,"v":49.53,"lab":false,"jev":false,"rank_v":94,"rank_pub":118,"name":"JoshuaSP diffgemma","eq":{"S":49.54,"O":49.5},"rank_tie":21,"tied":true,"parts":{"public":0,"same_skills":29.72,"new_domains":19.8}}},{"engine":"winnow-12b-q8","name":"Winnow-12B","long_name":"Winnow-12B","variant":"Q8_0","meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":11959730224,"params_estimated":false,"params_from_name":false,"trained_bytes":12669646592,"model_url":"https://huggingface.co/EldanRing/Winnow-12B","code_url":"https://github.com/EldanRing/winnow-inference","base_model":"google/gemma-4-12B","base_url":"https://huggingface.co/google/gemma-4-12B","weights_repo":"EldanRing/Winnow-12B","served_checkpoint":"EldanRing/Winnow-12B"},"completed":110201,"counts":{"ok":110201},"lifted":true,"calibration":{"acc":0.6697,"conf":0.8376,"ece":0.1679,"brier":0.4862,"over95":0.0624,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.0035,"conf":0.0806,"acc":0},{"lo":0.1,"w":0.0152,"conf":0.1522,"acc":0.1179},{"lo":0.2,"w":0.0227,"conf":0.2538,"acc":0.1849},{"lo":0.3,"w":0.0276,"conf":0.3543,"acc":0.2116},{"lo":0.4,"w":0.0335,"conf":0.4537,"acc":0.2994},{"lo":0.5,"w":0.0578,"conf":0.5508,"acc":0.435},{"lo":0.6,"w":0.054,"conf":0.6514,"acc":0.4727},{"lo":0.7,"w":0.076,"conf":0.7509,"acc":0.4814},{"lo":0.8,"w":0.1041,"conf":0.8558,"acc":0.553},{"lo":0.9,"w":0.6056,"conf":0.976,"acc":0.8306}],"areas":{"knowledge":{"ece":0.1675,"acc":0.6121,"conf":0.7783},"language":{"ece":0.1134,"acc":0.7995,"conf":0.9129},"retrieval":{"ece":0.3164,"acc":0.6017,"conf":0.918},"tools":{"ece":0.1021,"acc":0.8499,"conf":0.9437},"arts":{"ece":0.1953,"acc":0.5092,"conf":0.7045}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":72.5,"p95":356.2,"mean":134.4},"results":{"1":{"score":0.9475,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":125.2},"2":{"score":0.6519,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1053.8},"3":{"score":0.6732,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7511,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":173.1},"5":{"score":0.8695,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":233.6},"6":{"score":0.7976,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":290.9},"9":{"score":0.5455,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1189,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":145.9},"11":{"score":0.7289,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6674,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":52.5},"20":{"score":0.8314,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58},"21":{"score":0.6126,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":48.6},"22":{"score":0.081,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":667.1},"23":{"score":0.6018,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.1},"24":{"score":0.7732,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":48.6},"25":{"score":0.3878,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9798,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":49.4},"27":{"score":0.948,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":28.3},"28":{"score":0.7411,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":47.1},"29":{"score":0.8512,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":54.5},"30":{"score":0.4864,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1734,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":132.5},"32":{"score":0.6104,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3358,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":164.3},"36":{"score":0.4528,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5304,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":76},"38":{"score":0.1546,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":985.8},"39":{"score":0.9174,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5773,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":48.2,"tracks":[{"track":"A · Arabic","score":0.4759,"headline":false},{"track":"A · English","score":0.5773,"headline":true},{"track":"C · Arabic pairs","score":0.84,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6863,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":53.2},"42":{"score":0.8294,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":91.2},"43":{"score":0.6947,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6856,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":55.6},"45":{"score":0.1038,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4183,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":128.3},"56":{"score":0.659,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":148.9,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.5483,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":61.4,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.7129,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":62.5,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7708,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":116.5,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.7205,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":82.5,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.8324,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":108.6,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6477,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":62.84,"balanced_skill":46.57,"breadth_skill":49.05,"public_skill":50.66},"frozen_scores":{"balanced_raw":62.84,"balanced_skill":50.66,"breadth_skill":49.05},"categories":[{"id":"knowledge","raw":0.4982,"skill":0.3475,"combined":0.3123,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6901,"skill":0.5699,"combined":0.5057,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6664,"skill":0.5405,"combined":0.546,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.736,"skill":0.7102,"combined":0.6291,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5322,"skill":0.3138,"combined":0.3514,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9475,"skill":0.9291,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8921,"private_eq":0.9202},"2":{"raw":0.6519,"skill":0.598,"coverage":1,"random":0.1341,"tracks":[],"combined":0.682,"private_eq":0.7269},"3":{"raw":0.6732,"skill":0.6669,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4709,"private_eq":0.4858},"4":{"raw":0.7511,"skill":0.7479,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8527,"private_eq":0.8552},"5":{"raw":0.8695,"skill":0.8687,"coverage":1,"random":0.006,"tracks":[],"combined":0.6896,"private_eq":0.694},"6":{"raw":0.7976,"skill":0.5225,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7855,"headline":false},{"track":"RouterBench-5shot","score":0.8097,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.5455,"skill":0.5455,"coverage":1,"random":0,"tracks":[],"combined":0.5028,"private_eq":0.5075},"11":{"raw":0.7289,"skill":0.6079,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6674,"skill":0.5018,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5242,"private_eq":0.6853},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8342,"skill":0.6684,"coverage":1,"random":0.5,"tracks":[],"combined":0.56,"private_eq":0.782},"21":{"raw":0.6126,"skill":0.2253,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.0804,"skill":0.0732,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0998,"private_eq":0.116},"23":{"raw":0.6002,"skill":0.2004,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.782,"skill":0.7094,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[],"combined":0.1923,"private_eq":0.4004},"28":{"raw":0.7411,"skill":0.4822,"coverage":1,"random":0.5,"tracks":[],"combined":0.4058,"private_eq":0.7058},"29":{"raw":0.8512,"skill":0.8016,"coverage":1,"random":0.25,"tracks":[],"combined":0.7703,"private_eq":0.8289},"30":{"raw":0.4864,"skill":0.3862,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6118,"random":0.25},{"track":"10-choice","raw":0.3609,"random":0.1}],"combined":0.5621,"private_eq":0.642},"31":{"raw":0.1734,"skill":0.0997,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0734,"private_eq":0.1581},"32":{"raw":0.6104,"skill":0.3806,"coverage":1,"random":0.371,"tracks":[],"combined":0.2582,"private_eq":0.5381},"33":{"raw":0.3358,"skill":0.327,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3022,"private_eq":0.3182},"36":{"raw":0.4528,"skill":0.381,"coverage":1,"random":0.116,"tracks":[],"combined":0.3474,"private_eq":0.4288},"37":{"raw":0.5304,"skill":0.411,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4505,"private_eq":0.5661},"38":{"raw":0.1546,"skill":0.1276,"coverage":1,"random":0.031,"tracks":[],"combined":0.2346,"private_eq":0.2658},"39":{"raw":0.9174,"skill":0.8785,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5711,"private_eq":0.711},"40":{"raw":0.5773,"skill":0.4562,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4759,"headline":false},{"track":"A · English","score":0.5773,"headline":true},{"track":"C · Arabic pairs","score":0.84,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"combined":0.4277,"private_eq":0.5594},"41":{"raw":0.6863,"skill":0.5295,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4454,"private_eq":0.6338},"42":{"raw":0.8294,"skill":0.6681,"coverage":1,"random":0.486,"tracks":[],"combined":0.6686,"private_eq":0.8311},"43":{"raw":0.6947,"skill":0.5157,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4701,"private_eq":0.6692},"44":{"raw":0.6856,"skill":0.3712,"coverage":1,"random":0.5,"tracks":[],"combined":0.3103,"private_eq":0.6586},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4183,"skill":0.1557,"coverage":1,"random":0.311,"tracks":[],"combined":0.2539,"private_eq":0.4911},"56":{"raw":0.659,"skill":0.318,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3953,"private_eq":0.7006},"57":{"raw":0.5483,"skill":0.492,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3344,"private_eq":0.4141},"58":{"raw":0.7129,"skill":0.5839,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.2314,"private_eq":0.4751},"59":{"raw":0.7708,"skill":0.5248,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.3471,"private_eq":0.6882},"61":{"raw":0.7205,"skill":0.441,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4904,"private_eq":0.7476},"62":{"raw":0.8324,"skill":0.7765,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5767,"private_eq":0.6854},"64":{"raw":0.6477,"skill":0.5596,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5161,"private_eq":0.6165}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"winnow-inference llama.cpp CUDA server (6c2b3c0 + 255-label lift, WINNOW_LABELS_LIFT=1), Winnow-12B Q8_0 GGUF fully on GPU, n_ctx 131072, batch 2048 / ubatch 1024, one slot, one sequential HTTP client; same build and settings as the board","v03":{"engine":"winnow-12b-q8","pub":50.66,"same_skills":48.14,"new_domains":34.26,"v":46.57,"lab":false,"jev":false,"rank_v":104,"rank_pub":116,"name":"Winnow-12B","eq":{"S":47.64,"O":44.95},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":28.59,"new_domains":17.98}}},{"engine":"jevfire-uncapped","name":"Jevfire","long_name":"Jevfire Qwen3.8-27B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/kikoncuo/jevfire","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B-FP8"},"completed":110201,"counts":{"ok":110201},"lifted":true,"calibration":{"acc":0.6894,"conf":0.7415,"ece":0.0521,"brier":0.4139,"over95":0.0152,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0169,"conf":0.171,"acc":0.0521},{"lo":0.2,"w":0.0335,"conf":0.2552,"acc":0.1665},{"lo":0.3,"w":0.045,"conf":0.3573,"acc":0.2594},{"lo":0.4,"w":0.0687,"conf":0.4518,"acc":0.4308},{"lo":0.5,"w":0.1472,"conf":0.5496,"acc":0.5063},{"lo":0.6,"w":0.1004,"conf":0.6485,"acc":0.5991},{"lo":0.7,"w":0.1045,"conf":0.7494,"acc":0.7231},{"lo":0.8,"w":0.105,"conf":0.8542,"acc":0.7528},{"lo":0.9,"w":0.3788,"conf":0.9739,"acc":0.9302}],"areas":{"knowledge":{"ece":0.0643,"acc":0.6084,"conf":0.6726},"language":{"ece":0.031,"acc":0.7762,"conf":0.8035},"retrieval":{"ece":0.0794,"acc":0.7266,"conf":0.804},"tools":{"ece":0.0466,"acc":0.9297,"conf":0.9266},"arts":{"ece":0.0845,"acc":0.5374,"conf":0.6201}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":89.6,"p95":447.5,"mean":178.1},"results":{"1":{"score":0.9557,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":154.7},"2":{"score":0.649,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1575.7},"3":{"score":0.8406,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7517,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":117.7},"5":{"score":0.8128,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":292.3},"6":{"score":0.7995,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":305.6},"9":{"score":0.4773,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6497,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":134.1},"11":{"score":0.7547,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6355,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":88.3},"20":{"score":0.9492,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":88},"21":{"score":0.5784,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":87.7},"22":{"score":0.132,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":762.1},"23":{"score":0.6046,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":90},"24":{"score":0.7971,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":88.2},"25":{"score":0.4694,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9848,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":87.5},"27":{"score":0.9599,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":87.8},"28":{"score":0.7451,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":87.8},"29":{"score":0.9046,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":88.6},"30":{"score":0.3393,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1732,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.9},"32":{"score":0.5944,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2752,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":490.7},"36":{"score":0.4922,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5351,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":89.1},"38":{"score":0.2238,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1455.5},"39":{"score":0.8032,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5324,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":88,"tracks":[{"track":"A · Arabic","score":0.3969,"headline":false},{"track":"A · English","score":0.5324,"headline":true},{"track":"C · Arabic pairs","score":0.72,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6081,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":88.3},"42":{"score":0.7522,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":90.5},"43":{"score":0.5368,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6178,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":88},"45":{"score":0.1198,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.395,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":108},"56":{"score":0.7435,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":361.6,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.5727,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":88.5,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6797,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":88.4,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.8026,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":99.1,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.7117,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":89.9,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7755,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":97.2,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":62.4,"balanced_skill":44.39,"breadth_skill":47.88,"public_skill":49.8},"frozen_scores":{"balanced_raw":62.4,"balanced_skill":49.8,"breadth_skill":47.88},"categories":[{"id":"knowledge","raw":0.4701,"skill":0.3025,"combined":0.2452,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6729,"skill":0.539,"combined":0.4466,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6755,"skill":0.5622,"combined":0.537,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7514,"skill":0.7233,"combined":0.722,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5592,"skill":0.3568,"combined":0.3758,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9557,"skill":0.9402,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9329,"private_eq":0.9501},"2":{"raw":0.649,"skill":0.5947,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6843,"private_eq":0.7289},"3":{"raw":0.8406,"skill":0.8375,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8278,"private_eq":0.832},"4":{"raw":0.7517,"skill":0.7485,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8134,"private_eq":0.8169},"5":{"raw":0.8128,"skill":0.8117,"coverage":1,"random":0.006,"tracks":[],"combined":0.6039,"private_eq":0.6098},"6":{"raw":0.7996,"skill":0.5273,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7866,"headline":false},{"track":"RouterBench-5shot","score":0.8125,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.4773,"skill":0.4773,"coverage":1,"random":0,"tracks":[],"combined":0.4219,"private_eq":0.4275},"11":{"raw":0.7547,"skill":0.6453,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6355,"skill":0.454,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5003,"private_eq":0.6695},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.9508,"skill":0.9016,"coverage":1,"random":0.5,"tracks":[],"combined":0.8926,"private_eq":0.9464},"21":{"raw":0.5784,"skill":0.1568,"coverage":1,"random":0.5,"tracks":[],"combined":0.0856,"private_eq":0.5475},"22":{"raw":0.1287,"skill":0.1219,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1162,"private_eq":0.1321},"23":{"raw":0.6146,"skill":0.2292,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8022,"skill":0.7363,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4694,"skill":0.2925,"coverage":1,"random":0.25,"tracks":[],"combined":0.2401,"private_eq":0.4358},"28":{"raw":0.7451,"skill":0.4902,"coverage":1,"random":0.5,"tracks":[],"combined":0.1485,"private_eq":0.5786},"29":{"raw":0.9046,"skill":0.8728,"coverage":1,"random":0.25,"tracks":[],"combined":0.9138,"private_eq":0.9353},"30":{"raw":0.3393,"skill":0.204,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4435,"random":0.25},{"track":"10-choice","raw":0.235,"random":0.1}],"combined":0.4368,"private_eq":0.5398},"31":{"raw":0.1732,"skill":0.0995,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.5944,"skill":0.3552,"coverage":1,"random":0.371,"tracks":[],"combined":0.2351,"private_eq":0.5237},"33":{"raw":0.2752,"skill":0.2656,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2509,"private_eq":0.2681},"36":{"raw":0.4922,"skill":0.4256,"coverage":1,"random":0.116,"tracks":[],"combined":0.3379,"private_eq":0.4205},"37":{"raw":0.5351,"skill":0.4169,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4405,"private_eq":0.5582},"38":{"raw":0.2238,"skill":0.199,"coverage":1,"random":0.031,"tracks":[],"combined":0.2496,"private_eq":0.2802},"39":{"raw":0.8032,"skill":0.7105,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5587,"private_eq":0.7027},"40":{"raw":0.5324,"skill":0.3985,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3969,"headline":false},{"track":"A · English","score":0.5324,"headline":true},{"track":"C · Arabic pairs","score":0.72,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"combined":0.2936,"private_eq":0.4564},"41":{"raw":0.6081,"skill":0.4121,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4371,"private_eq":0.6283},"42":{"raw":0.7522,"skill":0.5179,"coverage":1,"random":0.486,"tracks":[],"combined":0.2319,"private_eq":0.6092},"43":{"raw":0.5368,"skill":0.2652,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3572,"private_eq":0.5988},"44":{"raw":0.6178,"skill":0.2356,"coverage":1,"random":0.5,"tracks":[],"combined":0.278,"private_eq":0.6426},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.395,"skill":0.1219,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.7435,"skill":0.487,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4558,"private_eq":0.7305},"57":{"raw":0.5727,"skill":0.5194,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2766,"private_eq":0.3633},"58":{"raw":0.6797,"skill":0.5357,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1739,"private_eq":0.4359},"59":{"raw":0.8026,"skill":0.5907,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.2837,"private_eq":0.658},"61":{"raw":0.7117,"skill":0.4234,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5414,"private_eq":0.7728},"62":{"raw":0.7755,"skill":0.7007,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.6794,"private_eq":0.7615},"64":{"raw":0.6875,"skill":0.6094,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.6034,"private_eq":0.6855}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine over vLLM 0.29.0 OpenAI server, Qwen3.8-27B-FP8, fp8 KV, prefix caching, CUDA graphs, max-model-len 262144 (board lift), gpu util 0.9","v03":{"engine":"jevfire-uncapped","pub":49.8,"same_skills":46.86,"new_domains":31.43,"v":44.39,"lab":false,"jev":false,"rank_v":114,"rank_pub":120,"name":"Jevfire Qwen3.8-27B","eq":{"S":46.35,"O":41.46},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":27.81,"new_domains":16.58}}},{"engine":"decision-2.0-lux-9b","name":"vLLM-SR Decision 2.0 Lux 9B","long_name":"vLLM-SR Decision 2.0 Lux 9B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":9650000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/Decision-2.0-Lux-9B","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.6775,"conf":0.7225,"ece":0.056,"brier":0.4216,"over95":0.0132,"n":34833,"benchmarks":32,"rel":[{"lo":0,"w":0.0504,"conf":0.0514,"acc":0.1607},{"lo":0.1,"w":0.0126,"conf":0.1456,"acc":0.1227},{"lo":0.2,"w":0.015,"conf":0.2586,"acc":0.1948},{"lo":0.3,"w":0.0463,"conf":0.3521,"acc":0.3289},{"lo":0.4,"w":0.0614,"conf":0.4527,"acc":0.3652},{"lo":0.5,"w":0.1094,"conf":0.5498,"acc":0.5261},{"lo":0.6,"w":0.1071,"conf":0.65,"acc":0.5976},{"lo":0.7,"w":0.1095,"conf":0.7492,"acc":0.6675},{"lo":0.8,"w":0.127,"conf":0.8525,"acc":0.7628},{"lo":0.9,"w":0.3613,"conf":0.9687,"acc":0.9292}],"areas":{"knowledge":{"ece":0.0598,"acc":0.5982,"conf":0.6572},"language":{"ece":0.0586,"acc":0.759,"conf":0.81},"retrieval":{"ece":0.0587,"acc":0.7463,"conf":0.7936},"tools":{"ece":0.065,"acc":0.869,"conf":0.9136},"arts":{"ece":0.0806,"acc":0.5352,"conf":0.5369}}},"gaps":null,"latency":{"median":37.1,"p95":null,"mean":74.2},"results":{"1":{"score":0.9398,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":69.9},"2":{"score":0.6645,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":732.6},"3":{"score":0.7126,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":260.1},"4":{"score":0.8403,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":83.8},"5":{"score":0.8989,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":136.9},"6":{"score":0.7871,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":167.4},"9":{"score":0.2727,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":635.9},"10":{"score":0.6422,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":55.1},"11":{"score":0.7615,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1165.1},"12":{"score":0.6464,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":30.8},"20":{"score":0.8082,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.8},"21":{"score":0.6161,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":31.2},"22":{"score":0.2286,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":256.3},"23":{"score":0.6272,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.1},"24":{"score":0.7569,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":29.9},"25":{"score":0.4082,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":32.3},"26":{"score":0.9815,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":29.5},"27":{"score":0.9437,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":29.1},"28":{"score":0.8745,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":30.7},"29":{"score":0.8849,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":33.4},"30":{"score":0.5944,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.124,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":59.5},"32":{"score":0.5678,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":56.3},"33":{"score":0.3255,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":137.2},"36":{"score":0.4438,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1283},"37":{"score":0.4461,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.2},"38":{"score":0.1144,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":379.8},"39":{"score":0.8273,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":36.8},"40":{"score":0.4309,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":33.8},"41":{"score":0.4466,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":32.1},"42":{"score":0.8167,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":40.9},"43":{"score":0.5895,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":32.1},"44":{"score":0.6636,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32},"45":{"score":0.0758,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":35},"50":{"score":0.3944,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":52.8},"56":{"score":0.5975,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":101.2},"57":{"score":0.5229,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":34.4},"58":{"score":0.7024,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33.2},"59":{"score":0.6408,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":49.5},"61":{"score":0.822,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":38.5},"62":{"score":0.5687,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":46.4},"64":{"score":0.6515,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":32.5}},"benchmarks":{"1":{"raw":0.9398,"skill":0.9187,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8921,"private_eq":0.9202},"2":{"raw":0.6635,"skill":0.6114,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.696,"private_eq":0.7389},"3":{"raw":0.7126,"skill":0.7071,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7753,"private_eq":0.7811},"4":{"raw":0.8403,"skill":0.8382,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8292,"private_eq":0.8323},"5":{"raw":0.8989,"skill":0.8983,"coverage":1,"random":0.006,"tracks":[],"combined":0.745,"private_eq":0.7484},"6":{"raw":0.7871,"skill":0.5008,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2727,"skill":0.2727,"coverage":1,"random":0,"tracks":[],"combined":0.298,"private_eq":0.305},"10":{"raw":0.6422,"skill":0.4047,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7615,"skill":0.6551,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6464,"skill":0.4703,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3493,"private_eq":0.5699},"20":{"raw":0.8119,"skill":0.6239,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.6161,"skill":0.2321,"coverage":1,"random":0.5,"tracks":[],"combined":0.1258,"private_eq":0.5674},"22":{"raw":0.219,"skill":0.2129,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.1732,"private_eq":0.188},"23":{"raw":0.6152,"skill":0.2303,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7569,"skill":0.6759,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4082,"skill":0.2109,"coverage":1,"random":0.25,"tracks":[],"combined":0.1517,"private_eq":0.3703},"26":{"raw":0.9815,"skill":0.9753,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9437,"skill":0.9249,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8745,"skill":0.749,"coverage":1,"random":0.5,"tracks":[],"combined":0.4815,"private_eq":0.7432},"29":{"raw":0.8849,"skill":0.8465,"coverage":1,"random":0.25,"tracks":[],"combined":0.7999,"private_eq":0.8509},"30":{"raw":0.5944,"skill":0.5134,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6763,"random":0.25},{"track":"10-choice","raw":0.5125,"random":0.1}],"combined":0.6464,"private_eq":0.7108},"31":{"raw":0.124,"skill":0.0459,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0442,"private_eq":0.1316},"32":{"raw":0.5678,"skill":0.3129,"coverage":1,"random":0.371,"tracks":[],"combined":0.0881,"private_eq":0.4323},"33":{"raw":0.3255,"skill":0.3165,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3126,"private_eq":0.3283},"36":{"raw":0.4438,"skill":0.3708,"coverage":1,"random":0.116,"tracks":[],"combined":0.3479,"private_eq":0.4292},"37":{"raw":0.4461,"skill":0.3053,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3764,"private_eq":0.5077},"38":{"raw":0.1144,"skill":0.0861,"coverage":1,"random":0.031,"tracks":[],"combined":0.1713,"private_eq":0.2052},"39":{"raw":0.8273,"skill":0.746,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5828,"private_eq":0.7189},"40":{"raw":0.4309,"skill":0.2679,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4391,"headline":false},{"track":"A · English","score":0.4309,"headline":true},{"track":"C · Arabic pairs","score":0.885,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"combined":0.2513,"private_eq":0.4239},"41":{"raw":0.4466,"skill":0.1699,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5148,"private_eq":0.6795},"42":{"raw":0.8167,"skill":0.6434,"coverage":1,"random":0.486,"tracks":[],"combined":0.8068,"private_eq":0.9013},"43":{"raw":0.5895,"skill":0.3487,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4616,"private_eq":0.6639},"44":{"raw":0.6636,"skill":0.3272,"coverage":1,"random":0.5,"tracks":[],"combined":0.3745,"private_eq":0.6903},"45":{"raw":0.0758,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3944,"skill":0.1211,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.5975,"skill":0.195,"coverage":1,"random":0.5,"tracks":[],"combined":0.2915,"private_eq":0.6493},"57":{"raw":0.5229,"skill":0.4634,"coverage":1,"random":0.1109,"tracks":[],"combined":0.3021,"private_eq":0.3857},"58":{"raw":0.7024,"skill":0.5686,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1807,"private_eq":0.4405},"59":{"raw":0.6408,"skill":0.2552,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1648,"private_eq":0.6013},"61":{"raw":0.822,"skill":0.644,"coverage":1,"random":0.5,"tracks":[],"combined":0.6096,"private_eq":0.8065},"62":{"raw":0.5687,"skill":0.4249,"coverage":1,"random":0.25,"tracks":[],"combined":0.5151,"private_eq":0.6397},"64":{"raw":0.6515,"skill":0.5644,"coverage":1,"random":0.2,"tracks":[],"combined":0.5355,"private_eq":0.6318}},"coverage":0.9999,"pending":0.0001,"scores":{"balanced_raw":59.93,"balanced_skill":45.97,"breadth_skill":45.87,"public_skill":46.76},"frozen_scores":{"balanced_raw":59.93,"balanced_skill":46.76,"breadth_skill":45.87},"categories":[{"id":"knowledge","raw":0.4971,"skill":0.3546,"combined":0.3025,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6251,"skill":0.4685,"combined":0.4608,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6796,"skill":0.5565,"combined":0.543,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6459,"skill":0.6037,"combined":0.65,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.5513,"skill":0.3308,"combined":0.3759,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"decision-2.0-lux-9b","pub":46.76,"same_skills":47.35,"new_domains":34.02,"v":45.97,"lab":false,"jev":false,"rank_v":106,"rank_pub":133,"name":"vLLM-SR Decision 2.0 Lux 9B","eq":{"S":46.85,"O":44.66},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":28.11,"new_domains":17.87}}},{"engine":"nimble-v3","name":"Bespoke Nimble 9B v3","long_name":"Bespoke Nimble 9B v3","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9650000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/bespokelabs/Bespoke-Nimble-9B-v3","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"bespokelabs","name":"Bespoke Labs","logo":"data/orgs/bespokelabs.png"}},"completed":110201,"counts":{"ok":110192,"unsupported":9,"error":0},"lifted":false,"calibration":{"acc":0.7497,"conf":0.8726,"ece":0.1229,"brier":0.3705,"over95":0.0523,"n":34833,"benchmarks":32,"rel":[{"lo":0,"w":0.0007,"conf":0.091,"acc":0.0526},{"lo":0.1,"w":0.0087,"conf":0.1571,"acc":0.1069},{"lo":0.2,"w":0.0094,"conf":0.2517,"acc":0.1304},{"lo":0.3,"w":0.0128,"conf":0.359,"acc":0.2087},{"lo":0.4,"w":0.0316,"conf":0.4609,"acc":0.2825},{"lo":0.5,"w":0.0651,"conf":0.5511,"acc":0.451},{"lo":0.6,"w":0.0623,"conf":0.6512,"acc":0.493},{"lo":0.7,"w":0.0618,"conf":0.7519,"acc":0.5365},{"lo":0.8,"w":0.0774,"conf":0.8529,"acc":0.5906},{"lo":0.9,"w":0.6702,"conf":0.9859,"acc":0.8908}],"areas":{"knowledge":{"ece":0.1631,"acc":0.658,"conf":0.8212},"language":{"ece":0.0741,"acc":0.8475,"conf":0.9215},"retrieval":{"ece":0.0819,"acc":0.7728,"conf":0.8547},"tools":{"ece":0.0395,"acc":0.9307,"conf":0.9689},"arts":{"ece":0.207,"acc":0.6485,"conf":0.8543}}},"gaps":null,"latency":{"median":73.9,"p95":null,"mean":251.2},"results":{"1":{"score":0.9746,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":214.2},"2":{"score":0.6852,"jev":null,"metric":"nDCG@10","requests":685,"answered":682,"unsupported":3,"errors":0,"median_ms":2001.9},"3":{"score":0.7913,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":887},"4":{"score":0.8365,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":223.2},"5":{"score":0.8831,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":386},"6":{"score":0.7655,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":402.9},"9":{"score":0.9886,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":605.7},"10":{"score":0.7179,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":158.8},"11":{"score":0.7409,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":695.6},"12":{"score":0.7027,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":83.3},"20":{"score":0.905,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77.1},"21":{"score":0.6164,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":80.5},"22":{"score":0.746,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":836.7},"23":{"score":0.6402,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.2},"24":{"score":0.7909,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":81.2},"25":{"score":0.4643,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":83.3},"26":{"score":0.9823,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":78.9},"27":{"score":0.9556,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":78.9},"28":{"score":0.8642,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":78.1},"29":{"score":0.8876,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":82.6},"30":{"score":0.5898,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1438,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":161.4},"32":{"score":0.5878,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":134.4},"33":{"score":0.2564,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":270.5},"36":{"score":0.3884,"jev":null,"metric":"nDCG@10","requests":220,"answered":211,"unsupported":9,"errors":0,"median_ms":2478.3},"37":{"score":0.5827,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.9},"38":{"score":0.3904,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2404.2},"39":{"score":0.9037,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":156.2},"40":{"score":0.535,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":90.9},"41":{"score":0.7312,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":83.9},"42":{"score":0.8241,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":99.5},"43":{"score":0.6474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":84.4},"44":{"score":0.7502,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.2},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":93.3},"50":{"score":0.3616,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":124.9},"56":{"score":0.5395,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":220.6},"57":{"score":0.6005,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":85.9},"58":{"score":0.8449,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":84.3},"59":{"score":0.7624,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":120.2},"61":{"score":0.8488,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":95.7},"62":{"score":0.868,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":121.5},"64":{"score":0.6155,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":87.4}},"benchmarks":{"1":{"raw":0.9746,"skill":0.9657,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.6822,"skill":0.633,"coverage":0.9956,"random":0.1341,"tracks":[],"combined":0.6856,"private_eq":0.73},"3":{"raw":0.7913,"skill":0.7873,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6389,"private_eq":0.6488},"4":{"raw":0.8365,"skill":0.8344,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8371,"private_eq":0.84},"5":{"raw":0.8831,"skill":0.8824,"coverage":1,"random":0.006,"tracks":[],"combined":0.7068,"private_eq":0.7109},"6":{"raw":0.7655,"skill":0.4502,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9886,"skill":0.9886,"coverage":1,"random":0,"tracks":[],"combined":0.8291,"private_eq":0.83},"10":{"raw":0.7179,"skill":0.5306,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7409,"skill":0.6252,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7027,"skill":0.5547,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4715,"private_eq":0.6505},"20":{"raw":0.9071,"skill":0.8142,"coverage":1,"random":0.5,"tracks":[],"combined":0.8493,"private_eq":0.925},"21":{"raw":0.6164,"skill":0.2329,"coverage":1,"random":0.5,"tracks":[],"combined":0.2365,"private_eq":0.6221},"22":{"raw":0.741,"skill":0.739,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5336,"private_eq":0.5415},"23":{"raw":0.6321,"skill":0.2641,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7909,"skill":0.7212,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4643,"skill":0.2857,"coverage":1,"random":0.25,"tracks":[],"combined":0.198,"private_eq":0.4046},"26":{"raw":0.9823,"skill":0.9764,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9556,"skill":0.9408,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8642,"skill":0.7284,"coverage":1,"random":0.5,"tracks":[],"combined":0.4562,"private_eq":0.7307},"29":{"raw":0.8876,"skill":0.8501,"coverage":1,"random":0.25,"tracks":[],"combined":0.8544,"private_eq":0.8913},"30":{"raw":0.5898,"skill":0.5076,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6702,"random":0.25},{"track":"10-choice","raw":0.5095,"random":0.1}],"combined":0.6105,"private_eq":0.6815},"31":{"raw":0.1438,"skill":0.0675,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0606,"private_eq":0.1464},"32":{"raw":0.5878,"skill":0.3447,"coverage":1,"random":0.371,"tracks":[],"combined":0.2273,"private_eq":0.5189},"33":{"raw":0.2564,"skill":0.2465,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1738,"private_eq":0.1929},"36":{"raw":0.3725,"skill":0.2902,"coverage":0.9591,"random":0.116,"tracks":[],"combined":0.37,"private_eq":0.4485},"37":{"raw":0.5827,"skill":0.4765,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5292,"private_eq":0.6281},"38":{"raw":0.3904,"skill":0.3709,"coverage":1,"random":0.031,"tracks":[],"combined":0.4075,"private_eq":0.4314},"39":{"raw":0.9037,"skill":0.8584,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5746,"private_eq":0.7134},"40":{"raw":0.535,"skill":0.4017,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4255,"headline":false},{"track":"A · English","score":0.535,"headline":true},{"track":"C · Arabic pairs","score":0.915,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0.3385,"private_eq":0.4909},"41":{"raw":0.7312,"skill":0.5968,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5706,"private_eq":0.7163},"42":{"raw":0.8241,"skill":0.6578,"coverage":1,"random":0.486,"tracks":[],"combined":0.7298,"private_eq":0.8622},"43":{"raw":0.6474,"skill":0.4405,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4616,"private_eq":0.6639},"44":{"raw":0.7502,"skill":0.5004,"coverage":1,"random":0.5,"tracks":[],"combined":0.5139,"private_eq":0.7592},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3616,"skill":0.0734,"coverage":1,"random":0.311,"tracks":[],"combined":0.2322,"private_eq":0.4763},"56":{"raw":0.5395,"skill":0.079,"coverage":1,"random":0.5,"tracks":[],"combined":0.1443,"private_eq":0.5765},"57":{"raw":0.6005,"skill":0.5507,"coverage":1,"random":0.1109,"tracks":[],"combined":0.4154,"private_eq":0.4853},"58":{"raw":0.8449,"skill":0.7752,"coverage":1,"random":0.3101,"tracks":[],"combined":0.183,"private_eq":0.4421},"59":{"raw":0.7624,"skill":0.5074,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3737,"private_eq":0.7009},"61":{"raw":0.8488,"skill":0.6976,"coverage":1,"random":0.5,"tracks":[],"combined":0.5187,"private_eq":0.7616},"62":{"raw":0.868,"skill":0.824,"coverage":1,"random":0.25,"tracks":[],"combined":0.5903,"private_eq":0.6955},"64":{"raw":0.6155,"skill":0.5194,"coverage":1,"random":0.2,"tracks":[],"combined":0.5064,"private_eq":0.6088}},"coverage":0.9983,"pending":0.0017,"scores":{"balanced_raw":68.38,"balanced_skill":51.06,"breadth_skill":55.66,"public_skill":57.19},"frozen_scores":{"balanced_raw":68.38,"balanced_skill":57.19,"breadth_skill":55.66},"categories":[{"id":"knowledge","raw":0.5367,"skill":0.4096,"combined":0.3288,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.723,"skill":0.6068,"combined":0.5459,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.679,"skill":0.5548,"combined":0.5286,"coverage":0.9926,"pending":0,"n":6},{"id":"tools","raw":0.8626,"skill":0.8424,"combined":0.7376,"coverage":0.9992,"pending":0,"n":5},{"id":"arts","raw":0.6456,"skill":0.4405,"combined":0.4716,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"nimble-v3","pub":57.19,"same_skills":52.34,"new_domains":38.2,"v":51.06,"lab":false,"jev":false,"rank_v":90,"rank_pub":89,"name":"Bespoke Nimble 9B v3","eq":{"S":51.89,"O":49.82},"rank_tie":20,"tied":true,"parts":{"public":0,"same_skills":31.13,"new_domains":19.93}}},{"engine":"jevk5-9b-v0.3.3","name":"JevK5 9B v0.3.3","long_name":"JevK5 9B v0.3.3","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9650000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/alibiserikbay/JevK5-9B","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6949,"conf":0.7547,"ece":0.0598,"brier":0.414,"over95":0.0151,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0052,"conf":0.0865,"acc":0.0146},{"lo":0.1,"w":0.0272,"conf":0.1492,"acc":0.0747},{"lo":0.2,"w":0.0236,"conf":0.2485,"acc":0.1379},{"lo":0.3,"w":0.0387,"conf":0.3499,"acc":0.2508},{"lo":0.4,"w":0.0503,"conf":0.4548,"acc":0.3828},{"lo":0.5,"w":0.1045,"conf":0.5495,"acc":0.5178},{"lo":0.6,"w":0.0973,"conf":0.649,"acc":0.6016},{"lo":0.7,"w":0.1148,"conf":0.7495,"acc":0.6633},{"lo":0.8,"w":0.1558,"conf":0.8551,"acc":0.7837},{"lo":0.9,"w":0.3824,"conf":0.9622,"acc":0.9143}],"areas":{"knowledge":{"ece":0.1011,"acc":0.6157,"conf":0.7169},"language":{"ece":0.0164,"acc":0.8105,"conf":0.8192},"retrieval":{"ece":0.0622,"acc":0.7181,"conf":0.7706},"tools":{"ece":0.0286,"acc":0.8655,"conf":0.8592},"arts":{"ece":0.1061,"acc":0.5404,"conf":0.6465}}},"gaps":null,"latency":{"median":40.4,"p95":null,"mean":180.6},"results":{"1":{"score":0.9333,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":108.4},"2":{"score":0.6282,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":854.7},"3":{"score":0.7146,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1754.7},"4":{"score":0.7509,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":164.5},"5":{"score":0.8145,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":268.8},"6":{"score":0.7905,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":312.2},"9":{"score":0.1818,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1412.3},"10":{"score":0.6664,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":79},"11":{"score":0.7465,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1790},"12":{"score":0.667,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":17.5},"20":{"score":0.8876,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.5},"21":{"score":0.6016,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.2},"22":{"score":0.079,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1560.1},"23":{"score":0.6564,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36},"24":{"score":0.7975,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.5},"25":{"score":0.4439,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":21.5},"26":{"score":0.9832,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":15.3},"27":{"score":0.9573,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":15.4},"28":{"score":0.8224,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":13.1},"29":{"score":0.9331,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":23.5},"30":{"score":0.4788,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1398,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":124.9},"32":{"score":0.5199,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":79.3},"33":{"score":0.3012,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":241.9},"36":{"score":0.4328,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1189.7},"37":{"score":0.4009,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.1},"38":{"score":0.1941,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1010.6},"39":{"score":0.8802,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":35.3},"40":{"score":0.5909,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":13.2},"41":{"score":0.4874,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":17.7},"42":{"score":0.789,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":36.4},"43":{"score":0.514,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":17.6},"44":{"score":0.6424,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.5},"45":{"score":0.1257,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":23.8},"50":{"score":0.4397,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":53.2},"56":{"score":0.6295,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":189.9},"57":{"score":0.5804,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":23.3},"58":{"score":0.6541,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":17.6},"59":{"score":0.771,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":52.6},"61":{"score":0.744,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":30.4},"62":{"score":0.8264,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":52.6},"64":{"score":0.6894,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":21.4}},"benchmarks":{"1":{"raw":0.9333,"skill":0.9099,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8581,"private_eq":0.8953},"2":{"raw":0.6282,"skill":0.5707,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6448,"private_eq":0.6951},"3":{"raw":0.7146,"skill":0.7091,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7123,"private_eq":0.72},"4":{"raw":0.7509,"skill":0.7477,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8086,"private_eq":0.8122},"5":{"raw":0.8145,"skill":0.8134,"coverage":1,"random":0.006,"tracks":[],"combined":0.6328,"private_eq":0.6382},"6":{"raw":0.7905,"skill":0.5088,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1818,"skill":0.1818,"coverage":1,"random":0,"tracks":[],"combined":0.1311,"private_eq":0.14},"10":{"raw":0.6664,"skill":0.4449,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7465,"skill":0.6334,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.667,"skill":0.5012,"coverage":1,"random":0.3324,"tracks":[],"combined":0.504,"private_eq":0.672},"20":{"raw":0.8882,"skill":0.7764,"coverage":1,"random":0.5,"tracks":[],"combined":0.7552,"private_eq":0.8785},"21":{"raw":0.6016,"skill":0.2032,"coverage":1,"random":0.5,"tracks":[],"combined":0.1157,"private_eq":0.5624},"22":{"raw":0.0752,"skill":0.068,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0785,"private_eq":0.0951},"23":{"raw":0.6478,"skill":0.2956,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7975,"skill":0.73,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[],"combined":0.1728,"private_eq":0.3859},"26":{"raw":0.9832,"skill":0.9776,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9573,"skill":0.9431,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8224,"skill":0.6448,"coverage":1,"random":0.5,"tracks":[],"combined":0.4261,"private_eq":0.7158},"29":{"raw":0.9331,"skill":0.9108,"coverage":1,"random":0.25,"tracks":[],"combined":0.9434,"private_eq":0.9573},"30":{"raw":0.4788,"skill":0.3758,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5944,"random":0.25},{"track":"10-choice","raw":0.3632,"random":0.1}],"combined":0.5531,"private_eq":0.6347},"31":{"raw":0.1398,"skill":0.0631,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.5199,"skill":0.2367,"coverage":1,"random":0.371,"tracks":[],"combined":0.2505,"private_eq":0.5333},"33":{"raw":0.3012,"skill":0.2919,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3126,"private_eq":0.3283},"36":{"raw":0.4328,"skill":0.3584,"coverage":1,"random":0.116,"tracks":[],"combined":0.369,"private_eq":0.4477},"37":{"raw":0.4009,"skill":0.2486,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4264,"private_eq":0.5471},"38":{"raw":0.1941,"skill":0.1683,"coverage":1,"random":0.031,"tracks":[],"combined":0.2113,"private_eq":0.2435},"39":{"raw":0.8802,"skill":0.8238,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5493,"private_eq":0.6964},"40":{"raw":0.5909,"skill":0.4737,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1404,"headline":false},{"track":"A · English","score":0.5909,"headline":true},{"track":"C · Arabic pairs","score":0.72,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"combined":0.4426,"private_eq":0.5709},"41":{"raw":0.4874,"skill":0.2312,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4724,"private_eq":0.6516},"42":{"raw":0.789,"skill":0.5895,"coverage":1,"random":0.486,"tracks":[],"combined":0.7249,"private_eq":0.8597},"43":{"raw":0.514,"skill":0.229,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4616,"private_eq":0.6639},"44":{"raw":0.6424,"skill":0.2848,"coverage":1,"random":0.5,"tracks":[],"combined":0.2567,"private_eq":0.6321},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4397,"skill":0.1869,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.6295,"skill":0.259,"coverage":1,"random":0.5,"tracks":[],"combined":0.3174,"private_eq":0.6621},"57":{"raw":0.5804,"skill":0.5281,"coverage":1,"random":0.1109,"tracks":[],"combined":0.5222,"private_eq":0.5791},"58":{"raw":0.6541,"skill":0.4986,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1704,"private_eq":0.4335},"59":{"raw":0.771,"skill":0.5252,"coverage":1,"random":0.5177,"tracks":[],"combined":0.372,"private_eq":0.7001},"61":{"raw":0.744,"skill":0.488,"coverage":1,"random":0.5,"tracks":[],"combined":0.5187,"private_eq":0.7616},"62":{"raw":0.8264,"skill":0.7685,"coverage":1,"random":0.25,"tracks":[],"combined":0.6246,"private_eq":0.7209},"64":{"raw":0.6894,"skill":0.6118,"coverage":1,"random":0.2,"tracks":[],"combined":0.6228,"private_eq":0.7009}},"coverage":1,"pending":0,"scores":{"balanced_raw":60.38,"balanced_skill":44.89,"breadth_skill":46.33,"public_skill":47.62},"frozen_scores":{"balanced_raw":60.38,"balanced_skill":47.62,"breadth_skill":46.33},"categories":[{"id":"knowledge","raw":0.479,"skill":0.3164,"combined":0.322,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6786,"skill":0.5468,"combined":0.5369,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6322,"skill":0.4998,"combined":0.5205,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6692,"skill":0.6414,"combined":0.6083,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.557,"skill":0.357,"combined":0.3507,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jevk5-9b-v0.3.3","pub":47.62,"same_skills":48.19,"new_domains":30.79,"v":44.89,"lab":false,"jev":false,"rank_v":112,"rank_pub":130,"name":"JevK5 9B v0.3.3","eq":{"S":47.69,"O":40.67},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":28.62,"new_domains":16.27}}},{"engine":"jpt-9b","name":"JPT-9B","long_name":"JPT-9B Qwen3.5-9B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":18819728184,"model_url":"https://huggingface.co/kirp/jpt-9b","code_url":"https://github.com/tic-top/llm2jev","base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":"kirp/jpt-9b","served_checkpoint":"7114b0c3d9bea6b82dfa2d0691e8d5562cd26d4e"},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.671,"conf":0.7482,"ece":0.0773,"brier":0.4346,"over95":0.0176,"n":35093,"benchmarks":32,"rel":[{"lo":0,"w":0.0283,"conf":0.0775,"acc":0.045},{"lo":0.1,"w":0.0278,"conf":0.1411,"acc":0.116},{"lo":0.2,"w":0.0188,"conf":0.255,"acc":0.1374},{"lo":0.3,"w":0.0319,"conf":0.3484,"acc":0.3028},{"lo":0.4,"w":0.0543,"conf":0.4553,"acc":0.3276},{"lo":0.5,"w":0.0939,"conf":0.5475,"acc":0.4734},{"lo":0.6,"w":0.0999,"conf":0.6509,"acc":0.5436},{"lo":0.7,"w":0.1056,"conf":0.755,"acc":0.6422},{"lo":0.8,"w":0.1439,"conf":0.8559,"acc":0.7612},{"lo":0.9,"w":0.3956,"conf":0.966,"acc":0.9107}],"areas":{"knowledge":{"ece":0.0973,"acc":0.6052,"conf":0.7017},"language":{"ece":0.0282,"acc":0.8225,"conf":0.8469},"retrieval":{"ece":0.185,"acc":0.561,"conf":0.745},"tools":{"ece":0.0348,"acc":0.8767,"conf":0.8991},"arts":{"ece":0.0776,"acc":0.5074,"conf":0.585}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":148.6,"p95":461.9,"mean":178.2},"results":{"1":{"score":0.9486,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":358.3},"2":{"score":0.6172,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":783.3},"3":{"score":0.7874,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6543,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":178.6},"5":{"score":0.362,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":202},"6":{"score":0.8007,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":417.1},"9":{"score":0.4318,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5299,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":194.1},"11":{"score":0.822,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6659,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":141.2},"20":{"score":0.8608,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":68.9},"21":{"score":0.5974,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":136.7},"22":{"score":0.047,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":450},"23":{"score":0.5752,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":166},"24":{"score":0.7794,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":69},"25":{"score":0.4133,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9781,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":68},"27":{"score":0.9471,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":65.2},"28":{"score":0.7459,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":65.2},"29":{"score":0.859,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":142.3},"30":{"score":0.4886,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.149,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":177.2},"32":{"score":0.5479,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2358,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":421.5},"36":{"score":0.469,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5264,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":153.4},"38":{"score":0.2278,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":798.5},"39":{"score":0.9317,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5639,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":140.1,"tracks":[{"track":"A · Arabic","score":0.2031,"headline":false},{"track":"A · English","score":0.5639,"headline":true},{"track":"C · Arabic pairs","score":0.67,"headline":false},{"track":"C · English pairs","score":0.925,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6443,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":71.6},"42":{"score":0.7899,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":168.1},"43":{"score":0.5561,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.669,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":66.8},"45":{"score":0.1218,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4021,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":179},"56":{"score":0.7605,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":381.7,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.5472,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":109.2,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.694,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":141,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7619,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":168.7,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6635,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":155.9,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.6796,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":180.8,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6212,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":59.66,"balanced_skill":44.08,"breadth_skill":45.16,"public_skill":46.96},"frozen_scores":{"balanced_raw":59.66,"balanced_skill":46.96,"breadth_skill":45.16},"categories":[{"id":"knowledge","raw":0.4728,"skill":0.3092,"combined":0.2906,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6991,"skill":0.5768,"combined":0.465,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5656,"skill":0.4459,"combined":0.423,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7059,"skill":0.6702,"combined":0.6104,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5137,"skill":0.2878,"combined":0.3321,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9486,"skill":0.9307,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8512,"private_eq":0.8903},"2":{"raw":0.6172,"skill":0.5579,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6783,"private_eq":0.7237},"3":{"raw":0.7874,"skill":0.7833,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6599,"private_eq":0.6691},"4":{"raw":0.6543,"skill":0.6499,"coverage":1,"random":0.0127,"tracks":[],"combined":0.515,"private_eq":0.5256},"5":{"raw":0.362,"skill":0.3581,"coverage":1,"random":0.006,"tracks":[],"combined":0.2413,"private_eq":0.2534},"6":{"raw":0.8007,"skill":0.5299,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7881,"headline":false},{"track":"RouterBench-5shot","score":0.8133,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.4318,"skill":0.4318,"coverage":1,"random":0,"tracks":[],"combined":0.3309,"private_eq":0.3375},"11":{"raw":0.822,"skill":0.7425,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6659,"skill":0.4996,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4448,"private_eq":0.6329},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8621,"skill":0.7243,"coverage":1,"random":0.5,"tracks":[],"combined":0.7263,"private_eq":0.8642},"21":{"raw":0.5974,"skill":0.1948,"coverage":1,"random":0.5,"tracks":[],"combined":0.176,"private_eq":0.5922},"22":{"raw":0.0428,"skill":0.0353,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0728,"private_eq":0.0895},"23":{"raw":0.5568,"skill":0.1137,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7954,"skill":0.7272,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[],"combined":0.1784,"private_eq":0.3901},"28":{"raw":0.7459,"skill":0.4918,"coverage":1,"random":0.5,"tracks":[],"combined":0.4663,"private_eq":0.7357},"29":{"raw":0.859,"skill":0.812,"coverage":1,"random":0.25,"tracks":[],"combined":0.7703,"private_eq":0.8289},"30":{"raw":0.4886,"skill":0.3887,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6118,"random":0.25},{"track":"10-choice","raw":0.3654,"random":0.1}],"combined":0.5936,"private_eq":0.6677},"31":{"raw":0.149,"skill":0.0731,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0518,"private_eq":0.1385},"32":{"raw":0.5479,"skill":0.2812,"coverage":1,"random":0.371,"tracks":[],"combined":0.2736,"private_eq":0.5477},"33":{"raw":0.2358,"skill":0.2257,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.469,"skill":0.3992,"coverage":1,"random":0.116,"tracks":[],"combined":0.3471,"private_eq":0.4285},"37":{"raw":0.5264,"skill":0.406,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4881,"private_eq":0.5957},"38":{"raw":0.2278,"skill":0.2031,"coverage":1,"random":0.031,"tracks":[],"combined":0.3348,"private_eq":0.3618},"39":{"raw":0.9317,"skill":0.8995,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5108,"private_eq":0.6705},"40":{"raw":0.5639,"skill":0.4389,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2031,"headline":false},{"track":"A · English","score":0.5639,"headline":true},{"track":"C · Arabic pairs","score":0.67,"headline":false},{"track":"C · English pairs","score":0.925,"headline":false}],"combined":0.4481,"private_eq":0.5751},"41":{"raw":0.6443,"skill":0.4664,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1553,"private_eq":0.4426},"42":{"raw":0.7899,"skill":0.5912,"coverage":1,"random":0.486,"tracks":[],"combined":0.612,"private_eq":0.8023},"43":{"raw":0.5561,"skill":0.2958,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4116,"private_eq":0.6327},"44":{"raw":0.669,"skill":0.338,"coverage":1,"random":0.5,"tracks":[],"combined":0.4762,"private_eq":0.7406},"45":{"raw":0.1218,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4021,"skill":0.1323,"coverage":1,"random":0.311,"tracks":[],"combined":0.1596,"private_eq":0.4269},"56":{"raw":0.7605,"skill":0.521,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.6203,"private_eq":0.8118},"57":{"raw":0.5472,"skill":0.4907,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2822,"private_eq":0.3682},"58":{"raw":0.694,"skill":0.5565,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1915,"private_eq":0.4479},"59":{"raw":0.7619,"skill":0.5063,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.3869,"private_eq":0.7072},"61":{"raw":0.6635,"skill":0.327,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3597,"private_eq":0.683},"62":{"raw":0.6796,"skill":0.5728,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4741,"private_eq":0.6093},"64":{"raw":0.6212,"skill":0.5265,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5259,"private_eq":0.6242}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"llm2jev 0.6.1 (chat prompt, thinking off, T=1.087) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","v03":{"engine":"jpt-9b","pub":46.96,"same_skills":43.45,"new_domains":34.98,"v":44.08,"lab":false,"jev":false,"rank_v":118,"rank_pub":132,"name":"JPT-9B Qwen3.5-9B LoRA","eq":{"S":42.9,"O":45.85},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":25.74,"new_domains":18.34}}},{"engine":"jev-9b","name":"JEV-9B","long_name":"JEV-9B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":9650000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/autotrust/JEV-9B","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6577,"conf":0.7302,"ece":0.0726,"brier":0.4462,"over95":0.014,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0096,"conf":0.0856,"acc":0.0475},{"lo":0.1,"w":0.0355,"conf":0.1457,"acc":0.124},{"lo":0.2,"w":0.0272,"conf":0.2526,"acc":0.2376},{"lo":0.3,"w":0.0423,"conf":0.3519,"acc":0.2594},{"lo":0.4,"w":0.0607,"conf":0.4498,"acc":0.3461},{"lo":0.5,"w":0.1198,"conf":0.5501,"acc":0.4946},{"lo":0.6,"w":0.1122,"conf":0.6498,"acc":0.5834},{"lo":0.7,"w":0.1144,"conf":0.7494,"acc":0.6353},{"lo":0.8,"w":0.1291,"conf":0.8529,"acc":0.7336},{"lo":0.9,"w":0.3491,"conf":0.9752,"acc":0.923}],"areas":{"knowledge":{"ece":0.0721,"acc":0.5915,"conf":0.6637},"language":{"ece":0.0658,"acc":0.7352,"conf":0.7994},"retrieval":{"ece":0.0845,"acc":0.688,"conf":0.7667},"tools":{"ece":0.0878,"acc":0.8231,"conf":0.9109},"arts":{"ece":0.0756,"acc":0.5401,"conf":0.6146}}},"gaps":null,"latency":{"median":49.5,"p95":null,"mean":312.6},"results":{"1":{"score":0.9262,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":68.1},"2":{"score":0.6266,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1046},"3":{"score":0.624,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2152.2},"4":{"score":0.72,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":146},"5":{"score":0.7898,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":205.9},"6":{"score":0.7973,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":118.5},"9":{"score":0.2614,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":485.1},"10":{"score":0.6259,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":36.5},"11":{"score":0.707,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":857.4},"12":{"score":0.6763,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":32.4},"20":{"score":0.8506,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.6},"21":{"score":0.605,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.2},"22":{"score":0.2055,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":615.8},"23":{"score":0.6156,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.6},"24":{"score":0.7653,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":4.3},"25":{"score":0.3929,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":17},"26":{"score":0.9827,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":3.4},"27":{"score":0.9428,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":3.7},"28":{"score":0.7443,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":1.8},"29":{"score":0.8476,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":7.2},"30":{"score":0.489,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.125,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44},"32":{"score":0.6024,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":27.8},"33":{"score":0.3364,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":107.7},"36":{"score":0.4416,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":728.8},"37":{"score":0.4621,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.6},"38":{"score":0.18,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":223.8},"39":{"score":0.7617,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":11.1},"40":{"score":0.3457,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":2.7},"41":{"score":0.4232,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":6.7},"42":{"score":0.7967,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":26.6},"43":{"score":0.5123,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":7.7},"44":{"score":0.6562,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.6},"45":{"score":0.0699,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":24.2},"50":{"score":0.4326,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":25.4},"56":{"score":0.5995,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":66.7},"57":{"score":0.5335,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":11.8},"58":{"score":0.6733,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":6},"59":{"score":0.6619,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":27},"61":{"score":0.6468,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":16},"62":{"score":0.7752,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":30.2},"64":{"score":0.6042,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":6}},"benchmarks":{"1":{"raw":0.9262,"skill":0.9004,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7968,"private_eq":0.8504},"2":{"raw":0.6266,"skill":0.5688,"coverage":1,"random":0.1341,"tracks":[],"combined":0.668,"private_eq":0.7149},"3":{"raw":0.624,"skill":0.6168,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5706,"private_eq":0.5825},"4":{"raw":0.72,"skill":0.7164,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7537,"private_eq":0.7586},"5":{"raw":0.7898,"skill":0.7885,"coverage":1,"random":0.006,"tracks":[],"combined":0.5775,"private_eq":0.5838},"6":{"raw":0.7973,"skill":0.5247,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2614,"skill":0.2614,"coverage":1,"random":0,"tracks":[],"combined":0.1817,"private_eq":0.19},"10":{"raw":0.6259,"skill":0.3775,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.707,"skill":0.5762,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6763,"skill":0.5151,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4251,"private_eq":0.6199},"20":{"raw":0.852,"skill":0.704,"coverage":1,"random":0.5,"tracks":[],"combined":0.7698,"private_eq":0.8857},"21":{"raw":0.605,"skill":0.21,"coverage":1,"random":0.5,"tracks":[],"combined":0.0656,"private_eq":0.5376},"22":{"raw":0.1921,"skill":0.1858,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2903,"private_eq":0.3029},"23":{"raw":0.594,"skill":0.188,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7653,"skill":0.6871,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3929,"skill":0.1905,"coverage":1,"random":0.25,"tracks":[],"combined":0.1335,"private_eq":0.3568},"26":{"raw":0.9827,"skill":0.9769,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9428,"skill":0.9237,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7443,"skill":0.4886,"coverage":1,"random":0.5,"tracks":[],"combined":0.4309,"private_eq":0.7182},"29":{"raw":0.8476,"skill":0.7968,"coverage":1,"random":0.25,"tracks":[],"combined":0.7108,"private_eq":0.7848},"30":{"raw":0.489,"skill":0.3879,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6012,"random":0.25},{"track":"10-choice","raw":0.3768,"random":0.1}],"combined":0.5285,"private_eq":0.6146},"31":{"raw":0.125,"skill":0.047,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0723,"private_eq":0.1571},"32":{"raw":0.6024,"skill":0.3679,"coverage":1,"random":0.371,"tracks":[],"combined":0.0881,"private_eq":0.4323},"33":{"raw":0.3364,"skill":0.3276,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3588,"private_eq":0.3734},"36":{"raw":0.4416,"skill":0.3683,"coverage":1,"random":0.116,"tracks":[],"combined":0.3611,"private_eq":0.4408},"37":{"raw":0.4621,"skill":0.3253,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4222,"private_eq":0.5438},"38":{"raw":0.18,"skill":0.1538,"coverage":1,"random":0.031,"tracks":[],"combined":0.2919,"private_eq":0.3207},"39":{"raw":0.7617,"skill":0.6495,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5801,"private_eq":0.7171},"40":{"raw":0.3457,"skill":0.1582,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3875,"headline":false},{"track":"A · English","score":0.3457,"headline":true},{"track":"C · Arabic pairs","score":0.635,"headline":false},{"track":"C · English pairs","score":0.855,"headline":false}],"combined":0.1127,"private_eq":0.3174},"41":{"raw":0.4232,"skill":0.1348,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3319,"private_eq":0.559},"42":{"raw":0.7967,"skill":0.6045,"coverage":1,"random":0.486,"tracks":[],"combined":0.6698,"private_eq":0.8317},"43":{"raw":0.5123,"skill":0.2262,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4324,"private_eq":0.6457},"44":{"raw":0.6562,"skill":0.3124,"coverage":1,"random":0.5,"tracks":[],"combined":0.4336,"private_eq":0.7195},"45":{"raw":0.0699,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4326,"skill":0.1765,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.5995,"skill":0.199,"coverage":1,"random":0.5,"tracks":[],"combined":0.4125,"private_eq":0.7091},"57":{"raw":0.5335,"skill":0.4753,"coverage":1,"random":0.1109,"tracks":[],"combined":0.3755,"private_eq":0.4502},"58":{"raw":0.6733,"skill":0.5265,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1511,"private_eq":0.4203},"59":{"raw":0.6619,"skill":0.299,"coverage":1,"random":0.5177,"tracks":[],"combined":0.0711,"private_eq":0.5566},"61":{"raw":0.6468,"skill":0.2936,"coverage":1,"random":0.5,"tracks":[],"combined":0.3316,"private_eq":0.6691},"62":{"raw":0.7752,"skill":0.7003,"coverage":1,"random":0.25,"tracks":[],"combined":0.5767,"private_eq":0.6854},"64":{"raw":0.6042,"skill":0.5052,"coverage":1,"random":0.2,"tracks":[],"combined":0.458,"private_eq":0.5705}},"coverage":1,"pending":0,"scores":{"balanced_raw":57.47,"balanced_skill":42.15,"breadth_skill":42.23,"public_skill":43.28},"frozen_scores":{"balanced_raw":57.47,"balanced_skill":43.28,"breadth_skill":42.23},"categories":[{"id":"knowledge","raw":0.4682,"skill":0.3072,"combined":0.2984,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6069,"skill":0.4415,"combined":0.4072,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6136,"skill":0.4645,"combined":0.4844,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6525,"skill":0.6206,"combined":0.568,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5466,"skill":0.3282,"combined":0.3457,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-9b","pub":43.28,"same_skills":42.83,"new_domains":31.85,"v":42.15,"lab":false,"jev":false,"rank_v":124,"rank_pub":140,"name":"JEV-9B","eq":{"S":42.27,"O":41.98},"rank_tie":28,"tied":true,"parts":{"public":0,"same_skills":25.36,"new_domains":16.79}}},{"engine":"sieve-9b","name":"Sieve 9B","long_name":"Sieve 9B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":9650000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/Sieve-9B","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6595,"conf":0.6959,"ece":0.0433,"brier":0.4409,"over95":0.0134,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0506,"conf":0.0588,"acc":0.0923},{"lo":0.1,"w":0.0179,"conf":0.1426,"acc":0.2036},{"lo":0.2,"w":0.0249,"conf":0.2544,"acc":0.2807},{"lo":0.3,"w":0.0531,"conf":0.3529,"acc":0.3166},{"lo":0.4,"w":0.0592,"conf":0.4542,"acc":0.4278},{"lo":0.5,"w":0.1229,"conf":0.5494,"acc":0.5105},{"lo":0.6,"w":0.1156,"conf":0.6509,"acc":0.6036},{"lo":0.7,"w":0.1082,"conf":0.7512,"acc":0.6919},{"lo":0.8,"w":0.1372,"conf":0.8528,"acc":0.7965},{"lo":0.9,"w":0.3105,"conf":0.958,"acc":0.9195}],"areas":{"knowledge":{"ece":0.0736,"acc":0.5781,"conf":0.6517},"language":{"ece":0.0522,"acc":0.768,"conf":0.8162},"retrieval":{"ece":0.097,"acc":0.7187,"conf":0.6497},"tools":{"ece":0.0439,"acc":0.8711,"conf":0.8769},"arts":{"ece":0.062,"acc":0.469,"conf":0.5053}}},"gaps":null,"latency":{"median":50.2,"p95":null,"mean":192.3},"results":{"1":{"score":0.954,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":269.9},"2":{"score":0.6514,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3289.3},"3":{"score":0.7264,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1276.8},"4":{"score":0.8362,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":203.3},"5":{"score":0.7603,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":293.1},"6":{"score":0.793,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":633},"9":{"score":0.3068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1130.1},"10":{"score":0.6267,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":236.1},"11":{"score":0.6145,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":893.6},"12":{"score":0.5615,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":74.2},"20":{"score":0.6654,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.4},"21":{"score":0.5506,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":72.9},"22":{"score":0.129,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1239.7},"23":{"score":0.5804,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.9},"24":{"score":0.7494,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":62.5},"25":{"score":0.4235,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":62.6},"26":{"score":0.9777,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":61.9},"27":{"score":0.9377,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":62.1},"28":{"score":0.7506,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":61.4},"29":{"score":0.8297,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":62.9},"30":{"score":0.3916,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1144,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":96},"32":{"score":0.5665,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":104.5},"33":{"score":0.2958,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":145.6},"36":{"score":0.4202,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2961.5},"37":{"score":0.4931,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":63.7},"38":{"score":0.2409,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":702.4},"39":{"score":0.8882,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":132.2},"40":{"score":0.5064,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":63.1},"41":{"score":0.5222,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":62.9},"42":{"score":0.779,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":69.3},"43":{"score":0.5368,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":62.5},"44":{"score":0.633,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.5},"45":{"score":0.1058,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":63.1},"50":{"score":0.4039,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":99.6},"56":{"score":0.541,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":154.7},"57":{"score":0.5081,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":63.3},"58":{"score":0.6715,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":62.6},"59":{"score":0.6285,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":95.3},"61":{"score":0.6348,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":64.5},"62":{"score":0.5613,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":93.7},"64":{"score":0.5814,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":62.6}},"benchmarks":{"1":{"raw":0.954,"skill":0.9378,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8989,"private_eq":0.9252},"2":{"raw":0.6514,"skill":0.5975,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6862,"private_eq":0.7305},"3":{"raw":0.7264,"skill":0.7211,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7386,"private_eq":0.7455},"4":{"raw":0.8362,"skill":0.8341,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8188,"private_eq":0.8221},"5":{"raw":0.7603,"skill":0.7589,"coverage":1,"random":0.006,"tracks":[],"combined":0.6128,"private_eq":0.6185},"6":{"raw":0.793,"skill":0.5147,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3068,"skill":0.3068,"coverage":1,"random":0,"tracks":[],"combined":0.2575,"private_eq":0.265},"10":{"raw":0.6267,"skill":0.3789,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6145,"skill":0.4425,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5615,"skill":0.3432,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4812,"private_eq":0.6569},"20":{"raw":0.676,"skill":0.352,"coverage":1,"random":0.5,"tracks":[],"combined":0.6468,"private_eq":0.8249},"21":{"raw":0.5506,"skill":0.1012,"coverage":1,"random":0.5,"tracks":[],"combined":0.0656,"private_eq":0.5376},"22":{"raw":0.1239,"skill":0.1171,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1438,"private_eq":0.1592},"23":{"raw":0.5649,"skill":0.1299,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7494,"skill":0.6659,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4235,"skill":0.2313,"coverage":1,"random":0.25,"tracks":[],"combined":0.1307,"private_eq":0.3547},"26":{"raw":0.9777,"skill":0.9703,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9377,"skill":0.9169,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7506,"skill":0.5012,"coverage":1,"random":0.5,"tracks":[],"combined":0.3603,"private_eq":0.6833},"29":{"raw":0.8297,"skill":0.7729,"coverage":1,"random":0.25,"tracks":[],"combined":0.7503,"private_eq":0.8141},"30":{"raw":0.3916,"skill":0.268,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4958,"random":0.25},{"track":"10-choice","raw":0.2873,"random":0.1}],"combined":0.4874,"private_eq":0.5811},"31":{"raw":0.1144,"skill":0.0354,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0355,"private_eq":0.1237},"32":{"raw":0.5665,"skill":0.3108,"coverage":1,"random":0.371,"tracks":[],"combined":0.0649,"private_eq":0.4179},"33":{"raw":0.2958,"skill":0.2865,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3383,"private_eq":0.3534},"36":{"raw":0.4202,"skill":0.344,"coverage":1,"random":0.116,"tracks":[],"combined":0.3456,"private_eq":0.4272},"37":{"raw":0.4931,"skill":0.3643,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4485,"private_eq":0.5645},"38":{"raw":0.2409,"skill":0.2166,"coverage":1,"random":0.031,"tracks":[],"combined":0.3495,"private_eq":0.3759},"39":{"raw":0.8882,"skill":0.8356,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5016,"private_eq":0.6643},"40":{"raw":0.5064,"skill":0.365,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3448,"headline":false},{"track":"A · English","score":0.5064,"headline":true},{"track":"C · Arabic pairs","score":0.61,"headline":false},{"track":"C · English pairs","score":0.82,"headline":false}],"combined":0.2844,"private_eq":0.4493},"41":{"raw":0.5222,"skill":0.2834,"coverage":1,"random":0.3333,"tracks":[],"combined":0.407,"private_eq":0.6085},"42":{"raw":0.779,"skill":0.57,"coverage":1,"random":0.486,"tracks":[],"combined":0.5141,"private_eq":0.7526},"43":{"raw":0.5368,"skill":0.2652,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3864,"private_eq":0.617},"44":{"raw":0.633,"skill":0.266,"coverage":1,"random":0.5,"tracks":[],"combined":0.4066,"private_eq":0.7062},"45":{"raw":0.1058,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4039,"skill":0.1349,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.541,"skill":0.082,"coverage":1,"random":0.5,"tracks":[],"combined":0.1012,"private_eq":0.5552},"57":{"raw":0.5081,"skill":0.4467,"coverage":1,"random":0.1109,"tracks":[],"combined":0.1678,"private_eq":0.2676},"58":{"raw":0.6715,"skill":0.5238,"coverage":1,"random":0.3101,"tracks":[],"combined":0.175,"private_eq":0.4366},"59":{"raw":0.6285,"skill":0.2297,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2714,"private_eq":0.6521},"61":{"raw":0.6348,"skill":0.2696,"coverage":1,"random":0.5,"tracks":[],"combined":0.3429,"private_eq":0.6747},"62":{"raw":0.5613,"skill":0.4151,"coverage":1,"random":0.25,"tracks":[],"combined":0.4466,"private_eq":0.5889},"64":{"raw":0.5814,"skill":0.4768,"coverage":1,"random":0.2,"tracks":[],"combined":0.4482,"private_eq":0.5628}},"coverage":1,"pending":0,"scores":{"balanced_raw":57.12,"balanced_skill":41.73,"breadth_skill":40.31,"public_skill":41.88},"frozen_scores":{"balanced_raw":57.12,"balanced_skill":41.88,"breadth_skill":40.31},"categories":[{"id":"knowledge","raw":0.4575,"skill":0.2896,"combined":0.2518,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6222,"skill":0.4555,"combined":0.4531,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6195,"skill":0.4607,"combined":0.4583,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6548,"skill":0.613,"combined":0.6214,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4834,"skill":0.2186,"combined":0.2971,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-9b","pub":41.88,"same_skills":42.85,"new_domains":30.96,"v":41.73,"lab":false,"jev":false,"rank_v":126,"rank_pub":146,"name":"Sieve 9B","eq":{"S":42.3,"O":40.87},"rank_tie":28,"tied":true,"parts":{"public":0,"same_skills":25.38,"new_domains":16.35}}},{"engine":"kev-9b-v2","name":"Kev 9B v2","long_name":"Kev 9B v2","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":9650000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/jaredpalmer/kev-9b","code_url":null,"base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6541,"conf":0.6795,"ece":0.0453,"brier":0.4506,"over95":0.0116,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0565,"conf":0.0582,"acc":0.1087},{"lo":0.1,"w":0.018,"conf":0.1471,"acc":0.4072},{"lo":0.2,"w":0.0212,"conf":0.2553,"acc":0.3453},{"lo":0.3,"w":0.0526,"conf":0.3516,"acc":0.3614},{"lo":0.4,"w":0.0679,"conf":0.4491,"acc":0.3853},{"lo":0.5,"w":0.1267,"conf":0.5484,"acc":0.514},{"lo":0.6,"w":0.1246,"conf":0.649,"acc":0.6079},{"lo":0.7,"w":0.1202,"conf":0.7511,"acc":0.7111},{"lo":0.8,"w":0.1494,"conf":0.8537,"acc":0.8173},{"lo":0.9,"w":0.2629,"conf":0.9548,"acc":0.9119}],"areas":{"knowledge":{"ece":0.063,"acc":0.6005,"conf":0.6579},"language":{"ece":0.0471,"acc":0.7566,"conf":0.7915},"retrieval":{"ece":0.1427,"acc":0.7096,"conf":0.6244},"tools":{"ece":0.0447,"acc":0.819,"conf":0.7892},"arts":{"ece":0.0828,"acc":0.4443,"conf":0.5037}}},"gaps":null,"latency":{"median":60.3,"p95":null,"mean":90.1},"results":{"1":{"score":0.9522,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6379,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.6535,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8327,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.771,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.8004,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.1136,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6043,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.653,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5812,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.5892,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.5731,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.1015,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.53,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.7608,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.3776,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9756,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9386,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.7506,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.8268,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.5444,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.12,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.5652,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3067,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.3818,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4906,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.1698,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.8971,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4324,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.5666,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.7523,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.5544,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6348,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1098,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4004,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.616,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.5653,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.6626,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.7057,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.6072,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.6071,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.5663,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9522,"skill":0.9355,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8717,"private_eq":0.9053},"2":{"raw":0.6379,"skill":0.5819,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6611,"private_eq":0.709},"3":{"raw":0.6535,"skill":0.6468,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6389,"private_eq":0.6488},"4":{"raw":0.8327,"skill":0.8305,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7901,"private_eq":0.7941},"5":{"raw":0.771,"skill":0.7696,"coverage":1,"random":0.006,"tracks":[],"combined":0.6057,"private_eq":0.6115},"6":{"raw":0.8004,"skill":0.532,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1136,"skill":0.1136,"coverage":1,"random":0,"tracks":[],"combined":0.2196,"private_eq":0.2275},"10":{"raw":0.6043,"skill":0.3416,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.653,"skill":0.4982,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5812,"skill":0.3727,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4696,"private_eq":0.6493},"20":{"raw":0.5957,"skill":0.1913,"coverage":1,"random":0.5,"tracks":[],"combined":0.4155,"private_eq":0.7106},"21":{"raw":0.5731,"skill":0.1461,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.1012,"skill":0.0942,"coverage":1,"random":0.0078,"tracks":[],"combined":0.115,"private_eq":0.1309},"23":{"raw":0.5156,"skill":0.0312,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7608,"skill":0.6811,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3776,"skill":0.1701,"coverage":1,"random":0.25,"tracks":[],"combined":0.1419,"private_eq":0.363},"26":{"raw":0.9756,"skill":0.9675,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9386,"skill":0.9181,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7506,"skill":0.5012,"coverage":1,"random":0.5,"tracks":[],"combined":0.4261,"private_eq":0.7158},"29":{"raw":0.8268,"skill":0.7691,"coverage":1,"random":0.25,"tracks":[],"combined":0.7703,"private_eq":0.8289},"30":{"raw":0.5444,"skill":0.4534,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6368,"random":0.25},{"track":"10-choice","raw":0.4519,"random":0.1}],"combined":0.5931,"private_eq":0.6673},"31":{"raw":0.12,"skill":0.0415,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0366,"private_eq":0.1247},"32":{"raw":0.5652,"skill":0.3087,"coverage":1,"random":0.371,"tracks":[],"combined":0.0649,"private_eq":0.4179},"33":{"raw":0.3067,"skill":0.2975,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2817,"private_eq":0.2982},"36":{"raw":0.3818,"skill":0.3007,"coverage":1,"random":0.116,"tracks":[],"combined":0.31,"private_eq":0.3961},"37":{"raw":0.4906,"skill":0.3611,"coverage":1,"random":0.2027,"tracks":[],"combined":0.488,"private_eq":0.5956},"38":{"raw":0.1698,"skill":0.1432,"coverage":1,"random":0.031,"tracks":[],"combined":0.2345,"private_eq":0.2657},"39":{"raw":0.8971,"skill":0.8487,"coverage":1,"random":0.3201,"tracks":[],"combined":0.502,"private_eq":0.6646},"40":{"raw":0.4324,"skill":0.2698,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3776,"headline":false},{"track":"A · English","score":0.4324,"headline":true},{"track":"C · Arabic pairs","score":0.55,"headline":false},{"track":"C · English pairs","score":0.785,"headline":false}],"combined":0.2227,"private_eq":0.4019},"41":{"raw":0.5666,"skill":0.3498,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3447,"private_eq":0.5674},"42":{"raw":0.7523,"skill":0.5181,"coverage":1,"random":0.486,"tracks":[],"combined":0.6531,"private_eq":0.8232},"43":{"raw":0.5544,"skill":0.293,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5035,"private_eq":0.69},"44":{"raw":0.6348,"skill":0.2696,"coverage":1,"random":0.5,"tracks":[],"combined":0.4548,"private_eq":0.73},"45":{"raw":0.1098,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4004,"skill":0.1297,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.616,"skill":0.232,"coverage":1,"random":0.5,"tracks":[],"combined":0.2654,"private_eq":0.6364},"57":{"raw":0.5653,"skill":0.5111,"coverage":1,"random":0.1109,"tracks":[],"combined":0.2655,"private_eq":0.3535},"58":{"raw":0.6626,"skill":0.5109,"coverage":1,"random":0.3101,"tracks":[],"combined":0.126,"private_eq":0.4032},"59":{"raw":0.7057,"skill":0.3898,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2777,"private_eq":0.6551},"61":{"raw":0.6072,"skill":0.2144,"coverage":1,"random":0.5,"tracks":[],"combined":0.3316,"private_eq":0.6691},"62":{"raw":0.6071,"skill":0.4761,"coverage":1,"random":0.25,"tracks":[],"combined":0.4741,"private_eq":0.6093},"64":{"raw":0.5663,"skill":0.4579,"coverage":1,"random":0.2,"tracks":[],"combined":0.3998,"private_eq":0.5245}},"coverage":1,"pending":0,"scores":{"balanced_raw":56.51,"balanced_skill":40.91,"breadth_skill":39.98,"public_skill":41.54},"frozen_scores":{"balanced_raw":56.51,"balanced_skill":41.54,"breadth_skill":39.98},"categories":[{"id":"knowledge","raw":0.4737,"skill":0.308,"combined":0.2857,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6241,"skill":0.4668,"combined":0.4432,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6207,"skill":0.4679,"combined":0.4745,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6084,"skill":0.5686,"combined":0.5865,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4587,"skill":0.1751,"combined":0.213,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"kev-9b-v2","pub":41.54,"same_skills":42.37,"new_domains":29.9,"v":40.91,"lab":false,"jev":false,"rank_v":129,"rank_pub":147,"name":"Kev 9B v2","eq":{"S":41.81,"O":39.57},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.08,"new_domains":15.83}},"notes":["Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."]},{"engine":"jev-omni","name":"Jev-Omni","long_name":"Jev-Omni","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":11959730224,"params_estimated":false,"params_from_name":false,"trained_bytes":23923515487,"model_url":"https://huggingface.co/akhilaaa3/Jev-Omni","code_url":null,"base_model":"google/gemma-4-12B","base_url":"https://huggingface.co/google/gemma-4-12B","weights_repo":"akhilaaa3/Jev-Omni","served_checkpoint":null},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.6137,"conf":0.7748,"ece":0.161,"brier":0.5344,"over95":0.0552,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0172,"conf":0.0763,"acc":0.0247},{"lo":0.1,"w":0.0431,"conf":0.1469,"acc":0.0401},{"lo":0.2,"w":0.0309,"conf":0.2471,"acc":0.1333},{"lo":0.3,"w":0.0367,"conf":0.353,"acc":0.2692},{"lo":0.4,"w":0.0414,"conf":0.4507,"acc":0.2998},{"lo":0.5,"w":0.0749,"conf":0.5541,"acc":0.3944},{"lo":0.6,"w":0.0764,"conf":0.649,"acc":0.4738},{"lo":0.7,"w":0.0725,"conf":0.7527,"acc":0.5175},{"lo":0.8,"w":0.0979,"conf":0.8536,"acc":0.577},{"lo":0.9,"w":0.5091,"conf":0.9796,"acc":0.8357}],"areas":{"knowledge":{"ece":0.1741,"acc":0.5739,"conf":0.748},"language":{"ece":0.1559,"acc":0.7534,"conf":0.9089},"retrieval":{"ece":0.2051,"acc":0.4204,"conf":0.6254},"tools":{"ece":0.0448,"acc":0.81,"conf":0.7923},"arts":{"ece":0.2145,"acc":0.4867,"conf":0.7012}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":54.9,"p95":627.9,"mean":189.4},"results":{"1":{"score":0.9675,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":149.5},"2":{"score":0.6443,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1812.2},"3":{"score":0.5236,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.3099,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":131.7},"5":{"score":0.1689,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":193},"6":{"score":0.7757,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":475.6},"9":{"score":0.4205,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5043,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":116.5},"11":{"score":0.6984,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6467,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":51.9},"20":{"score":0.7668,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.1},"21":{"score":0.6149,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":51.8},"22":{"score":0.042,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":780.7},"23":{"score":0.605,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.2},"24":{"score":0.7753,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":52.1},"25":{"score":0.3673,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9798,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":51.8},"27":{"score":0.9497,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.7},"28":{"score":0.7206,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":51.8},"29":{"score":0.8105,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":52.3},"30":{"score":0.2779,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1282,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":104.6},"32":{"score":0.5572,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2667,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":530.6},"36":{"score":0.4651,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.535,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.7},"38":{"score":0.16,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3310.9},"39":{"score":0.9081,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4107,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":52.3,"tracks":[{"track":"A · Arabic","score":0.4766,"headline":false},{"track":"A · English","score":0.4107,"headline":true},{"track":"C · Arabic pairs","score":0.875,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5462,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":51.9},"42":{"score":0.8232,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":71.7},"43":{"score":0.507,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6676,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.3},"45":{"score":0.0998,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4153,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":104},"56":{"score":0.7995,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":466.9,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4894,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":52.3,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6657,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":52,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7998,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":92,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6585,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":62.3,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.6303,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":84.1,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5985,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":54.33,"balanced_skill":43.83,"breadth_skill":39.32,"public_skill":41.04},"frozen_scores":{"balanced_raw":54.33,"balanced_skill":41.04,"breadth_skill":39.32},"categories":[{"id":"knowledge","raw":0.4347,"skill":0.2609,"combined":0.2345,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6484,"skill":0.5134,"combined":0.4815,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4736,"skill":0.3583,"combined":0.3934,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6453,"skill":0.6077,"combined":0.634,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5059,"skill":0.2742,"combined":0.3191,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9675,"skill":0.9562,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8989,"private_eq":0.9252},"2":{"raw":0.6443,"skill":0.5893,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6906,"private_eq":0.7343},"3":{"raw":0.5236,"skill":0.5144,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6442,"private_eq":0.6539},"4":{"raw":0.3099,"skill":0.301,"coverage":1,"random":0.0127,"tracks":[],"combined":0.3336,"private_eq":0.3486},"5":{"raw":0.1689,"skill":0.1639,"coverage":1,"random":0.006,"tracks":[],"combined":0.2084,"private_eq":0.2211},"6":{"raw":0.7757,"skill":0.4699,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7657,"headline":false},{"track":"RouterBench-5shot","score":0.7858,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.4205,"skill":0.4205,"coverage":1,"random":0,"tracks":[],"combined":0.3865,"private_eq":0.3925},"11":{"raw":0.6984,"skill":0.5639,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6467,"skill":0.4708,"coverage":1,"random":0.3324,"tracks":[],"combined":0.474,"private_eq":0.6522},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.7663,"skill":0.5326,"coverage":1,"random":0.5,"tracks":[],"combined":0.5021,"private_eq":0.7534},"21":{"raw":0.6149,"skill":0.2298,"coverage":1,"random":0.5,"tracks":[],"combined":0.3069,"private_eq":0.6569},"22":{"raw":0.0395,"skill":0.032,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0911,"private_eq":0.1075},"23":{"raw":0.6008,"skill":0.2016,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7849,"skill":0.7132,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3673,"skill":0.1565,"coverage":1,"random":0.25,"tracks":[],"combined":0.1307,"private_eq":0.3547},"28":{"raw":0.7206,"skill":0.4412,"coverage":1,"random":0.5,"tracks":[],"combined":0.547,"private_eq":0.7756},"29":{"raw":0.8105,"skill":0.7473,"coverage":1,"random":0.25,"tracks":[],"combined":0.7059,"private_eq":0.7812},"30":{"raw":0.2779,"skill":0.1279,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3723,"random":0.25},{"track":"10-choice","raw":0.1835,"random":0.1}],"combined":0.3417,"private_eq":0.4623},"31":{"raw":0.1282,"skill":0.0505,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0495,"private_eq":0.1364},"32":{"raw":0.5572,"skill":0.296,"coverage":1,"random":0.371,"tracks":[],"combined":0.2505,"private_eq":0.5333},"33":{"raw":0.2667,"skill":0.257,"coverage":1,"random":0.0131,"tracks":[],"combined":0.215,"private_eq":0.2331},"36":{"raw":0.4651,"skill":0.3949,"coverage":1,"random":0.116,"tracks":[],"combined":0.3481,"private_eq":0.4294},"37":{"raw":0.535,"skill":0.4167,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5164,"private_eq":0.618},"38":{"raw":0.16,"skill":0.1331,"coverage":1,"random":0.031,"tracks":[],"combined":0.2428,"private_eq":0.2737},"39":{"raw":0.9081,"skill":0.8648,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5352,"private_eq":0.6869},"40":{"raw":0.4107,"skill":0.2419,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4766,"headline":false},{"track":"A · English","score":0.4107,"headline":true},{"track":"C · Arabic pairs","score":0.875,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"combined":0.1937,"private_eq":0.3796},"41":{"raw":0.5462,"skill":0.3192,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3788,"private_eq":0.5899},"42":{"raw":0.8232,"skill":0.656,"coverage":1,"random":0.486,"tracks":[],"combined":0.8792,"private_eq":0.9381},"43":{"raw":0.507,"skill":0.2178,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2569,"private_eq":0.5363},"44":{"raw":0.6676,"skill":0.3352,"coverage":1,"random":0.5,"tracks":[],"combined":0.278,"private_eq":0.6426},"45":{"raw":0.0998,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4153,"skill":0.1514,"coverage":1,"random":0.311,"tracks":[],"combined":0.1306,"private_eq":0.4071},"56":{"raw":0.7995,"skill":0.599,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.6636,"private_eq":0.8332},"57":{"raw":0.4894,"skill":0.4257,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.1777,"private_eq":0.2763},"58":{"raw":0.6657,"skill":0.5154,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0952,"private_eq":0.3822},"59":{"raw":0.7998,"skill":0.5849,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.3987,"private_eq":0.7128},"61":{"raw":0.6585,"skill":0.317,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3486,"private_eq":0.6775},"62":{"raw":0.6303,"skill":0.5071,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4946,"private_eq":0.6245},"64":{"raw":0.5985,"skill":0.4981,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5647,"private_eq":0.6549}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author JevOmni.predict per question, single process (no shared-card lock); torch 2.14.0+cu130 / transformers 5.17.0","v03":{"engine":"jev-omni","pub":41.04,"same_skills":42.18,"new_domains":36.04,"v":43.83,"lab":false,"jev":false,"rank_v":120,"rank_pub":152,"name":"Jev-Omni","eq":{"S":41.62,"O":47.16},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":24.97,"new_domains":18.86}}},{"engine":"decision-2.0-nox-4b","name":"vLLM-SR Decision 2.0 Nox 4B","long_name":"vLLM-SR Decision 2.0 Nox 4B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4210000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/Decision-2.0-Nox-4B","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.6583,"conf":0.6782,"ece":0.0298,"brier":0.4358,"over95":0.0069,"n":34833,"benchmarks":32,"rel":[{"lo":0,"w":0.0546,"conf":0.0468,"acc":0.0786},{"lo":0.1,"w":0.0114,"conf":0.1506,"acc":0.1801},{"lo":0.2,"w":0.0256,"conf":0.2598,"acc":0.3424},{"lo":0.3,"w":0.0535,"conf":0.3548,"acc":0.2728},{"lo":0.4,"w":0.0784,"conf":0.4525,"acc":0.4021},{"lo":0.5,"w":0.1302,"conf":0.55,"acc":0.5528},{"lo":0.6,"w":0.1277,"conf":0.65,"acc":0.6528},{"lo":0.7,"w":0.1197,"conf":0.7489,"acc":0.7064},{"lo":0.8,"w":0.1306,"conf":0.8523,"acc":0.8038},{"lo":0.9,"w":0.2683,"conf":0.9591,"acc":0.9402}],"areas":{"knowledge":{"ece":0.0471,"acc":0.5766,"conf":0.6192},"language":{"ece":0.0215,"acc":0.7376,"conf":0.7411},"retrieval":{"ece":0.0597,"acc":0.7235,"conf":0.7253},"tools":{"ece":0.0618,"acc":0.904,"conf":0.8798},"arts":{"ece":0.0604,"acc":0.4958,"conf":0.5362}}},"gaps":null,"latency":{"median":33.5,"p95":null,"mean":67},"results":{"1":{"score":0.9079,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":57.1},"2":{"score":0.6556,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":553.7},"3":{"score":0.7756,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":184},"4":{"score":0.7853,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":59},"5":{"score":0.8765,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":95.5},"6":{"score":0.7781,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":110.6},"9":{"score":0.1818,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":421.3},"10":{"score":0.6151,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":38},"11":{"score":0.751,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":855},"12":{"score":0.5975,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31.4},"20":{"score":0.734,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.4},"21":{"score":0.6062,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":30.1},"22":{"score":0.0865,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":185.1},"23":{"score":0.5852,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.7},"24":{"score":0.7128,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":28},"25":{"score":0.3265,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":30.1},"26":{"score":0.9802,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":27.6},"27":{"score":0.9206,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.6},"28":{"score":0.8295,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":29.2},"29":{"score":0.814,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":29.2},"30":{"score":0.5815,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1068,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":42.1},"32":{"score":0.5665,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":41.4},"33":{"score":0.2891,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":102.3},"36":{"score":0.4318,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":988.2},"37":{"score":0.4609,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.2},"38":{"score":0.0868,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":259.3},"39":{"score":0.8888,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":34.8},"40":{"score":0.3967,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":33.1},"41":{"score":0.5217,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":32.1},"42":{"score":0.7962,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":34.9},"43":{"score":0.5333,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":30.3},"44":{"score":0.6646,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.3},"45":{"score":0.0938,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":34.8},"50":{"score":0.4248,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":41.9},"56":{"score":0.5575,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":72.4},"57":{"score":0.5051,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":31.6},"58":{"score":0.6635,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":30.3},"59":{"score":0.6803,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":38},"61":{"score":0.8017,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":31.4},"62":{"score":0.6153,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":37.2},"64":{"score":0.5928,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":30.5}},"benchmarks":{"1":{"raw":0.9079,"skill":0.8757,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7694,"private_eq":0.8304},"2":{"raw":0.6546,"skill":0.6011,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.6736,"private_eq":0.7197},"3":{"raw":0.7756,"skill":0.7713,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7439,"private_eq":0.7506},"4":{"raw":0.7853,"skill":0.7825,"coverage":1,"random":0.0127,"tracks":[],"combined":0.838,"private_eq":0.8409},"5":{"raw":0.8765,"skill":0.8758,"coverage":1,"random":0.006,"tracks":[],"combined":0.7227,"private_eq":0.7265},"6":{"raw":0.7781,"skill":0.4797,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1818,"skill":0.1818,"coverage":1,"random":0,"tracks":[],"combined":0.212,"private_eq":0.22},"10":{"raw":0.6151,"skill":0.3596,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.751,"skill":0.6399,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5975,"skill":0.3971,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3304,"private_eq":0.5574},"20":{"raw":0.7386,"skill":0.4773,"coverage":1,"random":0.5,"tracks":[],"combined":0.7119,"private_eq":0.8571},"21":{"raw":0.6062,"skill":0.2123,"coverage":1,"random":0.5,"tracks":[],"combined":0.156,"private_eq":0.5823},"22":{"raw":0.0858,"skill":0.0786,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.1043,"private_eq":0.1204},"23":{"raw":0.5752,"skill":0.1505,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7128,"skill":0.6171,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3265,"skill":0.102,"coverage":1,"random":0.25,"tracks":[],"combined":0.0985,"private_eq":0.3308},"26":{"raw":0.9802,"skill":0.9736,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9206,"skill":0.8941,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8295,"skill":0.659,"coverage":1,"random":0.5,"tracks":[],"combined":0.5319,"private_eq":0.7681},"29":{"raw":0.814,"skill":0.752,"coverage":1,"random":0.25,"tracks":[],"combined":0.6861,"private_eq":0.7665},"30":{"raw":0.5815,"skill":0.498,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6672,"random":0.25},{"track":"10-choice","raw":0.4958,"random":0.1}],"combined":0.6223,"private_eq":0.6911},"31":{"raw":0.1068,"skill":0.0272,"coverage":1,"random":0.0819,"tracks":[],"combined":0.029,"private_eq":0.1178},"32":{"raw":0.5665,"skill":0.3108,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.2891,"skill":0.2797,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2663,"private_eq":0.2832},"36":{"raw":0.4318,"skill":0.3572,"coverage":1,"random":0.116,"tracks":[],"combined":0.3412,"private_eq":0.4234},"37":{"raw":0.4609,"skill":0.3239,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3535,"private_eq":0.4896},"38":{"raw":0.0868,"skill":0.0576,"coverage":1,"random":0.031,"tracks":[],"combined":0.1097,"private_eq":0.1462},"39":{"raw":0.8888,"skill":0.8364,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5431,"private_eq":0.6922},"40":{"raw":0.3967,"skill":0.2239,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4151,"headline":false},{"track":"A · English","score":0.3967,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"combined":0.17,"private_eq":0.3614},"41":{"raw":0.5217,"skill":0.2825,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4888,"private_eq":0.6624},"42":{"raw":0.7962,"skill":0.6035,"coverage":1,"random":0.486,"tracks":[],"combined":0.779,"private_eq":0.8872},"43":{"raw":0.5333,"skill":0.2596,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4199,"private_eq":0.6379},"44":{"raw":0.6646,"skill":0.3292,"coverage":1,"random":0.5,"tracks":[],"combined":0.4493,"private_eq":0.7273},"45":{"raw":0.0938,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4248,"skill":0.1652,"coverage":1,"random":0.311,"tracks":[],"combined":0.174,"private_eq":0.4367},"56":{"raw":0.5575,"skill":0.115,"coverage":1,"random":0.5,"tracks":[],"combined":0.2569,"private_eq":0.6322},"57":{"raw":0.5051,"skill":0.4434,"coverage":1,"random":0.1109,"tracks":[],"combined":0.3088,"private_eq":0.3916},"58":{"raw":0.6635,"skill":0.5122,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1226,"private_eq":0.4009},"59":{"raw":0.6803,"skill":0.3371,"coverage":1,"random":0.5177,"tracks":[],"combined":0.233,"private_eq":0.6338},"61":{"raw":0.8017,"skill":0.6034,"coverage":1,"random":0.5,"tracks":[],"combined":0.5586,"private_eq":0.7813},"62":{"raw":0.6153,"skill":0.4871,"coverage":1,"random":0.25,"tracks":[],"combined":0.4534,"private_eq":0.594},"64":{"raw":0.5928,"skill":0.491,"coverage":1,"random":0.2,"tracks":[],"combined":0.4482,"private_eq":0.5628}},"coverage":0.9999,"pending":0.0001,"scores":{"balanced_raw":58.08,"balanced_skill":40.75,"breadth_skill":42.92,"public_skill":44.21},"frozen_scores":{"balanced_raw":58.08,"balanced_skill":44.21,"breadth_skill":42.92},"categories":[{"id":"knowledge","raw":0.4703,"skill":0.3137,"combined":0.2837,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6186,"skill":0.4638,"combined":0.4218,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6564,"skill":0.5244,"combined":0.5229,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6429,"skill":0.6012,"combined":0.5843,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.5039,"skill":0.2625,"combined":0.3189,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"decision-2.0-nox-4b","pub":44.21,"same_skills":43.72,"new_domains":27.91,"v":40.75,"lab":false,"jev":false,"rank_v":132,"rank_pub":138,"name":"vLLM-SR Decision 2.0 Nox 4B","eq":{"S":43.17,"O":37.12},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.9,"new_domains":14.85}}},{"engine":"decider-35b-nvfp4","name":"Decider 35B-A3B","long_name":"Decider 35B-A3B","variant":"NVFP4","meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":35951822704,"params_estimated":false,"params_from_name":false,"trained_bytes":21013206328,"model_url":"https://huggingface.co/Mapika/decider-35b-a3b-nvfp4","code_url":null,"base_model":"Qwen/Qwen3.5-35B-A3B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B-Base","weights_repo":"Mapika/decider-35b-a3b-nvfp4","served_checkpoint":"Mapika/decider-35b-a3b-nvfp4"},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.6959,"conf":0.6757,"ece":0.0226,"brier":0.3878,"over95":0.0038,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.0446,"conf":0.0708,"acc":0.0496},{"lo":0.1,"w":0.0314,"conf":0.1412,"acc":0.1385},{"lo":0.2,"w":0.0297,"conf":0.2567,"acc":0.3389},{"lo":0.3,"w":0.063,"conf":0.3522,"acc":0.3491},{"lo":0.4,"w":0.073,"conf":0.449,"acc":0.4597},{"lo":0.5,"w":0.1323,"conf":0.5494,"acc":0.5848},{"lo":0.6,"w":0.1206,"conf":0.6518,"acc":0.6765},{"lo":0.7,"w":0.1057,"conf":0.7497,"acc":0.7962},{"lo":0.8,"w":0.1065,"conf":0.8493,"acc":0.8918},{"lo":0.9,"w":0.2932,"conf":0.9704,"acc":0.9742}],"areas":{"knowledge":{"ece":0.0283,"acc":0.6093,"conf":0.6242},"language":{"ece":0.0387,"acc":0.8224,"conf":0.7843},"retrieval":{"ece":0.0538,"acc":0.7811,"conf":0.7372},"tools":{"ece":0.065,"acc":0.7947,"conf":0.7368},"arts":{"ece":0.0347,"acc":0.5312,"conf":0.5078}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":101.4,"p95":210.9,"mean":109.5},"results":{"1":{"score":0.9215,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":198.4},"2":{"score":0.663,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":370.2},"3":{"score":0.565,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8979,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":100.5},"5":{"score":0.9468,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":102.1},"6":{"score":0.7963,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":207.8},"9":{"score":0.2386,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6414,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":105},"11":{"score":0.7496,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6603,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":101.1},"20":{"score":0.8906,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":102.4},"21":{"score":0.6195,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":41.3},"22":{"score":0.0665,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":186.8},"23":{"score":0.649,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.8},"24":{"score":0.8261,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":40.8},"25":{"score":0.4082,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9899,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":41.2},"27":{"score":0.9633,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":40.2},"28":{"score":0.8863,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":40.7},"29":{"score":0.9665,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":100},"30":{"score":0.4151,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1088,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101},"32":{"score":0.6184,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3473,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":217.8},"36":{"score":0.4167,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5065,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":102.6},"38":{"score":0.2724,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":319.2},"39":{"score":0.8524,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5481,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":41,"tracks":[{"track":"A · Arabic","score":0.2937,"headline":false},{"track":"A · English","score":0.5481,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.845,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5452,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":100.7},"42":{"score":0.7847,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":102.6},"43":{"score":0.5982,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6672,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.2},"45":{"score":0.0878,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4642,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":103.7},"56":{"score":0.6685,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":206.6,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.6046,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":100.8,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.7133,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":99.3,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.631,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":103.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.635,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":101.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5972,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":101.7,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6402,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":60.69,"balanced_skill":46.64,"breadth_skill":46.95,"public_skill":47.7},"frozen_scores":{"balanced_raw":60.69,"balanced_skill":47.7,"breadth_skill":46.95},"categories":[{"id":"knowledge","raw":0.4996,"skill":0.3586,"combined":0.3203,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.674,"skill":0.5275,"combined":0.4845,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6854,"skill":0.5466,"combined":0.5213,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6079,"skill":0.5654,"combined":0.4825,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5515,"skill":0.3518,"combined":0.4285,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9215,"skill":0.894,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7763,"private_eq":0.8354},"2":{"raw":0.663,"skill":0.6108,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6881,"private_eq":0.7321},"3":{"raw":0.565,"skill":0.5566,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2871,"private_eq":0.3076},"4":{"raw":0.8979,"skill":0.8966,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8705,"private_eq":0.8726},"5":{"raw":0.9468,"skill":0.9465,"coverage":1,"random":0.006,"tracks":[],"combined":0.639,"private_eq":0.6443},"6":{"raw":0.7963,"skill":0.5196,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7832,"headline":false},{"track":"RouterBench-5shot","score":0.8094,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.2386,"skill":0.2386,"coverage":1,"random":0,"tracks":[],"combined":0.1741,"private_eq":0.1825},"11":{"raw":0.7496,"skill":0.6379,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6603,"skill":0.4912,"coverage":1,"random":0.3324,"tracks":[],"combined":0.444,"private_eq":0.6324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8918,"skill":0.7836,"coverage":1,"random":0.5,"tracks":[],"combined":0.8782,"private_eq":0.9393},"21":{"raw":0.6195,"skill":0.239,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.0635,"skill":0.0562,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0889,"private_eq":0.1053},"23":{"raw":0.6297,"skill":0.2594,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8331,"skill":0.7775,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4082,"skill":0.2109,"coverage":1,"random":0.25,"tracks":[],"combined":0.2217,"private_eq":0.4222},"28":{"raw":0.8863,"skill":0.7726,"coverage":1,"random":0.5,"tracks":[],"combined":0.441,"private_eq":0.7232},"29":{"raw":0.9665,"skill":0.9553,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.4151,"skill":0.2967,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5193,"random":0.25},{"track":"10-choice","raw":0.3108,"random":0.1}],"combined":0.5133,"private_eq":0.6022},"31":{"raw":0.1088,"skill":0.0293,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0193,"private_eq":0.109},"32":{"raw":0.6184,"skill":0.3933,"coverage":1,"random":0.371,"tracks":[],"combined":0.1423,"private_eq":0.466},"33":{"raw":0.3473,"skill":0.3386,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3588,"private_eq":0.3734},"36":{"raw":0.4167,"skill":0.3401,"coverage":1,"random":0.116,"tracks":[],"combined":0.3631,"private_eq":0.4425},"37":{"raw":0.5065,"skill":0.381,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4859,"private_eq":0.594},"38":{"raw":0.2724,"skill":0.2491,"coverage":1,"random":0.031,"tracks":[],"combined":0.3941,"private_eq":0.4186},"39":{"raw":0.8524,"skill":0.7829,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4032,"private_eq":0.5982},"40":{"raw":0.5481,"skill":0.4187,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2937,"headline":false},{"track":"A · English","score":0.5481,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.845,"headline":false}],"combined":0.3841,"private_eq":0.5259},"41":{"raw":0.5452,"skill":0.3178,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3641,"private_eq":0.5802},"42":{"raw":0.7847,"skill":0.5811,"coverage":1,"random":0.486,"tracks":[],"combined":0.8005,"private_eq":0.8981},"43":{"raw":0.5982,"skill":0.3626,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4449,"private_eq":0.6535},"44":{"raw":0.6672,"skill":0.3344,"coverage":1,"random":0.5,"tracks":[],"combined":0.4817,"private_eq":0.7433},"45":{"raw":0.0878,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4642,"skill":0.2224,"coverage":1,"random":0.311,"tracks":[],"combined":0.2974,"private_eq":0.5207},"56":{"raw":0.6685,"skill":0.337,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2569,"private_eq":0.6322},"57":{"raw":0.6046,"skill":0.5553,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3988,"private_eq":0.4707},"58":{"raw":0.7133,"skill":0.5844,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1876,"private_eq":0.4452},"59":{"raw":0.631,"skill":0.2349,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.635,"skill":0.27,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4507,"private_eq":0.728},"62":{"raw":0.5972,"skill":0.4629,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4672,"private_eq":0.6042},"64":{"raw":0.6402,"skill":0.5502,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.6615,"private_eq":0.7315}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"in-process vLLM 0.29.0 LLM, NVFP4 MoE, CUDA graphs, prefix caching off (board config; a prefix-cache run was slower and flipped answers), gpu util 0.9, venv-vllm torch 2.13.0+cu130","v03":{"engine":"decider-35b-nvfp4","pub":47.7,"same_skills":45.12,"new_domains":38.12,"v":46.64,"lab":false,"jev":false,"rank_v":103,"rank_pub":129,"name":"Decider 35B-A3B","eq":{"S":44.59,"O":49.72},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":26.76,"new_domains":19.89}}},{"engine":"djev","name":"djev","long_name":"djev diffgemma","variant":null,"meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/Davipar/djev-dev","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it"},"completed":109036,"counts":{"ok":109036,"unsupported":1165},"lifted":true,"calibration":{"acc":0.632,"conf":0.8445,"ece":0.2124,"brier":0.5655,"over95":0.0816,"n":29650,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0031,"conf":0.1634,"acc":0.0371},{"lo":0.2,"w":0.0141,"conf":0.258,"acc":0.1395},{"lo":0.3,"w":0.0258,"conf":0.3538,"acc":0.1978},{"lo":0.4,"w":0.0362,"conf":0.4563,"acc":0.2261},{"lo":0.5,"w":0.0682,"conf":0.549,"acc":0.4223},{"lo":0.6,"w":0.0757,"conf":0.6528,"acc":0.4607},{"lo":0.7,"w":0.0773,"conf":0.7524,"acc":0.4869},{"lo":0.8,"w":0.1148,"conf":0.8541,"acc":0.5321},{"lo":0.9,"w":0.5849,"conf":0.9774,"acc":0.7767}],"areas":{"knowledge":{"ece":0.2044,"acc":0.5853,"conf":0.7897},"language":{"ece":0.1444,"acc":0.7699,"conf":0.9143},"retrieval":{"ece":0.4174,"acc":0.4502,"conf":0.8677},"tools":{"ece":0.1155,"acc":0.7759,"conf":0.8833},"arts":{"ece":0.2516,"acc":0.5458,"conf":0.7974}}},"gaps":{"share":0.0097,"reasons":[{"reason":"more questions per request than its answer canvas holds","share":0.0097}]},"latency":{"median":84.4,"p95":211.2,"mean":108.8},"results":{"1":{"score":0.9292,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":96.3},"2":{"score":0.542,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":598.5},"3":{"score":0.7067,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6944,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":120.7},"5":{"score":0.1352,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":145.4},"6":{"score":0.7988,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":204.4},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1479,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":112.6},"11":{"score":0.7002,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5981,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":78.8},"20":{"score":0.9028,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":78.6},"21":{"score":0.6088,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":70.5},"22":{"score":0.15,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":169.1},"23":{"score":0.5942,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":85.6},"24":{"score":0.7943,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":77.2},"25":{"score":0.4745,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.984,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":74.8},"27":{"score":0.9522,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":76.4},"28":{"score":0.6938,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":70.3},"29":{"score":0.8386,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":80.5},"30":{"score":0.1967,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1402,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.8},"32":{"score":0.633,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2818,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":106.8},"36":{"score":0.3084,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5643,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":81.2},"38":{"score":0.2749,"jev":null,"metric":"per-review F1","requests":1565,"answered":400,"unsupported":1165,"errors":0,"median_ms":null},"39":{"score":0.8438,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5967,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":70.1,"tracks":[{"track":"A · Arabic","score":0.2825,"headline":false},{"track":"A · English","score":0.5967,"headline":true},{"track":"C · Arabic pairs","score":0.765,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6458,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":77.9},"42":{"score":0.7509,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":88.3},"43":{"score":0.4684,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6392,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.2},"45":{"score":0.1178,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4666,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":102},"56":{"score":0.8605,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":86.4,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4404,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":79.2,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6726,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":78.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7306,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":97.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6683,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":86.5,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5449,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":96.4,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6042,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":54.29,"balanced_skill":38.15,"breadth_skill":39.8,"public_skill":40.89},"frozen_scores":{"balanced_raw":54.29,"balanced_skill":40.89,"breadth_skill":39.8},"categories":[{"id":"knowledge","raw":0.4365,"skill":0.2593,"combined":0.2235,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6449,"skill":0.5003,"combined":0.4857,"coverage":0.9208,"pending":0,"n":9},{"id":"retrieval","raw":0.524,"skill":0.4177,"combined":0.4484,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5648,"skill":0.5168,"combined":0.5292,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.552,"skill":0.3441,"combined":0.374,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9292,"skill":0.9044,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8306,"private_eq":0.8752},"2":{"raw":0.542,"skill":0.4711,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6432,"private_eq":0.6937},"3":{"raw":0.7067,"skill":0.701,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6179,"private_eq":0.6284},"4":{"raw":0.6944,"skill":0.6905,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8033,"private_eq":0.807},"5":{"raw":0.1352,"skill":0.13,"coverage":1,"random":0.006,"tracks":[],"combined":0.2717,"private_eq":0.2833},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0.0021,"private_eq":0.0125},"11":{"raw":0.7002,"skill":0.5664,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5981,"skill":0.398,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5286,"private_eq":0.6882},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.9047,"skill":0.8095,"coverage":1,"random":0.5,"tracks":[],"combined":0.8347,"private_eq":0.9178},"21":{"raw":0.6088,"skill":0.2177,"coverage":1,"random":0.5,"tracks":[],"combined":0.1661,"private_eq":0.5873},"22":{"raw":0.1425,"skill":0.1358,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1404,"private_eq":0.1558},"23":{"raw":0.5852,"skill":0.1704,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8013,"skill":0.735,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4745,"skill":0.2993,"coverage":1,"random":0.25,"tracks":[],"combined":0.1588,"private_eq":0.3755},"28":{"raw":0.6938,"skill":0.3876,"coverage":1,"random":0.5,"tracks":[],"combined":0.3704,"private_eq":0.6883},"29":{"raw":0.8386,"skill":0.7848,"coverage":1,"random":0.25,"tracks":[],"combined":0.7503,"private_eq":0.8141},"30":{"raw":0.1967,"skill":0.0254,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2616,"random":0.25},{"track":"10-choice","raw":0.1319,"random":0.1}],"combined":0.09,"private_eq":0.257},"31":{"raw":0.1402,"skill":0.0635,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.633,"skill":0.4165,"coverage":1,"random":0.371,"tracks":[],"combined":0.1887,"private_eq":0.4949},"33":{"raw":0.2818,"skill":0.2723,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2509,"private_eq":0.2681},"36":{"raw":0.3084,"skill":0.2177,"coverage":1,"random":0.116,"tracks":[],"combined":0.182,"private_eq":0.2842},"37":{"raw":0.5643,"skill":0.4534,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4424,"private_eq":0.5597},"38":{"raw":0.0703,"skill":0.0405,"coverage":0.2556,"random":0.031,"tracks":[],"combined":0.1997,"private_eq":0.2324},"39":{"raw":0.8438,"skill":0.7703,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3964,"private_eq":0.5936},"40":{"raw":0.5967,"skill":0.4811,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2825,"headline":false},{"track":"A · English","score":0.5967,"headline":true},{"track":"C · Arabic pairs","score":0.765,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"combined":0.5271,"private_eq":0.6358},"41":{"raw":0.6458,"skill":0.4687,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4428,"private_eq":0.6321},"42":{"raw":0.7509,"skill":0.5154,"coverage":1,"random":0.486,"tracks":[],"combined":0.804,"private_eq":0.8999},"43":{"raw":0.4684,"skill":0.1566,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2066,"private_eq":0.505},"44":{"raw":0.6392,"skill":0.2784,"coverage":1,"random":0.5,"tracks":[],"combined":0.3692,"private_eq":0.6877},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4666,"skill":0.2258,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.8605,"skill":0.721,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.6203,"private_eq":0.8118},"57":{"raw":0.4404,"skill":0.3706,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3533,"private_eq":0.4307},"58":{"raw":0.6726,"skill":0.5254,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1385,"private_eq":0.4117},"59":{"raw":0.7306,"skill":0.4414,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.1749,"private_eq":0.6061},"61":{"raw":0.6683,"skill":0.3366,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3882,"private_eq":0.6971},"62":{"raw":0.5449,"skill":0.3932,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4741,"private_eq":0.6093},"64":{"raw":0.6042,"skill":0.5052,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4677,"private_eq":0.5782}},"coverage":0.76,"pending":0.24,"latency_rows":744,"latency_path":"board DjevLong: djev in-process adapter over pinned vLLM 0.29.1rc1.dev347, prefix caching, max-model-len 262144","v03":{"engine":"djev","pub":40.89,"same_skills":41.63,"new_domains":25.22,"v":38.15,"lab":false,"jev":false,"rank_v":140,"rank_pub":153,"name":"djev diffgemma","eq":{"S":41.06,"O":33.79},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":24.64,"new_domains":13.52}}},{"engine":"jpt-4b","name":"JPT-4B","long_name":"JPT-4B Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9078627496,"model_url":"https://huggingface.co/kirp/jpt-4b","code_url":"https://github.com/tic-top/llm2jev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"kirp/jpt-4b","served_checkpoint":"277ae37b7a6398a8ec3b86712ac16c1991ab6ef1"},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.6649,"conf":0.7443,"ece":0.0793,"brier":0.4478,"over95":0.0148,"n":35093,"benchmarks":32,"rel":[{"lo":0,"w":0.0103,"conf":0.0836,"acc":0.0688},{"lo":0.1,"w":0.0316,"conf":0.147,"acc":0.1099},{"lo":0.2,"w":0.0301,"conf":0.254,"acc":0.1719},{"lo":0.3,"w":0.045,"conf":0.352,"acc":0.3049},{"lo":0.4,"w":0.0525,"conf":0.4532,"acc":0.3683},{"lo":0.5,"w":0.0974,"conf":0.5512,"acc":0.4766},{"lo":0.6,"w":0.1008,"conf":0.6541,"acc":0.5386},{"lo":0.7,"w":0.0966,"conf":0.7547,"acc":0.6209},{"lo":0.8,"w":0.1584,"conf":0.8532,"acc":0.7374},{"lo":0.9,"w":0.3773,"conf":0.9642,"acc":0.9144}],"areas":{"knowledge":{"ece":0.0812,"acc":0.5976,"conf":0.6789},"language":{"ece":0.0478,"acc":0.7932,"conf":0.8404},"retrieval":{"ece":0.1172,"acc":0.6371,"conf":0.749},"tools":{"ece":0.0628,"acc":0.8495,"conf":0.8852},"arts":{"ece":0.133,"acc":0.4937,"conf":0.6267}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":139.1,"p95":405.7,"mean":154.9},"results":{"1":{"score":0.9156,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":247.4},"2":{"score":0.6212,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":617.7},"3":{"score":0.687,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7212,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":151.6},"5":{"score":0.6454,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":169.4},"6":{"score":0.7994,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":363},"9":{"score":0.125,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6048,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":164.5},"11":{"score":0.7966,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6199,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":132.9},"20":{"score":0.8048,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.4},"21":{"score":0.5974,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":131.9},"22":{"score":0.0165,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":347.8},"23":{"score":0.6,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":146.7},"24":{"score":0.7382,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":58.6},"25":{"score":0.4286,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9777,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":57.8},"27":{"score":0.9283,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":57.8},"28":{"score":0.7127,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":58.9},"29":{"score":0.8103,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":134.4},"30":{"score":0.3632,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1384,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":149.5},"32":{"score":0.5559,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2176,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":399.6},"36":{"score":0.4215,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4639,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":142.5},"38":{"score":0.2003,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":597.4},"39":{"score":0.9191,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5175,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":134.6,"tracks":[{"track":"A · Arabic","score":0.2694,"headline":false},{"track":"A · English","score":0.5175,"headline":true},{"track":"C · Arabic pairs","score":0.615,"headline":false},{"track":"C · English pairs","score":0.85,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6343,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":60.1},"42":{"score":0.7846,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":143.8},"43":{"score":0.5088,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6496,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.5},"45":{"score":0.1138,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.389,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":151.5},"56":{"score":0.689,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":352.5,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4889,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":132.9,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6307,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":77.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7418,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":155.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6085,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":142.3,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.6405,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":148.3,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5739,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":56.74,"balanced_skill":39.56,"breadth_skill":41.15,"public_skill":42.82},"frozen_scores":{"balanced_raw":56.74,"balanced_skill":42.82,"breadth_skill":41.15},"categories":[{"id":"knowledge","raw":0.4403,"skill":0.2651,"combined":0.2716,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6713,"skill":0.5373,"combined":0.4126,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.592,"skill":0.4501,"combined":0.4042,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6129,"skill":0.572,"combined":0.5615,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4945,"skill":0.2611,"combined":0.3055,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9156,"skill":0.886,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7285,"private_eq":0.8004},"2":{"raw":0.6212,"skill":0.5625,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6548,"private_eq":0.7036},"3":{"raw":0.687,"skill":0.681,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6651,"private_eq":0.6742},"4":{"raw":0.7212,"skill":0.7176,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6829,"private_eq":0.6895},"5":{"raw":0.6454,"skill":0.6433,"coverage":1,"random":0.006,"tracks":[],"combined":0.3994,"private_eq":0.4088},"6":{"raw":0.7995,"skill":0.5272,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7859,"headline":false},{"track":"RouterBench-5shot","score":0.813,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1,"random":0,"tracks":[],"combined":0.1488,"private_eq":0.1575},"11":{"raw":0.7966,"skill":0.7058,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6199,"skill":0.4306,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4296,"private_eq":0.6229},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8055,"skill":0.611,"coverage":1,"random":0.5,"tracks":[],"combined":0.7552,"private_eq":0.8785},"21":{"raw":0.5974,"skill":0.1948,"coverage":1,"random":0.5,"tracks":[],"combined":0.2264,"private_eq":0.6171},"22":{"raw":0.0149,"skill":0.0072,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0108,"private_eq":0.0287},"23":{"raw":0.5865,"skill":0.173,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7511,"skill":0.6681,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4286,"skill":0.2381,"coverage":1,"random":0.25,"tracks":[],"combined":0.1265,"private_eq":0.3516},"28":{"raw":0.7127,"skill":0.4254,"coverage":1,"random":0.5,"tracks":[],"combined":0.3654,"private_eq":0.6858},"29":{"raw":0.8103,"skill":0.7471,"coverage":1,"random":0.25,"tracks":[],"combined":0.6514,"private_eq":0.7408},"30":{"raw":0.3632,"skill":0.2351,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4845,"random":0.25},{"track":"10-choice","raw":0.2418,"random":0.1}],"combined":0.4418,"private_eq":0.5439},"31":{"raw":0.1384,"skill":0.0616,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0853,"private_eq":0.1689},"32":{"raw":0.5559,"skill":0.294,"coverage":1,"random":0.371,"tracks":[],"combined":0.2119,"private_eq":0.5093},"33":{"raw":0.2176,"skill":0.2072,"coverage":1,"random":0.0131,"tracks":[],"combined":0.148,"private_eq":0.1678},"36":{"raw":0.4215,"skill":0.3456,"coverage":1,"random":0.116,"tracks":[],"combined":0.269,"private_eq":0.3603},"37":{"raw":0.4639,"skill":0.3276,"coverage":1,"random":0.2027,"tracks":[],"combined":0.346,"private_eq":0.4837},"38":{"raw":0.2003,"skill":0.1747,"coverage":1,"random":0.031,"tracks":[],"combined":0.295,"private_eq":0.3237},"39":{"raw":0.9191,"skill":0.881,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5321,"private_eq":0.6848},"40":{"raw":0.5175,"skill":0.3792,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2694,"headline":false},{"track":"A · English","score":0.5175,"headline":true},{"track":"C · Arabic pairs","score":0.615,"headline":false},{"track":"C · English pairs","score":0.85,"headline":false}],"combined":0.281,"private_eq":0.4467},"41":{"raw":0.6343,"skill":0.4514,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2324,"private_eq":0.4934},"42":{"raw":0.7846,"skill":0.5809,"coverage":1,"random":0.486,"tracks":[],"combined":0.6356,"private_eq":0.8143},"43":{"raw":0.5088,"skill":0.2206,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3864,"private_eq":0.617},"44":{"raw":0.6496,"skill":0.2992,"coverage":1,"random":0.5,"tracks":[],"combined":0.4817,"private_eq":0.7433},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.389,"skill":0.1133,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.689,"skill":0.378,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3174,"private_eq":0.6621},"57":{"raw":0.4889,"skill":0.4251,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3377,"private_eq":0.417},"58":{"raw":0.6307,"skill":0.4647,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1521,"private_eq":0.421},"59":{"raw":0.7418,"skill":0.4646,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.1925,"private_eq":0.6145},"61":{"raw":0.6085,"skill":0.217,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3826,"private_eq":0.6943},"62":{"raw":0.6405,"skill":0.5207,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5562,"private_eq":0.6702},"64":{"raw":0.5739,"skill":0.4674,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.3901,"private_eq":0.5168}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"llm2jev 0.6.1 (chat prompt, thinking off, T=1.036) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","v03":{"engine":"jpt-4b","pub":42.82,"same_skills":40.11,"new_domains":29.93,"v":39.56,"lab":false,"jev":false,"rank_v":137,"rank_pub":141,"name":"JPT-4B Qwen3.5-4B LoRA","eq":{"S":39.52,"O":39.61},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":23.71,"new_domains":15.84}}},{"engine":"nimble-v2","name":"Bespoke Nimble 9B v2","long_name":"Bespoke Nimble 9B v2","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":173188512,"model_url":"https://huggingface.co/bespokelabs/Bespoke-Nimble-9B-v2","code_url":"https://github.com/bespokelabsai/nimble","base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":"bespokelabs/Bespoke-Nimble-9B-v2","served_checkpoint":null,"org":{"id":"bespokelabs","name":"Bespoke Labs","logo":"data/orgs/bespokelabs.png"}},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.6425,"conf":0.6479,"ece":0.024,"brier":0.4671,"over95":0.0019,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0464,"conf":0.066,"acc":0.1059},{"lo":0.1,"w":0.0274,"conf":0.1453,"acc":0.2065},{"lo":0.2,"w":0.032,"conf":0.2583,"acc":0.3046},{"lo":0.3,"w":0.0625,"conf":0.3532,"acc":0.3399},{"lo":0.4,"w":0.0698,"conf":0.4535,"acc":0.4014},{"lo":0.5,"w":0.1575,"conf":0.5492,"acc":0.5765},{"lo":0.6,"w":0.1215,"conf":0.6511,"acc":0.6422},{"lo":0.7,"w":0.1332,"conf":0.7523,"acc":0.7196},{"lo":0.8,"w":0.1762,"conf":0.8557,"acc":0.8434},{"lo":0.9,"w":0.1736,"conf":0.9348,"acc":0.9201}],"areas":{"knowledge":{"ece":0.0354,"acc":0.5899,"conf":0.597},"language":{"ece":0.0307,"acc":0.7503,"conf":0.7524},"retrieval":{"ece":0.0992,"acc":0.5737,"conf":0.5898},"tools":{"ece":0.0441,"acc":0.8315,"conf":0.7935},"arts":{"ece":0.0532,"acc":0.506,"conf":0.5309}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":77.3,"p95":621.4,"mean":982.3},"results":{"1":{"score":0.9156,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":169.9},"2":{"score":0.5795,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":31412.8},"3":{"score":0.8051,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6938,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":143.1},"5":{"score":0.3084,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":254.4},"6":{"score":0.7831,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":431.8},"9":{"score":0.0227,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5992,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":98},"11":{"score":0.7032,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6819,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":74.1},"20":{"score":0.8666,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.6},"21":{"score":0.5753,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":72.9},"22":{"score":0.1465,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":561.6},"23":{"score":0.543,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.7},"24":{"score":0.7724,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":74.3},"25":{"score":0.4388,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9811,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":73.7},"27":{"score":0.9411,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":73.9},"28":{"score":0.7561,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":72.2},"29":{"score":0.8499,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":76.3},"30":{"score":0.4329,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1362,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":106.9},"32":{"score":0.5638,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2352,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":713.8},"36":{"score":0.2814,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4553,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.2},"38":{"score":0.2314,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":46407.9},"39":{"score":0.6547,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5372,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":74.1,"tracks":[{"track":"A · Arabic","score":0.1293,"headline":false},{"track":"A · English","score":0.5372,"headline":true},{"track":"C · Arabic pairs","score":0.685,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3913,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":74.8},"42":{"score":0.7633,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":76.1},"43":{"score":0.5632,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6692,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74},"45":{"score":0.0858,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4039,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":93.9},"56":{"score":0.656,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":762.1,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.5172,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":79.1,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6744,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":74.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.6507,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":79.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6295,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":77.9,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7267,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":85.3,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5568,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":54.42,"balanced_skill":42.06,"breadth_skill":38.84,"public_skill":39.9},"frozen_scores":{"balanced_raw":54.42,"balanced_skill":39.9,"breadth_skill":38.84},"categories":[{"id":"knowledge","raw":0.4639,"skill":0.3048,"combined":0.2661,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6138,"skill":0.4431,"combined":0.4437,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4971,"skill":0.3493,"combined":0.3605,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6285,"skill":0.5921,"combined":0.5297,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5114,"skill":0.2756,"combined":0.3418,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9156,"skill":0.886,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7421,"private_eq":0.8104},"2":{"raw":0.5795,"skill":0.5144,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6438,"private_eq":0.6942},"3":{"raw":0.8051,"skill":0.8013,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7281,"private_eq":0.7353},"4":{"raw":0.6938,"skill":0.6899,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7304,"private_eq":0.7359},"5":{"raw":0.3084,"skill":0.3042,"coverage":1,"random":0.006,"tracks":[],"combined":0.112,"private_eq":0.1264},"6":{"raw":0.7832,"skill":0.4907,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7612,"headline":false},{"track":"RouterBench-5shot","score":0.8051,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0227,"skill":0.0227,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0.0025},"11":{"raw":0.7032,"skill":0.5708,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6819,"skill":0.5235,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4945,"private_eq":0.6657},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8624,"skill":0.7248,"coverage":1,"random":0.5,"tracks":[],"combined":0.7409,"private_eq":0.8714},"21":{"raw":0.5753,"skill":0.1507,"coverage":1,"random":0.5,"tracks":[],"combined":0.156,"private_eq":0.5823},"22":{"raw":0.1363,"skill":0.1295,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1344,"private_eq":0.1499},"23":{"raw":0.5339,"skill":0.0677,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7844,"skill":0.7125,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4388,"skill":0.2517,"coverage":1,"random":0.25,"tracks":[],"combined":0.1616,"private_eq":0.3776},"28":{"raw":0.7561,"skill":0.5122,"coverage":1,"random":0.5,"tracks":[],"combined":0.4109,"private_eq":0.7083},"29":{"raw":0.8499,"skill":0.7999,"coverage":1,"random":0.25,"tracks":[],"combined":0.7009,"private_eq":0.7775},"30":{"raw":0.4329,"skill":0.3225,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5732,"random":0.25},{"track":"10-choice","raw":0.2926,"random":0.1}],"combined":0.5138,"private_eq":0.6026},"31":{"raw":0.1362,"skill":0.0592,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0582,"private_eq":0.1443},"32":{"raw":0.5638,"skill":0.3065,"coverage":1,"random":0.371,"tracks":[],"combined":0.1112,"private_eq":0.4467},"33":{"raw":0.2352,"skill":0.225,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1893,"private_eq":0.208},"36":{"raw":0.2814,"skill":0.1871,"coverage":1,"random":0.116,"tracks":[],"combined":0.2967,"private_eq":0.3845},"37":{"raw":0.4553,"skill":0.3168,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3216,"private_eq":0.4645},"38":{"raw":0.2314,"skill":0.2068,"coverage":1,"random":0.031,"tracks":[],"combined":0.302,"private_eq":0.3304},"39":{"raw":0.6547,"skill":0.4921,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3744,"private_eq":0.5788},"40":{"raw":0.5372,"skill":0.4047,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1293,"headline":false},{"track":"A · English","score":0.5372,"headline":true},{"track":"C · Arabic pairs","score":0.685,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}],"combined":0.4421,"private_eq":0.5705},"41":{"raw":0.3913,"skill":0.087,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2089,"private_eq":0.4779},"42":{"raw":0.7633,"skill":0.5395,"coverage":1,"random":0.486,"tracks":[],"combined":0.8231,"private_eq":0.9096},"43":{"raw":0.5632,"skill":0.3069,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3822,"private_eq":0.6144},"44":{"raw":0.6692,"skill":0.3384,"coverage":1,"random":0.5,"tracks":[],"combined":0.2515,"private_eq":0.6295},"45":{"raw":0.0858,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4039,"skill":0.1349,"coverage":1,"random":0.311,"tracks":[],"combined":0.1523,"private_eq":0.4219},"56":{"raw":0.656,"skill":0.312,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4558,"private_eq":0.7305},"57":{"raw":0.5172,"skill":0.457,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3977,"private_eq":0.4697},"58":{"raw":0.6744,"skill":0.528,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.17,"private_eq":0.4332},"59":{"raw":0.6507,"skill":0.2758,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.1424,"private_eq":0.5906},"61":{"raw":0.6295,"skill":0.259,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2351,"private_eq":0.6214},"62":{"raw":0.7267,"skill":0.6356,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4603,"private_eq":0.5991},"64":{"raw":0.5568,"skill":0.446,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5259,"private_eq":0.6242}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"ParallelScorer.score (one full-prompt forward per question), author pins torch 2.8.0+cu128 / transformers 5.17.0 / fla 0.5.2 / causal-conv1d 1.7.0, Triton autotune cache from the board HF Job","v03":{"engine":"nimble-v2","pub":39.9,"same_skills":39.51,"new_domains":35.74,"v":42.06,"lab":false,"jev":false,"rank_v":125,"rank_pub":155,"name":"Bespoke Nimble 9B v2","eq":{"S":38.91,"O":46.78},"rank_tie":28,"tied":true,"parts":{"public":0,"same_skills":23.35,"new_domains":18.71}}},{"engine":"hopper-g-1.2","name":"Hopper (G) 1.2","long_name":"Hopper (G) 1.2 Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":129927008,"model_url":"https://huggingface.co/HopitAI/hopper-g","code_url":"https://github.com/hopit-ai/hopper","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"HopitAI/hopper-g","served_checkpoint":null,"org":{"id":"HopitAI","name":"HopitAI","logo":"data/orgs/HopitAI.png"}},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.651,"conf":0.7437,"ece":0.0927,"brier":0.4672,"over95":0.0021,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0004,"conf":0.0937,"acc":0},{"lo":0.1,"w":0.0086,"conf":0.1646,"acc":0.0574},{"lo":0.2,"w":0.0249,"conf":0.2509,"acc":0.1464},{"lo":0.3,"w":0.0389,"conf":0.3478,"acc":0.2482},{"lo":0.4,"w":0.0634,"conf":0.4538,"acc":0.2713},{"lo":0.5,"w":0.1087,"conf":0.5493,"acc":0.4367},{"lo":0.6,"w":0.1139,"conf":0.6509,"acc":0.5037},{"lo":0.7,"w":0.1206,"conf":0.7519,"acc":0.6122},{"lo":0.8,"w":0.216,"conf":0.8544,"acc":0.7424},{"lo":0.9,"w":0.3046,"conf":0.9343,"acc":0.9225}],"areas":{"knowledge":{"ece":0.1264,"acc":0.553,"conf":0.6793},"language":{"ece":0.0313,"acc":0.7484,"conf":0.778},"retrieval":{"ece":0.1105,"acc":0.6944,"conf":0.8042},"tools":{"ece":0.039,"acc":0.8847,"conf":0.872},"arts":{"ece":0.1815,"acc":0.5166,"conf":0.6981}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":23.3,"p95":238.2,"mean":105.1},"results":{"1":{"score":0.9209,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":94.5},"2":{"score":0.6195,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":798.6},"3":{"score":0.7559,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7255,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":114.1},"5":{"score":0.7957,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":209.1},"6":{"score":0.7986,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":186.4},"9":{"score":0.2273,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6438,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":65.4},"11":{"score":0.7073,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5995,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":14.5},"20":{"score":0.8322,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.5},"21":{"score":0.6058,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.3},"22":{"score":0.068,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":741.7},"23":{"score":0.6064,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.1},"24":{"score":0.7389,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.3},"25":{"score":0.3776,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9739,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":15.2},"27":{"score":0.9275,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":14.6},"28":{"score":0.6914,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":13.3},"29":{"score":0.8451,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":21.4},"30":{"score":0.2904,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1392,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":100.6},"32":{"score":0.5505,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2952,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":202.5},"36":{"score":0.4244,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3951,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.3},"38":{"score":0.1645,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":889.5},"39":{"score":0.8319,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4371,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":13.4,"tracks":[{"track":"A · Arabic","score":0.3065,"headline":false},{"track":"A · English","score":0.4371,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4432,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":18.7},"42":{"score":0.7857,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":33.2},"43":{"score":0.4719,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6302,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.4},"45":{"score":0.0918,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4272,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":42.8},"56":{"score":0.6135,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":185.1,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4894,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":21.3,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6414,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":15.2,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.6957,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":36.6,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6455,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":32.2,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.6117,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":43.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5758,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":56,"balanced_skill":40.17,"breadth_skill":39.56,"public_skill":41.15},"frozen_scores":{"balanced_raw":56,"balanced_skill":41.15,"breadth_skill":39.56},"categories":[{"id":"knowledge","raw":0.4273,"skill":0.2459,"combined":0.2645,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6169,"skill":0.4567,"combined":0.3983,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6044,"skill":0.4542,"combined":0.4655,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6427,"skill":0.6006,"combined":0.622,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5162,"skill":0.2917,"combined":0.3401,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9209,"skill":0.8932,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8717,"private_eq":0.9053},"2":{"raw":0.6195,"skill":0.5606,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6319,"private_eq":0.684},"3":{"raw":0.7559,"skill":0.7512,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7858,"private_eq":0.7913},"4":{"raw":0.7255,"skill":0.722,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7377,"private_eq":0.743},"5":{"raw":0.7957,"skill":0.7945,"coverage":1,"random":0.006,"tracks":[],"combined":0.6258,"private_eq":0.6313},"6":{"raw":0.7986,"skill":0.5252,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7848,"headline":false},{"track":"RouterBench-5shot","score":0.8124,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.2273,"skill":0.2273,"coverage":1,"random":0,"tracks":[],"combined":0.1817,"private_eq":0.19},"11":{"raw":0.7073,"skill":0.5768,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5995,"skill":0.4001,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4469,"private_eq":0.6343},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8352,"skill":0.6703,"coverage":1,"random":0.5,"tracks":[],"combined":0.7336,"private_eq":0.8678},"21":{"raw":0.6058,"skill":0.2116,"coverage":1,"random":0.5,"tracks":[],"combined":0.2064,"private_eq":0.6072},"22":{"raw":0.0697,"skill":0.0624,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0829,"private_eq":0.0994},"23":{"raw":0.5836,"skill":0.1673,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7487,"skill":0.6649,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3776,"skill":0.1701,"coverage":1,"random":0.25,"tracks":[],"combined":0.1013,"private_eq":0.3329},"28":{"raw":0.6914,"skill":0.3828,"coverage":1,"random":0.5,"tracks":[],"combined":0.3553,"private_eq":0.6808},"29":{"raw":0.8451,"skill":0.7935,"coverage":1,"random":0.25,"tracks":[],"combined":0.7306,"private_eq":0.7995},"30":{"raw":0.2904,"skill":0.144,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3927,"random":0.25},{"track":"10-choice","raw":0.188,"random":0.1}],"combined":0.3508,"private_eq":0.4697},"31":{"raw":0.1392,"skill":0.0625,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0918,"private_eq":0.1748},"32":{"raw":0.5505,"skill":0.2854,"coverage":1,"random":0.371,"tracks":[],"combined":0.1577,"private_eq":0.4756},"33":{"raw":0.2952,"skill":0.2858,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2766,"private_eq":0.2932},"36":{"raw":0.4244,"skill":0.3488,"coverage":1,"random":0.116,"tracks":[],"combined":0.3159,"private_eq":0.4013},"37":{"raw":0.3951,"skill":0.2412,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3688,"private_eq":0.5017},"38":{"raw":0.1645,"skill":0.1378,"coverage":1,"random":0.031,"tracks":[],"combined":0.2446,"private_eq":0.2754},"39":{"raw":0.8319,"skill":0.7528,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5151,"private_eq":0.6734},"40":{"raw":0.4371,"skill":0.2758,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3065,"headline":false},{"track":"A · English","score":0.4371,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}],"combined":0.227,"private_eq":0.4052},"41":{"raw":0.4432,"skill":0.1649,"coverage":1,"random":0.3333,"tracks":[],"combined":0.206,"private_eq":0.476},"42":{"raw":0.7857,"skill":0.5831,"coverage":1,"random":0.486,"tracks":[],"combined":0.496,"private_eq":0.7434},"43":{"raw":0.4719,"skill":0.1622,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3738,"private_eq":0.6092},"44":{"raw":0.6302,"skill":0.2604,"coverage":1,"random":0.5,"tracks":[],"combined":0.4336,"private_eq":0.7195},"45":{"raw":0.0918,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4272,"skill":0.1687,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.6135,"skill":0.227,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2828,"private_eq":0.645},"57":{"raw":0.4894,"skill":0.4257,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3844,"private_eq":0.458},"58":{"raw":0.6414,"skill":0.4802,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1363,"private_eq":0.4102},"59":{"raw":0.6957,"skill":0.3691,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.2437,"private_eq":0.6389},"61":{"raw":0.6455,"skill":0.291,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4052,"private_eq":0.7055},"62":{"raw":0.6117,"skill":0.4823,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5562,"private_eq":0.6702},"64":{"raw":0.5758,"skill":0.4697,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4968,"private_eq":0.6012}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"hopper-decisions 1.1.1 at tag g-1.2.0 with the HopitAI/hopper-g adapter (tournament shortlist for choice questions over 26 options, package default), Decider(cuda_graphs=True) package opt-in CUDA graphs per length bucket, eager outside captured buckets; author pins torch 2.8.0+cu128 / transformers 5.17.0 / fla 0.5.2 / causal-conv1d 1.7.0; eager variant kept as latency-v1-full.eager","v03":{"engine":"hopper-g-1.2","pub":41.15,"same_skills":42.37,"new_domains":28.39,"v":40.17,"lab":false,"jev":false,"rank_v":133,"rank_pub":150,"name":"Hopper (G) 1.2 Qwen3.5-4B LoRA","eq":{"S":41.8,"O":37.71},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":25.08,"new_domains":15.08}}},{"engine":"razorback-one-read-3e296f08","name":"razorback16 openjev diffusiongemma (NVFP4, vLLM)","long_name":"razorback16 diffgemma","variant":"one read, NVFP4","meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/vllm-project/vllm/pull/57250","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"nvidia/diffusiongemma-26B-A4B-it-NVFP4"},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.6205,"conf":0.8525,"ece":0.232,"brier":0.6004,"over95":0.1142,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0026,"conf":0.1666,"acc":0.081},{"lo":0.2,"w":0.0139,"conf":0.2539,"acc":0.1579},{"lo":0.3,"w":0.0206,"conf":0.3586,"acc":0.1896},{"lo":0.4,"w":0.0336,"conf":0.4503,"acc":0.2936},{"lo":0.5,"w":0.0698,"conf":0.5486,"acc":0.3721},{"lo":0.6,"w":0.0679,"conf":0.6504,"acc":0.455},{"lo":0.7,"w":0.0784,"conf":0.7516,"acc":0.4817},{"lo":0.8,"w":0.1138,"conf":0.8561,"acc":0.5411},{"lo":0.9,"w":0.5996,"conf":0.9795,"acc":0.7474}],"areas":{"knowledge":{"ece":0.223,"acc":0.5878,"conf":0.8109},"language":{"ece":0.1665,"acc":0.7462,"conf":0.9127},"retrieval":{"ece":0.4133,"acc":0.4638,"conf":0.8771},"tools":{"ece":0.1201,"acc":0.7218,"conf":0.8355},"arts":{"ece":0.2959,"acc":0.5303,"conf":0.8262}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":37.7,"p95":117.8,"mean":80.7},"results":{"1":{"score":0.9569,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":54.8},"2":{"score":0.4657,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":296},"3":{"score":0.5728,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6903,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":70.4},"5":{"score":0.0201,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":98.3},"6":{"score":0.7988,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":101.8},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3397,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":62.6},"11":{"score":0.5409,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5693,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":33.7},"20":{"score":0.8906,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.3},"21":{"score":0.6024,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":31.1},"22":{"score":0.112,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":127.4},"23":{"score":0.5956,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.3},"24":{"score":0.7775,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":32.6},"25":{"score":0.4796,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9806,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":31.7},"27":{"score":0.936,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":32.1},"28":{"score":0.6993,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":30.8},"29":{"score":0.8112,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":36.1},"30":{"score":0.2043,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1402,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.4},"32":{"score":0.6303,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2994,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":62.4},"36":{"score":0.2319,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5629,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.9},"38":{"score":0.1475,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":227.4},"39":{"score":0.8503,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5638,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":30.7,"tracks":[{"track":"A · Arabic","score":0.3514,"headline":false},{"track":"A · English","score":0.5638,"headline":true},{"track":"C · Arabic pairs","score":0.82,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5923,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":34},"42":{"score":0.7406,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":53.7},"43":{"score":0.4719,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.623,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.2},"45":{"score":0.1018,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4761,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":57.3},"56":{"score":0.813,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":42.6,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4176,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":35.7,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6621,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33.8,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7081,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":56.7,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6837,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":39.4,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5181,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":55.2,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5758,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":51.79,"balanced_skill":37.6,"breadth_skill":36.97,"public_skill":37.75},"frozen_scores":{"balanced_raw":51.79,"balanced_skill":37.75,"breadth_skill":36.97},"categories":[{"id":"knowledge","raw":0.4335,"skill":0.2569,"combined":0.2293,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.617,"skill":0.455,"combined":0.4625,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4834,"skill":0.3702,"combined":0.3763,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5221,"skill":0.4719,"combined":0.5395,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5411,"skill":0.3307,"combined":0.3587,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9569,"skill":0.9418,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.4657,"skill":0.3829,"coverage":1,"random":0.1341,"tracks":[],"combined":0.5711,"private_eq":0.632},"3":{"raw":0.5728,"skill":0.5646,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6284,"private_eq":0.6386},"4":{"raw":0.6903,"skill":0.6863,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7887,"private_eq":0.7928},"5":{"raw":0.0201,"skill":0.0142,"coverage":1,"random":0.006,"tracks":[],"combined":0,"private_eq":0.008},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0.0025},"11":{"raw":0.5409,"skill":0.3361,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5693,"skill":0.3549,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4862,"private_eq":0.6602},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8915,"skill":0.783,"coverage":1,"random":0.5,"tracks":[],"combined":0.8347,"private_eq":0.9178},"21":{"raw":0.6024,"skill":0.2047,"coverage":1,"random":0.5,"tracks":[],"combined":0.1661,"private_eq":0.5873},"22":{"raw":0.1073,"skill":0.1003,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1462,"private_eq":0.1615},"23":{"raw":0.5934,"skill":0.1867,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7849,"skill":0.7132,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"combined":0.1195,"private_eq":0.3464},"28":{"raw":0.6993,"skill":0.3986,"coverage":1,"random":0.5,"tracks":[],"combined":0.5268,"private_eq":0.7656},"29":{"raw":0.8112,"skill":0.7483,"coverage":1,"random":0.25,"tracks":[],"combined":0.7356,"private_eq":0.8032},"30":{"raw":0.2043,"skill":0.0374,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2934,"random":0.25},{"track":"10-choice","raw":0.1152,"random":0.1}],"combined":0.1255,"private_eq":0.2859},"31":{"raw":0.1402,"skill":0.0635,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.6303,"skill":0.4122,"coverage":1,"random":0.371,"tracks":[],"combined":0.1654,"private_eq":0.4804},"33":{"raw":0.2994,"skill":0.2901,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2612,"private_eq":0.2782},"36":{"raw":0.2319,"skill":0.1311,"coverage":1,"random":0.116,"tracks":[],"combined":0.1488,"private_eq":0.2552},"37":{"raw":0.5629,"skill":0.4518,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4559,"private_eq":0.5703},"38":{"raw":0.1475,"skill":0.1202,"coverage":1,"random":0.031,"tracks":[],"combined":0.2056,"private_eq":0.238},"39":{"raw":0.8503,"skill":0.7798,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5165,"private_eq":0.6743},"40":{"raw":0.5638,"skill":0.4388,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3514,"headline":false},{"track":"A · English","score":0.5638,"headline":true},{"track":"C · Arabic pairs","score":0.82,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}],"combined":0.3946,"private_eq":0.534},"41":{"raw":0.5923,"skill":0.3885,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4838,"private_eq":0.6591},"42":{"raw":0.7406,"skill":0.4953,"coverage":1,"random":0.486,"tracks":[],"combined":0.7643,"private_eq":0.8797},"43":{"raw":0.4719,"skill":0.1622,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2108,"private_eq":0.5076},"44":{"raw":0.623,"skill":0.246,"coverage":1,"random":0.5,"tracks":[],"combined":0.2515,"private_eq":0.6295},"45":{"raw":0.1018,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4761,"skill":0.2397,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.813,"skill":0.626,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5078,"private_eq":0.7562},"57":{"raw":0.4176,"skill":0.345,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3512,"private_eq":0.4288},"58":{"raw":0.6621,"skill":0.5102,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1437,"private_eq":0.4153},"59":{"raw":0.7081,"skill":0.3948,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.0541,"private_eq":0.5485},"61":{"raw":0.6837,"skill":0.3674,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4052,"private_eq":0.7055},"62":{"raw":0.5181,"skill":0.3575,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5013,"private_eq":0.6295},"64":{"raw":0.5758,"skill":0.4697,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.458,"private_eq":0.5705}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"openjev 0.4 API over vLLM 0.29.1rc1.dev551 NVFP4, async scheduling, prefix caching, TRITON_ATTN, one read per question group, one sequential HTTP client","v03":{"engine":"razorback-one-read-3e296f08","pub":37.75,"same_skills":39.74,"new_domains":26.43,"v":37.6,"lab":false,"jev":false,"rank_v":142,"rank_pub":166,"name":"razorback16 diffgemma","eq":{"S":39.15,"O":35.28},"rank_tie":32,"tied":true,"parts":{"public":0,"same_skills":23.49,"new_domains":14.11}}},{"engine":"kev-4b-r10","name":"Kev 4B r10","long_name":"Kev 4B r10","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/jaredpalmer/kev-4b","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6252,"conf":0.6268,"ece":0.032,"brier":0.4847,"over95":0.0078,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0624,"conf":0.0463,"acc":0.0657},{"lo":0.1,"w":0.026,"conf":0.1555,"acc":0.465},{"lo":0.2,"w":0.0312,"conf":0.2556,"acc":0.3893},{"lo":0.3,"w":0.073,"conf":0.3498,"acc":0.355},{"lo":0.4,"w":0.0818,"conf":0.4496,"acc":0.4579},{"lo":0.5,"w":0.16,"conf":0.5492,"acc":0.5537},{"lo":0.6,"w":0.1382,"conf":0.6482,"acc":0.6411},{"lo":0.7,"w":0.1172,"conf":0.749,"acc":0.714},{"lo":0.8,"w":0.1155,"conf":0.8536,"acc":0.8097},{"lo":0.9,"w":0.1949,"conf":0.9534,"acc":0.9194}],"areas":{"knowledge":{"ece":0.024,"acc":0.579,"conf":0.5915},"language":{"ece":0.0479,"acc":0.7181,"conf":0.7465},"retrieval":{"ece":0.1671,"acc":0.6646,"conf":0.515},"tools":{"ece":0.0652,"acc":0.7633,"conf":0.8043},"arts":{"ece":0.0273,"acc":0.445,"conf":0.4716}}},"gaps":null,"latency":{"median":59.2,"p95":null,"mean":74.7},"results":{"1":{"score":0.9309,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6098,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.5335,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8287,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.75,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.8,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.1477,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4776,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.6436,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4813,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.6514,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.5441,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.0115,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.5788,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.7102,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.3776,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9722,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9027,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.7024,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.7892,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.4647,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0886,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.5625,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2885,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.3895,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3972,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.0978,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.871,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4684,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.5236,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.7458,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.4807,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6308,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1018,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.42,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.5525,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.5068,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.6248,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.6752,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.5968,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.667,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.5227,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9309,"skill":0.9068,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7831,"private_eq":0.8404},"2":{"raw":0.6098,"skill":0.5493,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6473,"private_eq":0.6972},"3":{"raw":0.5335,"skill":0.5245,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4237,"private_eq":0.44},"4":{"raw":0.8287,"skill":0.8265,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7105,"private_eq":0.7164},"5":{"raw":0.75,"skill":0.7485,"coverage":1,"random":0.006,"tracks":[],"combined":0.6005,"private_eq":0.6064},"6":{"raw":0.8,"skill":0.5311,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1477,"skill":0.1477,"coverage":1,"random":0,"tracks":[],"combined":0.1159,"private_eq":0.125},"10":{"raw":0.4776,"skill":0.1308,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6436,"skill":0.4846,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4813,"skill":0.223,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4021,"private_eq":0.6047},"20":{"raw":0.6543,"skill":0.3087,"coverage":1,"random":0.5,"tracks":[],"combined":0.5311,"private_eq":0.7677},"21":{"raw":0.5441,"skill":0.0883,"coverage":1,"random":0.5,"tracks":[],"combined":0.0352,"private_eq":0.5226},"22":{"raw":0.0112,"skill":0.0035,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0168,"private_eq":0.0346},"23":{"raw":0.5559,"skill":0.1118,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7102,"skill":0.6136,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3776,"skill":0.1701,"coverage":1,"random":0.25,"tracks":[],"combined":0.1111,"private_eq":0.3402},"26":{"raw":0.9722,"skill":0.9629,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9027,"skill":0.8702,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7024,"skill":0.4048,"coverage":1,"random":0.5,"tracks":[],"combined":0.3453,"private_eq":0.6759},"29":{"raw":0.7892,"skill":0.7189,"coverage":1,"random":0.25,"tracks":[],"combined":0.7059,"private_eq":0.7812},"30":{"raw":0.4647,"skill":0.3557,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5534,"random":0.25},{"track":"10-choice","raw":0.376,"random":0.1}],"combined":0.5712,"private_eq":0.6494},"31":{"raw":0.0886,"skill":0.0073,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0193,"private_eq":0.109},"32":{"raw":0.5625,"skill":0.3045,"coverage":1,"random":0.371,"tracks":[],"combined":0.0493,"private_eq":0.4082},"33":{"raw":0.2885,"skill":0.2791,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3022,"private_eq":0.3182},"36":{"raw":0.3895,"skill":0.3093,"coverage":1,"random":0.116,"tracks":[],"combined":0.3178,"private_eq":0.4029},"37":{"raw":0.3972,"skill":0.244,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3938,"private_eq":0.5214},"38":{"raw":0.0978,"skill":0.0689,"coverage":1,"random":0.031,"tracks":[],"combined":0.1341,"private_eq":0.1695},"39":{"raw":0.871,"skill":0.8103,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5139,"private_eq":0.6726},"40":{"raw":0.4684,"skill":0.3161,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3061,"headline":false},{"track":"A · English","score":0.4684,"headline":true},{"track":"C · Arabic pairs","score":0.475,"headline":false},{"track":"C · English pairs","score":0.77,"headline":false}],"combined":0.1123,"private_eq":0.3171},"41":{"raw":0.5236,"skill":0.2854,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2335,"private_eq":0.4941},"42":{"raw":0.7458,"skill":0.5054,"coverage":1,"random":0.486,"tracks":[],"combined":0.4559,"private_eq":0.723},"43":{"raw":0.4807,"skill":0.1761,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3782,"private_eq":0.6119},"44":{"raw":0.6308,"skill":0.2616,"coverage":1,"random":0.5,"tracks":[],"combined":0.3425,"private_eq":0.6745},"45":{"raw":0.1018,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.42,"skill":0.1583,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.5525,"skill":0.105,"coverage":1,"random":0.5,"tracks":[],"combined":0.0923,"private_eq":0.5508},"57":{"raw":0.5068,"skill":0.4453,"coverage":1,"random":0.1109,"tracks":[],"combined":0.1155,"private_eq":0.2217},"58":{"raw":0.6248,"skill":0.4562,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1374,"private_eq":0.411},"59":{"raw":0.6752,"skill":0.3266,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2103,"private_eq":0.623},"61":{"raw":0.5968,"skill":0.1936,"coverage":1,"random":0.5,"tracks":[],"combined":0.3089,"private_eq":0.6579},"62":{"raw":0.667,"skill":0.556,"coverage":1,"random":0.25,"tracks":[],"combined":0.4946,"private_eq":0.6245},"64":{"raw":0.5227,"skill":0.4034,"coverage":1,"random":0.2,"tracks":[],"combined":0.3706,"private_eq":0.5014}},"coverage":1,"pending":0,"scores":{"balanced_raw":53.8,"balanced_skill":34.93,"breadth_skill":36.46,"public_skill":37.95},"frozen_scores":{"balanced_raw":53.8,"balanced_skill":37.95,"breadth_skill":36.46},"categories":[{"id":"knowledge","raw":0.441,"skill":0.263,"combined":0.2328,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5904,"skill":0.4178,"combined":0.3559,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5922,"skill":0.4248,"combined":0.4166,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5892,"skill":0.5501,"combined":0.5011,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4514,"skill":0.179,"combined":0.2299,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"kev-4b-r10","pub":37.95,"same_skills":36.17,"new_domains":25.4,"v":34.93,"lab":false,"jev":false,"rank_v":148,"rank_pub":165,"name":"Kev 4B r10","eq":{"S":35.54,"O":34.01},"rank_tie":34,"tied":true,"parts":{"public":0,"same_skills":21.32,"new_domains":13.6}},"notes":["Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."]},{"engine":"lev","name":"lev","long_name":"Interfaze lev Qwen3.5-4B LoRA","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":184601885,"model_url":"https://huggingface.co/interfaze-ai/lev","code_url":"https://github.com/Abhinavexists/lev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"interfaze-ai/lev","served_checkpoint":null,"org":{"id":"interfaze-ai","name":"Interfaze","logo":"data/orgs/interfaze-ai.png"}},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.6365,"conf":0.6899,"ece":0.0638,"brier":0.4905,"over95":0.0123,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.032,"conf":0.0479,"acc":0.078},{"lo":0.1,"w":0.0261,"conf":0.1494,"acc":0.1772},{"lo":0.2,"w":0.0256,"conf":0.2554,"acc":0.2821},{"lo":0.3,"w":0.0595,"conf":0.3597,"acc":0.3631},{"lo":0.4,"w":0.0724,"conf":0.4517,"acc":0.4413},{"lo":0.5,"w":0.1393,"conf":0.5441,"acc":0.5627},{"lo":0.6,"w":0.0971,"conf":0.6497,"acc":0.5451},{"lo":0.7,"w":0.1044,"conf":0.7524,"acc":0.6601},{"lo":0.8,"w":0.1592,"conf":0.8537,"acc":0.735},{"lo":0.9,"w":0.2842,"conf":0.9513,"acc":0.8839}],"areas":{"knowledge":{"ece":0.1221,"acc":0.5537,"conf":0.6758},"language":{"ece":0.1019,"acc":0.7001,"conf":0.7358},"retrieval":{"ece":0.0647,"acc":0.672,"conf":0.6628},"tools":{"ece":0.047,"acc":0.8834,"conf":0.8426},"arts":{"ece":0.0521,"acc":0.5274,"conf":0.5684}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":70.8,"p95":687.7,"mean":399.5},"results":{"1":{"score":0.902,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":151.1},"2":{"score":0.6223,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":11519.9},"3":{"score":0.7913,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8675,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":103.9,"wrong_rows":22,"contamination":"22 BANKING77 test items found in lev's training records count as wrong"},"5":{"score":0.9037,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":176,"wrong_rows":12,"contamination":"12 CLINC150+OOS test items found in lev's training records count as wrong"},"6":{"score":0.7986,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":536.5},"9":{"score":0.1932,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3398,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":122.4},"11":{"score":0.7043,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6445,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":69.5,"wrong_rows":1,"contamination":"1 ANLI test items found in lev's training records count as wrong"},"20":{"score":0.8766,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.4},"21":{"score":0.6008,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":68.8},"22":{"score":0.054,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":750.8},"23":{"score":0.6224,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.7},"24":{"score":0.7295,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":69.5},"25":{"score":0.4031,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9752,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":69.1,"wrong_rows":9,"contamination":"9 ARC-Easy test items found in lev's training records count as wrong"},"27":{"score":0.9326,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":69.1},"28":{"score":0.6685,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":68.8},"29":{"score":0.8118,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":70.4,"wrong_rows":1,"contamination":"1 HellaSwag test items found in lev's training records count as wrong"},"30":{"score":0.2915,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1432,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101},"32":{"score":0.5359,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1824,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":783.1},"36":{"score":0.4398,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4594,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.9},"38":{"score":0.0711,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1531.2},"39":{"score":0.7395,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":68.6,"tracks":[{"track":"A · Arabic","score":0.2566,"headline":false},{"track":"A · English","score":0.4,"headline":true},{"track":"C · Arabic pairs","score":0.6,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4836,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":69.8},"42":{"score":0.7606,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":71.3},"43":{"score":0.5035,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5966,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.5},"45":{"score":0.1238,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4624,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":94.9},"56":{"score":0.5055,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":194.7,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4866,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":70.1,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6134,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":69.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.0832,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":71.5,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6338,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":71.1,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5225,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":80.8,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5398,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":54.04,"balanced_skill":35.47,"breadth_skill":37.62,"public_skill":39.2},"frozen_scores":{"balanced_raw":54.04,"balanced_skill":39.2,"breadth_skill":37.62},"categories":[{"id":"knowledge","raw":0.4173,"skill":0.2278,"combined":0.2083,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5308,"skill":0.3913,"combined":0.2949,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6442,"skill":0.4792,"combined":0.4232,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6241,"skill":0.5752,"combined":0.5798,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5224,"skill":0.3081,"combined":0.3943,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.902,"skill":0.8677,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8103,"private_eq":0.8603},"2":{"raw":0.6223,"skill":0.5638,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6585,"private_eq":0.7068},"3":{"raw":0.7913,"skill":0.7873,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7858,"private_eq":0.7913},"4":{"raw":0.8675,"skill":0.8658,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7901,"private_eq":0.7941},"5":{"raw":0.9037,"skill":0.9031,"coverage":1,"random":0.006,"tracks":[],"combined":0.6193,"private_eq":0.6249},"6":{"raw":0.7986,"skill":0.5252,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7849,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.1932,"skill":0.1932,"coverage":1,"random":0,"tracks":[],"combined":0.0831,"private_eq":0.0925},"11":{"raw":0.7043,"skill":0.5724,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6445,"skill":0.4675,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3531,"private_eq":0.5724},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8753,"skill":0.7506,"coverage":1,"random":0.5,"tracks":[],"combined":0.8782,"private_eq":0.9393},"21":{"raw":0.6008,"skill":0.2017,"coverage":1,"random":0.5,"tracks":[],"combined":0.2365,"private_eq":0.6221},"22":{"raw":0.0524,"skill":0.045,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1155,"private_eq":0.1314},"23":{"raw":0.6036,"skill":0.2071,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7444,"skill":0.6592,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4031,"skill":0.2041,"coverage":1,"random":0.25,"tracks":[],"combined":0.135,"private_eq":0.3579},"28":{"raw":0.6685,"skill":0.337,"coverage":1,"random":0.5,"tracks":[],"combined":0.315,"private_eq":0.6609},"29":{"raw":0.8118,"skill":0.7491,"coverage":1,"random":0.25,"tracks":[],"combined":0.6614,"private_eq":0.7482},"30":{"raw":0.2915,"skill":0.144,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3813,"random":0.25},{"track":"10-choice","raw":0.2017,"random":0.1}],"combined":0.334,"private_eq":0.456},"31":{"raw":0.1432,"skill":0.0668,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0821,"private_eq":0.166},"32":{"raw":0.5359,"skill":0.2622,"coverage":1,"random":0.371,"tracks":[],"combined":0.181,"private_eq":0.4901},"33":{"raw":0.1824,"skill":0.1715,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.4398,"skill":0.3663,"coverage":1,"random":0.116,"tracks":[],"combined":0.284,"private_eq":0.3734},"37":{"raw":0.4594,"skill":0.3219,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3841,"private_eq":0.5137},"38":{"raw":0.0711,"skill":0.0414,"coverage":1,"random":0.031,"tracks":[],"combined":0.0959,"private_eq":0.1329},"39":{"raw":0.7395,"skill":0.6169,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3009,"private_eq":0.5294},"40":{"raw":0.4,"skill":0.2281,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2566,"headline":false},{"track":"A · English","score":0.4,"headline":true},{"track":"C · Arabic pairs","score":0.6,"headline":false},{"track":"C · English pairs","score":0.885,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4836,"skill":0.2254,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2746,"private_eq":0.5212},"42":{"raw":0.7606,"skill":0.5342,"coverage":1,"random":0.486,"tracks":[],"combined":0.6096,"private_eq":0.8011},"43":{"raw":0.5035,"skill":0.2123,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3782,"private_eq":0.6119},"44":{"raw":0.5966,"skill":0.1932,"coverage":1,"random":0.5,"tracks":[],"combined":0.2086,"private_eq":0.6083},"45":{"raw":0.1238,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4624,"skill":0.2198,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.5055,"skill":0.011,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0059,"private_eq":0.5081},"57":{"raw":0.4866,"skill":0.4226,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3445,"private_eq":0.4229},"58":{"raw":0.6134,"skill":0.4396,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1027,"private_eq":0.3873},"59":{"raw":0.0832,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.6338,"skill":0.2676,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3712,"private_eq":0.6887},"62":{"raw":0.5225,"skill":0.3633,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4741,"private_eq":0.6093},"64":{"raw":0.5398,"skill":0.4247,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5452,"private_eq":0.6395}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.system_one single batched forward, bf16, LEV_LOGITS_KEEP=last (board setting), torch 2.14.0+cu130","contamination":[{"catalog_id":4,"benchmark":"BANKING77","wrong":22,"flag":"22 BANKING77 test items found in lev's training records count as wrong"},{"catalog_id":5,"benchmark":"CLINC150+OOS","wrong":12,"flag":"12 CLINC150+OOS test items found in lev's training records count as wrong"},{"catalog_id":12,"benchmark":"ANLI","wrong":1,"flag":"1 ANLI test items found in lev's training records count as wrong"},{"catalog_id":26,"benchmark":"ARC-Easy","wrong":9,"flag":"9 ARC-Easy test items found in lev's training records count as wrong"},{"catalog_id":29,"benchmark":"HellaSwag","wrong":1,"flag":"1 HellaSwag test items found in lev's training records count as wrong"}],"v03":{"engine":"lev","pub":39.2,"same_skills":37.2,"new_domains":25.23,"v":35.47,"lab":false,"jev":false,"rank_v":146,"rank_pub":158,"name":"Interfaze lev Qwen3.5-4B LoRA","eq":{"S":36.58,"O":33.8},"rank_tie":34,"tied":true,"parts":{"public":0,"same_skills":21.95,"new_domains":13.52}}},{"engine":"solomon-v11-bf16-encoding","name":"Solomon v1.1","long_name":"Solomon v1.1","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":903468342,"model_url":"https://huggingface.co/DoccyHealth/Solomon","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"DoccyHealth/Solomon","served_checkpoint":null,"org":{"id":"DoccyHealth","name":"Doccy Health","logo":"data/orgs/DoccyHealth.png"}},"completed":82691,"counts":{"ok":82691,"unsupported":27498},"lifted":false,"calibration":{"acc":0.6907,"conf":0.7415,"ece":0.0807,"brier":0.4423,"over95":0.0302,"n":69332,"benchmarks":27,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0046,"conf":0.1867,"acc":0.3613},{"lo":0.2,"w":0.0128,"conf":0.2605,"acc":0.2044},{"lo":0.3,"w":0.0956,"conf":0.3491,"acc":0.4359},{"lo":0.4,"w":0.0721,"conf":0.4484,"acc":0.5093},{"lo":0.5,"w":0.1317,"conf":0.5513,"acc":0.5625},{"lo":0.6,"w":0.1045,"conf":0.6462,"acc":0.621},{"lo":0.7,"w":0.0818,"conf":0.751,"acc":0.653},{"lo":0.8,"w":0.1018,"conf":0.8521,"acc":0.7032},{"lo":0.9,"w":0.3952,"conf":0.97,"acc":0.8706}],"areas":{"knowledge":{"ece":0.0999,"acc":0.6208,"conf":0.6241},"language":{"ece":0.0728,"acc":0.8105,"conf":0.8641},"retrieval":{"ece":0.2617,"acc":0.5834,"conf":0.8451},"tools":{"ece":0.0072,"acc":0.991,"conf":0.9929},"arts":{"ece":0.0823,"acc":0.5919,"conf":0.6736}}},"gaps":{"share":0.2285,"reasons":[{"reason":"supports only 2–8 answer options per question","share":0.2251},{"reason":"request shape outside the model's schema","share":0.0034},{"reason":"request larger than its native request budget","share":0}]},"latency":{"median":223.9,"p95":1176.8,"mean":503.2},"results":{"1":{"score":0.9799,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":434.6},"2":{"score":0.6558,"jev":null,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":4192.5},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":null,"jev":null,"metric":null,"requests":10000,"answered":0,"unsupported":10000,"errors":0,"median_ms":null},"9":{"score":null,"jev":null,"metric":null,"requests":88,"answered":0,"unsupported":88,"errors":0,"median_ms":null},"10":{"score":0.592,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":478.6},"11":{"score":0.7153,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6708,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":220.6},"20":{"score":0.9414,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.4},"21":{"score":0.5422,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":217.3},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.502,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":258.4},"24":{"score":0.6995,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":218.6},"25":{"score":0.4286,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8657,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":219.1},"27":{"score":0.8456,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":218.5},"28":{"score":0.8106,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":215.9},"29":{"score":0.9099,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":221.6},"30":{"score":0.2756,"jev":null,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":1319,"unsupported":0,"errors":1319,"median_ms":null},"31":{"score":0.4019,"jev":null,"metric":"accuracy","requests":5000,"answered":525,"unsupported":4475,"errors":0,"median_ms":null},"32":{"score":0.6064,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2055,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1075.6},"36":{"score":0.4688,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5618,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":224.5},"38":{"score":0.2068,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":7732},"39":{"score":0.9186,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4822,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":221.7,"tracks":[{"track":"A · Arabic","score":0.4583,"headline":false},{"track":"A · English","score":0.4822,"headline":true},{"track":"C · Arabic pairs","score":0.715,"headline":false},{"track":"C · English pairs","score":0.9,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.694,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":219.5},"42":{"score":0.8176,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":257.8},"43":{"score":0.6933,"jev":null,"metric":"accuracy","requests":570,"answered":564,"unsupported":6,"errors":0,"median_ms":null},"44":{"score":0.7044,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.9},"45":{"score":0.1474,"jev":null,"metric":"accuracy","requests":501,"answered":407,"unsupported":94,"errors":0,"median_ms":null},"50":{"score":0.3729,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":268.7},"56":{"score":0.6815,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1181.1,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.7072,"metric":"accuracy","requests":12032,"answered":1250,"unsupported":10782,"errors":0,"median_ms":219,"edition_added":"v0.2","jev":null},"58":{"score":0.7254,"metric":"accuracy","requests":5507,"answered":5081,"unsupported":426,"errors":0,"median_ms":220.1,"edition_added":"v0.2","jev":null},"59":{"score":0.7552,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":286.7,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.711,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":234.5,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.8264,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":295.8,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.517,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":50.13,"balanced_skill":43.23,"breadth_skill":36.32,"public_skill":38.07},"frozen_scores":{"balanced_raw":50.13,"balanced_skill":38.07,"breadth_skill":36.32},"categories":[{"id":"knowledge","raw":0.4202,"skill":0.278,"combined":0.2144,"coverage":0.7668,"pending":0,"n":11},{"id":"language","raw":0.6901,"skill":0.5666,"combined":0.5421,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3813,"skill":0.2597,"combined":0.2689,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4919,"skill":0.4697,"combined":0.4555,"coverage":0.5921,"pending":0,"n":5},{"id":"arts","raw":0.4801,"skill":0.2447,"combined":0.3068,"coverage":0.8333,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9799,"skill":0.9729,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6539,"skill":0.6003,"coverage":0.9971,"random":0.1341,"tracks":[],"combined":0.68,"private_eq":0.7252},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0,"skill":0,"coverage":0,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0,"headline":false},{"track":"RouterBench-5shot","score":0,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":0,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.7153,"skill":0.5882,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6708,"skill":0.5069,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4854,"private_eq":0.6597},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.9437,"skill":0.8874,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.5422,"skill":0.0845,"coverage":1,"random":0.5,"tracks":[],"combined":0.0152,"private_eq":0.5127},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.505,"skill":0.01,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6953,"skill":0.5938,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4286,"skill":0.2381,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"28":{"raw":0.8106,"skill":0.6212,"coverage":1,"random":0.5,"tracks":[],"combined":0.4613,"private_eq":0.7332},"29":{"raw":0.9099,"skill":0.8799,"coverage":1,"random":0.25,"tracks":[],"combined":0.8593,"private_eq":0.8949},"30":{"raw":0.2756,"skill":0.2008,"coverage":0.5,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5512,"random":0.25},{"track":"10-choice","raw":0,"random":0.1}],"combined":0.3132,"private_eq":0.439},"31":{"raw":0.0422,"skill":0,"coverage":0.105,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0824},"32":{"raw":0.6064,"skill":0.3742,"coverage":1,"random":0.371,"tracks":[],"combined":0.2661,"private_eq":0.543},"33":{"raw":0.2055,"skill":0.195,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0967,"private_eq":0.1177},"36":{"raw":0.4688,"skill":0.399,"coverage":1,"random":0.116,"tracks":[],"combined":0.3455,"private_eq":0.4271},"37":{"raw":0.5618,"skill":0.4504,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5067,"private_eq":0.6104},"38":{"raw":0.2068,"skill":0.1814,"coverage":1,"random":0.031,"tracks":[],"combined":0.3143,"private_eq":0.3422},"39":{"raw":0.9186,"skill":0.8803,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6068,"private_eq":0.735},"40":{"raw":0.4822,"skill":0.3338,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4583,"headline":false},{"track":"A · English","score":0.4822,"headline":true},{"track":"C · Arabic pairs","score":0.715,"headline":false},{"track":"C · English pairs","score":0.9,"headline":false}],"combined":0.2922,"private_eq":0.4553},"41":{"raw":0.694,"skill":0.541,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5261,"private_eq":0.687},"42":{"raw":0.8176,"skill":0.6451,"coverage":1,"random":0.486,"tracks":[],"combined":0.8963,"private_eq":0.9468},"43":{"raw":0.686,"skill":0.5018,"coverage":0.9895,"random":0.3697,"tracks":[],"combined":0.4952,"private_eq":0.6848},"44":{"raw":0.7044,"skill":0.4088,"coverage":1,"random":0.5,"tracks":[],"combined":0.353,"private_eq":0.6797},"45":{"raw":0.1197,"skill":0,"coverage":0.8124,"random":0.1641,"tracks":[]},"50":{"raw":0.3729,"skill":0.0899,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.6815,"skill":0.363,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5078,"private_eq":0.7562},"57":{"raw":0.0735,"skill":0,"coverage":0.1039,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.1688,"private_eq":0.2685},"58":{"raw":0.6693,"skill":0.5207,"coverage":0.9226,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0484,"private_eq":0.3503},"59":{"raw":0.7552,"skill":0.4924,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.3039,"private_eq":0.6676},"61":{"raw":0.711,"skill":0.422,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3712,"private_eq":0.6887},"62":{"raw":0.8264,"skill":0.7685,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.6589,"private_eq":0.7463},"64":{"raw":0.517,"skill":0.3962,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4191,"private_eq":0.5398}},"coverage":0.6037,"pending":0.24,"latency_rows":562,"latency_path":"Solomon v1.1 bf16 serving binding with the encoding-fix path (solomon-v11-bf16-empty-context engine), sdpa (flash-attn optional, absent), venv-solomon torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2 as the binding requires","v03":{"engine":"solomon-v11-bf16-encoding","pub":38.07,"same_skills":36.95,"new_domains":41.25,"v":43.23,"lab":false,"jev":false,"rank_v":121,"rank_pub":162,"name":"Solomon v1.1","eq":{"S":36.32,"O":53.59},"rank_tie":27,"tied":true,"parts":{"public":0,"same_skills":21.79,"new_domains":21.44}}},{"engine":"jevk5-v0.3","name":"JevK5 v0.3","long_name":"JevK5 v0.3","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/alibiserikbay/JevK5","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6566,"conf":0.7171,"ece":0.0605,"brier":0.4587,"over95":0.0097,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0033,"conf":0.0871,"acc":0.0115},{"lo":0.1,"w":0.0264,"conf":0.1549,"acc":0.0687},{"lo":0.2,"w":0.0294,"conf":0.2494,"acc":0.1463},{"lo":0.3,"w":0.0505,"conf":0.3541,"acc":0.2923},{"lo":0.4,"w":0.0616,"conf":0.4517,"acc":0.3809},{"lo":0.5,"w":0.1279,"conf":0.5479,"acc":0.527},{"lo":0.6,"w":0.1341,"conf":0.6478,"acc":0.5753},{"lo":0.7,"w":0.1164,"conf":0.7495,"acc":0.6633},{"lo":0.8,"w":0.1503,"conf":0.8529,"acc":0.7682},{"lo":0.9,"w":0.3,"conf":0.9576,"acc":0.9166}],"areas":{"knowledge":{"ece":0.0795,"acc":0.5967,"conf":0.6762},"language":{"ece":0.0412,"acc":0.7557,"conf":0.7878},"retrieval":{"ece":0.0636,"acc":0.655,"conf":0.6957},"tools":{"ece":0.0521,"acc":0.8124,"conf":0.8223},"arts":{"ece":0.1158,"acc":0.5177,"conf":0.6336}}},"gaps":null,"latency":{"median":26.2,"p95":null,"mean":125},"results":{"1":{"score":0.8406,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":74.8},"2":{"score":0.6152,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":607.7},"3":{"score":0.6063,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1218.1},"4":{"score":0.6504,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":113.3},"5":{"score":0.7694,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":190.8},"6":{"score":0.7876,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":215.9},"9":{"score":0.2159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":958.5},"10":{"score":0.5757,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":53.3},"11":{"score":0.6633,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1237.7},"12":{"score":0.5992,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":14.1},"20":{"score":0.833,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.1},"21":{"score":0.6206,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":11.5},"22":{"score":0.041,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1079.9},"23":{"score":0.6278,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.9},"24":{"score":0.7465,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":12.9},"25":{"score":0.4133,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":16.8},"26":{"score":0.9794,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":12.9},"27":{"score":0.9317,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":12.9},"28":{"score":0.7474,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":11.5},"29":{"score":0.9044,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":17.3},"30":{"score":0.3271,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1232,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":87},"32":{"score":0.508,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":53.4},"33":{"score":0.2982,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":165.6},"36":{"score":0.4229,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":807.4},"37":{"score":0.4185,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.1},"38":{"score":0.1816,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":786.7},"39":{"score":0.8337,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":26.6},"40":{"score":0.5098,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":10.8},"41":{"score":0.4135,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":13.3},"42":{"score":0.7611,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":24.5},"43":{"score":0.4439,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":13.3},"44":{"score":0.6358,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.2},"45":{"score":0.1377,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":16.9},"50":{"score":0.432,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":37.3},"56":{"score":0.5965,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":140.7},"57":{"score":0.5173,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":16.5},"58":{"score":0.6419,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.3},"59":{"score":0.7116,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":36.9},"61":{"score":0.679,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":20.7},"62":{"score":0.7322,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":36.4},"64":{"score":0.6269,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":16}},"benchmarks":{"1":{"raw":0.8406,"skill":0.7848,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6876,"private_eq":0.7705},"2":{"raw":0.6152,"skill":0.5556,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6336,"private_eq":0.6855},"3":{"raw":0.6063,"skill":0.5987,"coverage":1,"random":0.0189,"tracks":[],"combined":0.597,"private_eq":0.6081},"4":{"raw":0.6504,"skill":0.6459,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7479,"private_eq":0.7529},"5":{"raw":0.7694,"skill":0.768,"coverage":1,"random":0.006,"tracks":[],"combined":0.626,"private_eq":0.6315},"6":{"raw":0.7876,"skill":0.502,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2159,"skill":0.2159,"coverage":1,"random":0,"tracks":[],"combined":0.1083,"private_eq":0.1175},"10":{"raw":0.5757,"skill":0.294,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6633,"skill":0.513,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5992,"skill":0.3996,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5012,"private_eq":0.6701},"20":{"raw":0.835,"skill":0.6701,"coverage":1,"random":0.5,"tracks":[],"combined":0.6684,"private_eq":0.8356},"21":{"raw":0.6206,"skill":0.2412,"coverage":1,"random":0.5,"tracks":[],"combined":0.2064,"private_eq":0.6072},"22":{"raw":0.0412,"skill":0.0337,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0887,"private_eq":0.1051},"23":{"raw":0.5977,"skill":0.1953,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7465,"skill":0.662,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[],"combined":0.1293,"private_eq":0.3537},"26":{"raw":0.9794,"skill":0.9725,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9317,"skill":0.9089,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7474,"skill":0.4948,"coverage":1,"random":0.5,"tracks":[],"combined":0.2897,"private_eq":0.6484},"29":{"raw":0.9044,"skill":0.8725,"coverage":1,"random":0.25,"tracks":[],"combined":0.8544,"private_eq":0.8913},"30":{"raw":0.3271,"skill":0.1916,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4534,"random":0.25},{"track":"10-choice","raw":0.2009,"random":0.1}],"combined":0.366,"private_eq":0.4821},"31":{"raw":0.1232,"skill":0.045,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0939,"private_eq":0.1767},"32":{"raw":0.508,"skill":0.2178,"coverage":1,"random":0.371,"tracks":[],"combined":0.0649,"private_eq":0.4179},"33":{"raw":0.2982,"skill":0.2889,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3177,"private_eq":0.3333},"36":{"raw":0.4229,"skill":0.3472,"coverage":1,"random":0.116,"tracks":[],"combined":0.3051,"private_eq":0.3918},"37":{"raw":0.4185,"skill":0.2706,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3591,"private_eq":0.494},"38":{"raw":0.1816,"skill":0.1554,"coverage":1,"random":0.031,"tracks":[],"combined":0.2713,"private_eq":0.301},"39":{"raw":0.8337,"skill":0.7554,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5153,"private_eq":0.6735},"40":{"raw":0.5098,"skill":0.3694,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3091,"headline":false},{"track":"A · English","score":0.5098,"headline":true},{"track":"C · Arabic pairs","score":0.65,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"combined":0.2516,"private_eq":0.4241},"41":{"raw":0.4135,"skill":0.1203,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3925,"private_eq":0.5989},"42":{"raw":0.7611,"skill":0.5352,"coverage":1,"random":0.486,"tracks":[],"combined":0.5954,"private_eq":0.7939},"43":{"raw":0.4439,"skill":0.1176,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3613,"private_eq":0.6014},"44":{"raw":0.6358,"skill":0.2716,"coverage":1,"random":0.5,"tracks":[],"combined":0.2834,"private_eq":0.6453},"45":{"raw":0.1377,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.432,"skill":0.1756,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.5965,"skill":0.193,"coverage":1,"random":0.5,"tracks":[],"combined":0.153,"private_eq":0.5808},"57":{"raw":0.5173,"skill":0.4571,"coverage":1,"random":0.1109,"tracks":[],"combined":0.5077,"private_eq":0.5664},"58":{"raw":0.6419,"skill":0.4809,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1534,"private_eq":0.4219},"59":{"raw":0.7116,"skill":0.402,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2368,"private_eq":0.6356},"61":{"raw":0.679,"skill":0.358,"coverage":1,"random":0.5,"tracks":[],"combined":0.4847,"private_eq":0.7448},"62":{"raw":0.7322,"skill":0.6429,"coverage":1,"random":0.25,"tracks":[],"combined":0.5767,"private_eq":0.6854},"64":{"raw":0.6269,"skill":0.5336,"coverage":1,"random":0.2,"tracks":[],"combined":0.5064,"private_eq":0.6088}},"coverage":1,"pending":0,"scores":{"balanced_raw":55.82,"balanced_skill":36.59,"breadth_skill":39.87,"public_skill":41.14},"frozen_scores":{"balanced_raw":55.82,"balanced_skill":41.14,"breadth_skill":39.87},"categories":[{"id":"knowledge","raw":0.4404,"skill":0.2594,"combined":0.2577,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6254,"skill":0.4657,"combined":0.463,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5917,"skill":0.4447,"combined":0.4563,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.611,"skill":0.5694,"combined":0.5297,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5256,"skill":0.3082,"combined":0.3346,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jevk5-v0.3","pub":41.14,"same_skills":41.75,"new_domains":21.91,"v":36.59,"lab":false,"jev":false,"rank_v":144,"rank_pub":151,"name":"JevK5 v0.3","eq":{"S":41.18,"O":29.71},"rank_tie":33,"tied":true,"parts":{"public":0,"same_skills":24.71,"new_domains":11.88}}},{"engine":"neohorse-jev-4b","name":"NeoHorse-Jev-4B","long_name":"TokenRhythm NeoHorse-Jev-4B","variant":null,"meta":{"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9083860816,"model_url":"https://huggingface.co/TokenRhythm/NeoHorse-Jev-4B","code_url":"https://github.com/TokenRhythm/NeoHorse/tree/main/jev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"TokenRhythm/NeoHorse-Jev-4B","served_checkpoint":null,"org":{"id":"TokenRhythm","name":"TokenRhythm","logo":"data/orgs/TokenRhythm.png"}},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.6071,"conf":0.711,"ece":0.1039,"brier":0.5134,"over95":0.0277,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0413,"conf":0.0732,"acc":0.0537},{"lo":0.1,"w":0.0279,"conf":0.1459,"acc":0.1244},{"lo":0.2,"w":0.0236,"conf":0.2574,"acc":0.174},{"lo":0.3,"w":0.051,"conf":0.3549,"acc":0.3136},{"lo":0.4,"w":0.0671,"conf":0.454,"acc":0.3694},{"lo":0.5,"w":0.103,"conf":0.5488,"acc":0.4829},{"lo":0.6,"w":0.1012,"conf":0.652,"acc":0.5617},{"lo":0.7,"w":0.1072,"conf":0.7506,"acc":0.634},{"lo":0.8,"w":0.1342,"conf":0.8512,"acc":0.6751},{"lo":0.9,"w":0.3435,"conf":0.9667,"acc":0.8483}],"areas":{"knowledge":{"ece":0.0809,"acc":0.5452,"conf":0.6255},"language":{"ece":0.1292,"acc":0.6703,"conf":0.7995},"retrieval":{"ece":0.0726,"acc":0.7064,"conf":0.7543},"tools":{"ece":0.0689,"acc":0.8587,"conf":0.8892},"arts":{"ece":0.1992,"acc":0.3991,"conf":0.5982}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":49,"p95":273.9,"mean":208.6},"results":{"1":{"score":0.9622,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":64.5},"2":{"score":0.6453,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":8815.4},"3":{"score":0.6614,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8379,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":52.6},"5":{"score":0.7812,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":71.6},"6":{"score":0.7947,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":191.7},"9":{"score":0.3068,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5987,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":56.7},"11":{"score":0.7256,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5189,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":47.1},"20":{"score":0.5054,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.2},"21":{"score":0.5175,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":46.4},"22":{"score":0.063,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":302.3},"23":{"score":0.5718,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":49.2},"24":{"score":0.6786,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":47.5},"25":{"score":0.3214,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.963,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":46.9},"27":{"score":0.8916,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":47.1},"28":{"score":0.6867,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":46.2},"29":{"score":0.722,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":47.7},"30":{"score":0.337,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.11,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.1},"32":{"score":0.5612,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2158,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":467.5},"36":{"score":0.4079,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4435,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.7},"38":{"score":0.0536,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":383.6},"39":{"score":0.8192,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3804,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":46.3,"tracks":[{"track":"A · Arabic","score":0.3177,"headline":false},{"track":"A · English","score":0.3804,"headline":true},{"track":"C · Arabic pairs","score":0.39,"headline":false},{"track":"C · English pairs","score":0.685,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4739,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":47.4},"42":{"score":0.7498,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":49.8},"43":{"score":0.4965,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6494,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.3},"45":{"score":0.1078,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3383,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":51.8},"56":{"score":0.5375,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":84,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4541,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":47.6,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6368,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":47.3,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.6848,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":50.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.593,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":49.2,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5183,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":50.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5265,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":53.49,"balanced_skill":37.26,"breadth_skill":34.52,"public_skill":37},"frozen_scores":{"balanced_raw":53.49,"balanced_skill":37,"breadth_skill":34.52},"categories":[{"id":"knowledge","raw":0.4182,"skill":0.2361,"combined":0.2325,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.572,"skill":0.3962,"combined":0.3422,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.607,"skill":0.4391,"combined":0.4466,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6331,"skill":0.5888,"combined":0.5572,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4174,"skill":0.1103,"combined":0.1554,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9622,"skill":0.949,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.6453,"skill":0.5903,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6632,"private_eq":0.7108},"3":{"raw":0.6614,"skill":0.6549,"coverage":1,"random":0.0189,"tracks":[],"combined":0.597,"private_eq":0.6081},"4":{"raw":0.8379,"skill":0.8358,"coverage":1,"random":0.0127,"tracks":[],"combined":0.823,"private_eq":0.8262},"5":{"raw":0.7812,"skill":0.7799,"coverage":1,"random":0.006,"tracks":[],"combined":0.591,"private_eq":0.5971},"6":{"raw":0.7947,"skill":0.5154,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.783,"headline":false},{"track":"RouterBench-5shot","score":0.8065,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.3068,"skill":0.3068,"coverage":1,"random":0,"tracks":[],"combined":0.1083,"private_eq":0.1175},"11":{"raw":0.7256,"skill":0.6031,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5189,"skill":0.2794,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3907,"private_eq":0.5972},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5083,"skill":0.0166,"coverage":1,"random":0.5,"tracks":[],"combined":0.119,"private_eq":0.564},"21":{"raw":0.5175,"skill":0.035,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0586,"skill":0.0512,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0702,"private_eq":0.087},"23":{"raw":0.5554,"skill":0.1109,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6918,"skill":0.589,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[],"combined":0.0816,"private_eq":0.3183},"28":{"raw":0.6867,"skill":0.3734,"coverage":1,"random":0.5,"tracks":[],"combined":0.3553,"private_eq":0.6808},"29":{"raw":0.722,"skill":0.6293,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"30":{"raw":0.337,"skill":0.2006,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4352,"random":0.25},{"track":"10-choice","raw":0.2388,"random":0.1}],"combined":0.4487,"private_eq":0.5495},"31":{"raw":0.11,"skill":0.0307,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0463,"private_eq":0.1335},"32":{"raw":0.5612,"skill":0.3024,"coverage":1,"random":0.371,"tracks":[],"combined":0.0726,"private_eq":0.4227},"33":{"raw":0.2158,"skill":0.2054,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1122,"private_eq":0.1328},"36":{"raw":0.4079,"skill":0.3302,"coverage":1,"random":0.116,"tracks":[],"combined":0.3296,"private_eq":0.4132},"37":{"raw":0.4435,"skill":0.3019,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4217,"private_eq":0.5434},"38":{"raw":0.0536,"skill":0.0233,"coverage":1,"random":0.031,"tracks":[],"combined":0.0411,"private_eq":0.0804},"39":{"raw":0.8192,"skill":0.7341,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5401,"private_eq":0.6902},"40":{"raw":0.3804,"skill":0.2029,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3177,"headline":false},{"track":"A · English","score":0.3804,"headline":true},{"track":"C · Arabic pairs","score":0.39,"headline":false},{"track":"C · English pairs","score":0.685,"headline":false}],"combined":0.2216,"private_eq":0.4011},"41":{"raw":0.4739,"skill":0.2109,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2597,"private_eq":0.5114},"42":{"raw":0.7498,"skill":0.5132,"coverage":1,"random":0.486,"tracks":[],"combined":0.7025,"private_eq":0.8483},"43":{"raw":0.4965,"skill":0.2011,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3822,"private_eq":0.6144},"44":{"raw":0.6494,"skill":0.2988,"coverage":1,"random":0.5,"tracks":[],"combined":0.3959,"private_eq":0.7009},"45":{"raw":0.1078,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3383,"skill":0.0397,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.5375,"skill":0.075,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.153,"private_eq":0.5808},"57":{"raw":0.4541,"skill":0.386,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2732,"private_eq":0.3603},"58":{"raw":0.6368,"skill":0.4735,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1676,"private_eq":0.4316},"59":{"raw":0.6848,"skill":0.3465,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.593,"skill":0.186,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2804,"private_eq":0.6438},"62":{"raw":0.5183,"skill":0.3577,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4261,"private_eq":0.5737},"64":{"raw":0.5265,"skill":0.4081,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4095,"private_eq":0.5322}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author neohorse_decision 1.0.0 wheel in its own env (py3.12.10, torch 2.8.0+cu128, triton 3.7.1), board guard lifts, one process","v03":{"engine":"neohorse-jev-4b","pub":37,"same_skills":36.9,"new_domains":29.23,"v":37.26,"lab":false,"jev":false,"rank_v":143,"rank_pub":171,"name":"TokenRhythm NeoHorse-Jev-4B","eq":{"S":36.27,"O":38.74},"rank_tie":32,"tied":true,"parts":{"public":0,"same_skills":21.76,"new_domains":15.5}}},{"engine":"ryotide-qwen-v0.3.1","name":"Ryotide Qwen v0.3.1","long_name":"Ryotide Qwen v0.3.1","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4000000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/csabag/ryotide","base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6309,"conf":0.7225,"ece":0.0916,"brier":0.4852,"over95":0.0182,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0005,"conf":0.0887,"acc":0.0769},{"lo":0.1,"w":0.0094,"conf":0.1665,"acc":0.1275},{"lo":0.2,"w":0.0281,"conf":0.2583,"acc":0.1161},{"lo":0.3,"w":0.0629,"conf":0.3522,"acc":0.1889},{"lo":0.4,"w":0.0693,"conf":0.4498,"acc":0.3216},{"lo":0.5,"w":0.1519,"conf":0.551,"acc":0.4819},{"lo":0.6,"w":0.1115,"conf":0.6474,"acc":0.525},{"lo":0.7,"w":0.1268,"conf":0.7501,"acc":0.6712},{"lo":0.8,"w":0.1331,"conf":0.8524,"acc":0.7735},{"lo":0.9,"w":0.3066,"conf":0.9651,"acc":0.8885}],"areas":{"knowledge":{"ece":0.1323,"acc":0.5363,"conf":0.6686},"language":{"ece":0.0562,"acc":0.731,"conf":0.7872},"retrieval":{"ece":0.095,"acc":0.6797,"conf":0.7748},"tools":{"ece":0.0874,"acc":0.8178,"conf":0.8245},"arts":{"ece":0.1143,"acc":0.5074,"conf":0.6218}}},"gaps":null,"latency":{"median":80.6,"p95":null,"mean":249.4},"results":{"1":{"score":0.9115,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":613},"2":{"score":0.6003,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":7767.4},"3":{"score":0.6516,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3250.4},"4":{"score":0.7198,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1299.1},"5":{"score":0.8105,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":2099.9},"6":{"score":0.7698,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":969.5},"9":{"score":0.0682,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":5634.8},"10":{"score":0.516,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":418.8},"11":{"score":0.652,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":5426.8},"12":{"score":0.5659,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":298.6},"20":{"score":0.8782,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":188.3},"21":{"score":0.6081,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":290.1},"22":{"score":0.0245,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":4064.2},"23":{"score":0.571,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":348.2},"24":{"score":0.7579,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":194},"25":{"score":0.4184,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":196.2},"26":{"score":0.9806,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":162.4},"27":{"score":0.9445,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":162.4},"28":{"score":0.7032,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":161},"29":{"score":0.8701,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":209.7},"30":{"score":0.2513,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.138,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":721.6},"32":{"score":0.5386,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":461.6},"33":{"score":0.0727,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1305.5},"36":{"score":0.403,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":8248},"37":{"score":0.385,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":211.6},"38":{"score":0.2067,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":10309.6},"39":{"score":0.6633,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":392.8},"40":{"score":0.4217,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":177.2},"41":{"score":0.4379,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":197.3},"42":{"score":0.6749,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":245.2},"43":{"score":0.4351,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":188},"44":{"score":0.6058,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":196.1},"45":{"score":0.1038,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":233.2},"50":{"score":0.4475,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":337.2},"56":{"score":0.7195,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1472.9},"57":{"score":0.4759,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":202.7},"58":{"score":0.5774,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":184.5},"59":{"score":0.5215,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":268.5},"61":{"score":0.6242,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":216.8},"62":{"score":0.6531,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":311.6},"64":{"score":0.6231,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":188.4}},"benchmarks":{"1":{"raw":0.9115,"skill":0.8805,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7218,"private_eq":0.7955},"2":{"raw":0.6003,"skill":0.5384,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6323,"private_eq":0.6844},"3":{"raw":0.6516,"skill":0.6449,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6284,"private_eq":0.6386},"4":{"raw":0.7198,"skill":0.7162,"coverage":1,"random":0.0127,"tracks":[],"combined":0.822,"private_eq":0.8253},"5":{"raw":0.8105,"skill":0.8094,"coverage":1,"random":0.006,"tracks":[],"combined":0.7019,"private_eq":0.7061},"6":{"raw":0.7698,"skill":0.4603,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0682,"skill":0.0682,"coverage":1,"random":0,"tracks":[],"combined":0.0325,"private_eq":0.0425},"10":{"raw":0.516,"skill":0.1947,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.652,"skill":0.4967,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5659,"skill":0.3498,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4624,"private_eq":0.6445},"20":{"raw":0.8814,"skill":0.7629,"coverage":1,"random":0.5,"tracks":[],"combined":0.8493,"private_eq":0.925},"21":{"raw":0.6081,"skill":0.2161,"coverage":1,"random":0.5,"tracks":[],"combined":0.2264,"private_eq":0.6171},"22":{"raw":0.0238,"skill":0.0162,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0039,"private_eq":0.0219},"23":{"raw":0.5333,"skill":0.0665,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7579,"skill":0.6772,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4184,"skill":0.2245,"coverage":1,"random":0.25,"tracks":[],"combined":0.1419,"private_eq":0.363},"26":{"raw":0.9806,"skill":0.9741,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9445,"skill":0.926,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7032,"skill":0.4064,"coverage":1,"random":0.5,"tracks":[],"combined":0.2191,"private_eq":0.6135},"29":{"raw":0.8701,"skill":0.8268,"coverage":1,"random":0.25,"tracks":[],"combined":0.7801,"private_eq":0.8362},"30":{"raw":0.2513,"skill":0.0967,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3571,"random":0.25},{"track":"10-choice","raw":0.1456,"random":0.1}],"combined":0.2553,"private_eq":0.3918},"31":{"raw":0.138,"skill":0.0611,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.5386,"skill":0.2665,"coverage":1,"random":0.371,"tracks":[],"combined":0.1191,"private_eq":0.4516},"33":{"raw":0.0727,"skill":0.0604,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.403,"skill":0.3247,"coverage":1,"random":0.116,"tracks":[],"combined":0.275,"private_eq":0.3655},"37":{"raw":0.385,"skill":0.2286,"coverage":1,"random":0.2027,"tracks":[],"combined":0.2133,"private_eq":0.3791},"38":{"raw":0.2067,"skill":0.1813,"coverage":1,"random":0.031,"tracks":[],"combined":0.2907,"private_eq":0.3195},"39":{"raw":0.6633,"skill":0.5048,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3939,"private_eq":0.5919},"40":{"raw":0.4217,"skill":0.256,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3317,"headline":false},{"track":"A · English","score":0.4217,"headline":true},{"track":"C · Arabic pairs","score":0.67,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"combined":0.141,"private_eq":0.3391},"41":{"raw":0.4379,"skill":0.1568,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4918,"private_eq":0.6644},"42":{"raw":0.6749,"skill":0.3675,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4351,"skill":0.1037,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3488,"private_eq":0.5936},"44":{"raw":0.6058,"skill":0.2116,"coverage":1,"random":0.5,"tracks":[],"combined":0.23,"private_eq":0.6189},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4475,"skill":0.1981,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.7195,"skill":0.439,"coverage":1,"random":0.5,"tracks":[],"combined":0.2654,"private_eq":0.6364},"57":{"raw":0.4759,"skill":0.4105,"coverage":1,"random":0.1109,"tracks":[],"combined":0.3278,"private_eq":0.4083},"58":{"raw":0.5774,"skill":0.3874,"coverage":1,"random":0.3101,"tracks":[],"combined":0.0883,"private_eq":0.3775},"59":{"raw":0.5215,"skill":0.0079,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6242,"skill":0.2484,"coverage":1,"random":0.5,"tracks":[],"combined":0.3882,"private_eq":0.6971},"62":{"raw":0.6531,"skill":0.5375,"coverage":1,"random":0.25,"tracks":[],"combined":0.5082,"private_eq":0.6346},"64":{"raw":0.6231,"skill":0.5289,"coverage":1,"random":0.2,"tracks":[],"combined":0.4968,"private_eq":0.6012}},"coverage":1,"pending":0,"scores":{"balanced_raw":53.17,"balanced_skill":31.99,"breadth_skill":35.64,"public_skill":37.28},"frozen_scores":{"balanced_raw":53.17,"balanced_skill":37.28,"breadth_skill":35.64},"categories":[{"id":"knowledge","raw":0.398,"skill":0.2095,"combined":0.1822,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.564,"skill":0.3599,"combined":0.3318,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6134,"skill":0.4752,"combined":0.4584,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5921,"skill":0.5508,"combined":0.5172,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5195,"skill":0.2981,"combined":0.3573,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"ryotide-qwen-v0.3.1","pub":37.28,"same_skills":36.67,"new_domains":18.83,"v":31.99,"lab":false,"jev":false,"rank_v":162,"rank_pub":170,"name":"Ryotide Qwen v0.3.1","eq":{"S":36.04,"O":25.9},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":21.62,"new_domains":10.36}}},{"engine":"intern-decision-4b","name":"Intern-Decision-4B","long_name":"InternLM Intern-Decision Qwen3.5-4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9078626736,"model_url":"https://huggingface.co/internlm/Intern-Decision-4B","code_url":"https://github.com/internlm/Intern-Decision","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"internlm/Intern-Decision-4B","served_checkpoint":null,"org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"}},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.6355,"conf":0.6633,"ece":0.0278,"brier":0.4737,"over95":0.0025,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0193,"conf":0.0783,"acc":0.0288},{"lo":0.1,"w":0.0374,"conf":0.1445,"acc":0.1325},{"lo":0.2,"w":0.0286,"conf":0.2507,"acc":0.2472},{"lo":0.3,"w":0.048,"conf":0.3511,"acc":0.3018},{"lo":0.4,"w":0.0715,"conf":0.4555,"acc":0.4349},{"lo":0.5,"w":0.1605,"conf":0.5478,"acc":0.5165},{"lo":0.6,"w":0.1491,"conf":0.6497,"acc":0.631},{"lo":0.7,"w":0.1475,"conf":0.751,"acc":0.703},{"lo":0.8,"w":0.1634,"conf":0.8534,"acc":0.8175},{"lo":0.9,"w":0.1746,"conf":0.9434,"acc":0.9339}],"areas":{"knowledge":{"ece":0.064,"acc":0.5564,"conf":0.6199},"language":{"ece":0.034,"acc":0.7187,"conf":0.7509},"retrieval":{"ece":0.0692,"acc":0.6735,"conf":0.6306},"tools":{"ece":0.092,"acc":0.8465,"conf":0.7553},"arts":{"ece":0.0689,"acc":0.5029,"conf":0.5718}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":44.2,"p95":115.4,"mean":60},"results":{"1":{"score":0.9274,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":46.2},"2":{"score":0.5282,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":286.4},"3":{"score":0.7598,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6261,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.2},"5":{"score":0.7273,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":65.8},"6":{"score":0.7984,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":98.9},"9":{"score":0.125,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.665,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":63.4},"11":{"score":0.6202,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5839,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":43.4},"20":{"score":0.8182,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"21":{"score":0.5864,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":43.2},"22":{"score":0.07,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":239.1},"23":{"score":0.5854,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.4},"24":{"score":0.7137,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":43.2},"25":{"score":0.4133,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9663,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":43},"27":{"score":0.9138,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":43.2},"28":{"score":0.6835,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":43.5},"29":{"score":0.7755,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":43.5},"30":{"score":0.2559,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1262,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46},"32":{"score":0.5372,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3079,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":48},"36":{"score":0.331,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.453,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44},"38":{"score":0.1587,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":274.3},"39":{"score":0.8087,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4636,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":43.4,"tracks":[{"track":"A · Arabic","score":0.2965,"headline":false},{"track":"A · English","score":0.4636,"headline":true},{"track":"C · Arabic pairs","score":0.675,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4066,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.3},"42":{"score":0.7783,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":44.8},"43":{"score":0.4509,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5998,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"45":{"score":0.1277,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4511,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":45.6},"56":{"score":0.615,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":46.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4614,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":43.5,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6207,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":43.3,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.7186,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":45.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6052,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":44.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5775,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":46.8,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5852,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":53.59,"balanced_skill":38.69,"breadth_skill":36.62,"public_skill":38},"frozen_scores":{"balanced_raw":53.59,"balanced_skill":38,"breadth_skill":36.62},"categories":[{"id":"knowledge","raw":0.4205,"skill":0.2296,"combined":0.2294,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5943,"skill":0.4254,"combined":0.4162,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5598,"skill":0.4034,"combined":0.3952,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6028,"skill":0.5566,"combined":0.5581,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5129,"skill":0.2818,"combined":0.329,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9274,"skill":0.902,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8512,"private_eq":0.8903},"2":{"raw":0.5282,"skill":0.4551,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6691,"private_eq":0.7159},"3":{"raw":0.7598,"skill":0.7552,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6651,"private_eq":0.6742},"4":{"raw":0.6261,"skill":0.6213,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6733,"private_eq":0.6801},"5":{"raw":0.7273,"skill":0.7257,"coverage":1,"random":0.006,"tracks":[],"combined":0.5865,"private_eq":0.5927},"6":{"raw":0.7984,"skill":0.5245,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7857,"headline":false},{"track":"RouterBench-5shot","score":0.8111,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1,"random":0,"tracks":[],"combined":0.0097,"private_eq":0.02},"11":{"raw":0.6202,"skill":0.4508,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5839,"skill":0.3767,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4833,"private_eq":0.6583},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8202,"skill":0.6405,"coverage":1,"random":0.5,"tracks":[],"combined":0.7409,"private_eq":0.8714},"21":{"raw":0.5864,"skill":0.1728,"coverage":1,"random":0.5,"tracks":[],"combined":0.136,"private_eq":0.5724},"22":{"raw":0.069,"skill":0.0617,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1339,"private_eq":0.1495},"23":{"raw":0.5656,"skill":0.1311,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7235,"skill":0.6314,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[],"combined":0.1083,"private_eq":0.3381},"28":{"raw":0.6835,"skill":0.367,"coverage":1,"random":0.5,"tracks":[],"combined":0.2343,"private_eq":0.621},"29":{"raw":0.7755,"skill":0.7007,"coverage":1,"random":0.25,"tracks":[],"combined":0.5623,"private_eq":0.6747},"30":{"raw":0.2559,"skill":0.101,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3503,"random":0.25},{"track":"10-choice","raw":0.1615,"random":0.1}],"combined":0.3238,"private_eq":0.4477},"31":{"raw":0.1262,"skill":0.0483,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0755,"private_eq":0.16},"32":{"raw":0.5372,"skill":0.2642,"coverage":1,"random":0.371,"tracks":[],"combined":0.1191,"private_eq":0.4516},"33":{"raw":0.3079,"skill":0.2987,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2766,"private_eq":0.2932},"36":{"raw":0.331,"skill":0.2432,"coverage":1,"random":0.116,"tracks":[],"combined":0.2824,"private_eq":0.372},"37":{"raw":0.453,"skill":0.3139,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3286,"private_eq":0.47},"38":{"raw":0.1587,"skill":0.1318,"coverage":1,"random":0.031,"tracks":[],"combined":0.2451,"private_eq":0.2759},"39":{"raw":0.8087,"skill":0.7186,"coverage":1,"random":0.3201,"tracks":[],"combined":0.53,"private_eq":0.6834},"40":{"raw":0.4636,"skill":0.3099,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2965,"headline":false},{"track":"A · English","score":0.4636,"headline":true},{"track":"C · Arabic pairs","score":0.675,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"combined":0.2422,"private_eq":0.4169},"41":{"raw":0.4066,"skill":0.11,"coverage":1,"random":0.3333,"tracks":[],"combined":0.269,"private_eq":0.5175},"42":{"raw":0.7783,"skill":0.5687,"coverage":1,"random":0.486,"tracks":[],"combined":0.6822,"private_eq":0.838},"43":{"raw":0.4509,"skill":0.1288,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3488,"private_eq":0.5936},"44":{"raw":0.5998,"skill":0.1996,"coverage":1,"random":0.5,"tracks":[],"combined":0.2727,"private_eq":0.64},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4511,"skill":0.2033,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.615,"skill":0.23,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0751,"private_eq":0.5423},"57":{"raw":0.4614,"skill":0.3942,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.4022,"private_eq":0.4737},"58":{"raw":0.6207,"skill":0.4502,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1398,"private_eq":0.4126},"59":{"raw":0.7186,"skill":0.4165,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.2726,"private_eq":0.6527},"61":{"raw":0.6052,"skill":0.2104,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3542,"private_eq":0.6803},"62":{"raw":0.5775,"skill":0.4367,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5151,"private_eq":0.6397},"64":{"raw":0.5852,"skill":0.4815,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4677,"private_eq":0.5782}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.predict on the whole row, SDPA, max_length 262144 and the board lifts; transformers fast path enabled with causal-conv1d 1.7.0 (fla 0.4.2 gated delta + CUDA conv). The board ran the torch fallback because causal-conv1d was absent; that run is kept as latency-v1-full.torch-fallback. venv-intern torch 2.9.1+cu128","v03":{"engine":"intern-decision-4b","pub":38,"same_skills":39.1,"new_domains":29.41,"v":38.69,"lab":false,"jev":false,"rank_v":138,"rank_pub":163,"name":"InternLM Intern-Decision Qwen3.5-4B","eq":{"S":38.5,"O":38.97},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":23.1,"new_domains":15.59}}},{"engine":"winnow-e4b-q8","name":"Winnow-E4B","long_name":"Winnow-E4B","variant":"Q8_0","meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":7996156490,"params_estimated":false,"params_from_name":false,"trained_bytes":8005437472,"model_url":"https://huggingface.co/EldanRing/Winnow-E4B","code_url":"https://github.com/EldanRing/winnow-inference","base_model":"google/gemma-4-E4B","base_url":"https://huggingface.co/google/gemma-4-E4B","weights_repo":"EldanRing/Winnow-E4B","served_checkpoint":"EldanRing/Winnow-E4B"},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.6357,"conf":0.6933,"ece":0.058,"brier":0.4613,"over95":0.0069,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0284,"conf":0.063,"acc":0.0211},{"lo":0.1,"w":0.0309,"conf":0.1529,"acc":0.132},{"lo":0.2,"w":0.0331,"conf":0.2598,"acc":0.2652},{"lo":0.3,"w":0.0523,"conf":0.3491,"acc":0.2461},{"lo":0.4,"w":0.0502,"conf":0.4511,"acc":0.3741},{"lo":0.5,"w":0.1308,"conf":0.5503,"acc":0.4842},{"lo":0.6,"w":0.1353,"conf":0.6511,"acc":0.5491},{"lo":0.7,"w":0.1184,"conf":0.7494,"acc":0.6597},{"lo":0.8,"w":0.137,"conf":0.8518,"acc":0.799},{"lo":0.9,"w":0.2836,"conf":0.9583,"acc":0.9358}],"areas":{"knowledge":{"ece":0.0838,"acc":0.5408,"conf":0.6246},"language":{"ece":0.0325,"acc":0.7688,"conf":0.7998},"retrieval":{"ece":0.1093,"acc":0.6314,"conf":0.7336},"tools":{"ece":0.0471,"acc":0.8372,"conf":0.8238},"arts":{"ece":0.0567,"acc":0.4874,"conf":0.5423}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":45,"p95":188.3,"mean":75.6},"results":{"1":{"score":0.9144,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":75.3},"2":{"score":0.603,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":560.1},"3":{"score":0.7303,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.712,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":95.3},"5":{"score":0.826,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":120},"6":{"score":0.7983,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":149.1},"9":{"score":0.1023,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3729,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":81.4},"11":{"score":0.7334,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5617,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":34.8},"20":{"score":0.8152,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.6},"21":{"score":0.6027,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.3},"22":{"score":0.036,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":328.6},"23":{"score":0.5662,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51},"24":{"score":0.6701,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":32.5},"25":{"score":0.3622,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.96,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.2},"27":{"score":0.8857,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.9},"28":{"score":0.6606,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":31.8},"29":{"score":0.7662,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":34.7},"30":{"score":0.2638,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.15,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.4},"32":{"score":0.5519,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2921,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":94.9},"36":{"score":0.4155,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4655,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.1},"38":{"score":0.2056,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":552.3},"39":{"score":0.8853,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5383,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":32.3,"tracks":[{"track":"A · Arabic","score":0.3571,"headline":false},{"track":"A · English","score":0.5383,"headline":true},{"track":"C · Arabic pairs","score":0.625,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.631,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":35.7},"42":{"score":0.7391,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":54.2},"43":{"score":0.5018,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6032,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.8},"45":{"score":0.1257,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4421,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":74.6},"56":{"score":0.5095,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":92.7,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4377,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.6,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6287,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":38.8,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.6081,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":66.2,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6478,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":49.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.8502,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":66,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5568,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":56.13,"balanced_skill":32.72,"breadth_skill":38.86,"public_skill":40.86},"frozen_scores":{"balanced_raw":56.13,"balanced_skill":40.86,"breadth_skill":38.86},"categories":[{"id":"knowledge","raw":0.4184,"skill":0.2276,"combined":0.2005,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6313,"skill":0.4677,"combined":0.3753,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6011,"skill":0.438,"combined":0.4077,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6535,"skill":0.6251,"combined":0.5352,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5014,"skill":0.2693,"combined":0.2824,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9144,"skill":0.8844,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6605,"private_eq":0.7506},"2":{"raw":0.603,"skill":0.5415,"coverage":1,"random":0.1341,"tracks":[],"combined":0.666,"private_eq":0.7132},"3":{"raw":0.7303,"skill":0.7251,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5916,"private_eq":0.6029},"4":{"raw":0.712,"skill":0.7083,"coverage":1,"random":0.0127,"tracks":[],"combined":0.796,"private_eq":0.7999},"5":{"raw":0.826,"skill":0.8249,"coverage":1,"random":0.006,"tracks":[],"combined":0.5255,"private_eq":0.5327},"6":{"raw":0.7983,"skill":0.5244,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7849,"headline":false},{"track":"RouterBench-5shot","score":0.8117,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.1023,"skill":0.1023,"coverage":1,"random":0,"tracks":[],"combined":0.1109,"private_eq":0.12},"11":{"raw":0.7334,"skill":0.6145,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5617,"skill":0.3435,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3069,"private_eq":0.5419},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8184,"skill":0.6368,"coverage":1,"random":0.5,"tracks":[],"combined":0.6106,"private_eq":0.807},"21":{"raw":0.6027,"skill":0.2055,"coverage":1,"random":0.5,"tracks":[],"combined":0.1058,"private_eq":0.5575},"22":{"raw":0.0321,"skill":0.0245,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0542,"private_eq":0.0713},"23":{"raw":0.5564,"skill":0.1129,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6759,"skill":0.5679,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3622,"skill":0.1497,"coverage":1,"random":0.25,"tracks":[],"combined":0.0888,"private_eq":0.3236},"28":{"raw":0.6606,"skill":0.3212,"coverage":1,"random":0.5,"tracks":[],"combined":0.3504,"private_eq":0.6784},"29":{"raw":0.7662,"skill":0.6883,"coverage":1,"random":0.25,"tracks":[],"combined":0.587,"private_eq":0.693},"30":{"raw":0.2638,"skill":0.1109,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3601,"random":0.25},{"track":"10-choice","raw":0.1676,"random":0.1}],"combined":0.2755,"private_eq":0.4083},"31":{"raw":0.15,"skill":0.0742,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0518,"private_eq":0.1385},"32":{"raw":0.5519,"skill":0.2876,"coverage":1,"random":0.371,"tracks":[],"combined":0.1963,"private_eq":0.4996},"33":{"raw":0.2921,"skill":0.2827,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3433,"private_eq":0.3583},"36":{"raw":0.4155,"skill":0.3388,"coverage":1,"random":0.116,"tracks":[],"combined":0.2357,"private_eq":0.3312},"37":{"raw":0.4655,"skill":0.3296,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3084,"private_eq":0.4541},"38":{"raw":0.2056,"skill":0.1802,"coverage":1,"random":0.031,"tracks":[],"combined":0.3048,"private_eq":0.3331},"39":{"raw":0.8853,"skill":0.8313,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5502,"private_eq":0.697},"40":{"raw":0.5383,"skill":0.406,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3571,"headline":false},{"track":"A · English","score":0.5383,"headline":true},{"track":"C · Arabic pairs","score":0.625,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.4754,"private_eq":0.5961},"41":{"raw":0.631,"skill":0.4465,"coverage":1,"random":0.3333,"tracks":[],"combined":0.285,"private_eq":0.5281},"42":{"raw":0.7391,"skill":0.4924,"coverage":1,"random":0.486,"tracks":[],"combined":0.34,"private_eq":0.6641},"43":{"raw":0.5018,"skill":0.2095,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3655,"private_eq":0.604},"44":{"raw":0.6032,"skill":0.2064,"coverage":1,"random":0.5,"tracks":[],"combined":0.1283,"private_eq":0.5686},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4421,"skill":0.1903,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.5095,"skill":0.019,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0577,"private_eq":0.5337},"57":{"raw":0.4377,"skill":0.3676,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.1444,"private_eq":0.2471},"58":{"raw":0.6287,"skill":0.4618,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0873,"private_eq":0.3768},"59":{"raw":0.6081,"skill":0.1874,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.1241,"private_eq":0.5819},"61":{"raw":0.6478,"skill":0.2956,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4564,"private_eq":0.7308},"62":{"raw":0.8502,"skill":0.8003,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.6108,"private_eq":0.7107},"64":{"raw":0.5568,"skill":0.446,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4385,"private_eq":0.5551}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"winnow-inference llama.cpp CUDA server (77d1458 labels-lift build), Winnow-E4B Q8_0 GGUF fully on GPU, n_ctx 131072, batch 2048 / ubatch 1024, one server and one sequential HTTP client (board: 4 servers per card)","v03":{"engine":"winnow-e4b-q8","pub":40.86,"same_skills":36.79,"new_domains":20.16,"v":32.72,"lab":false,"jev":false,"rank_v":158,"rank_pub":154,"name":"Winnow-E4B","eq":{"S":36.17,"O":27.54},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":21.7,"new_domains":11.02}}},{"engine":"clef-flash","name":"Cloudflare clef-flash","long_name":"Cloudflare clef-flash","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":9650000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Cloudflare/clef-flash","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"Cloudflare","name":"Cloudflare","logo":"data/orgs/Cloudflare.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7563,"conf":0.7764,"ece":0.0249,"brier":0.3333,"over95":0.0088,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0383,"conf":0.0325,"acc":0.0405},{"lo":0.1,"w":0.0174,"conf":0.1504,"acc":0.201},{"lo":0.2,"w":0.0233,"conf":0.2542,"acc":0.2736},{"lo":0.3,"w":0.0322,"conf":0.3543,"acc":0.3361},{"lo":0.4,"w":0.0365,"conf":0.4527,"acc":0.4505},{"lo":0.5,"w":0.0657,"conf":0.5497,"acc":0.5619},{"lo":0.6,"w":0.065,"conf":0.6526,"acc":0.6154},{"lo":0.7,"w":0.0866,"conf":0.7524,"acc":0.7096},{"lo":0.8,"w":0.1468,"conf":0.8567,"acc":0.8158},{"lo":0.9,"w":0.4883,"conf":0.9609,"acc":0.941}],"areas":{"knowledge":{"ece":0.0411,"acc":0.7332,"conf":0.7528},"language":{"ece":0.0393,"acc":0.8467,"conf":0.886},"retrieval":{"ece":0.196,"acc":0.6304,"conf":0.6984},"tools":{"ece":0.0578,"acc":0.9709,"conf":0.9193},"arts":{"ece":0.0221,"acc":0.6162,"conf":0.6076}}},"gaps":null,"latency":{"median":53.4,"p95":null,"mean":149.8},"results":{"1":{"score":0.9876,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":108.1},"2":{"score":0.6624,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1169.4},"3":{"score":0.9311,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":800.3},"4":{"score":0.9083,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":206.7},"5":{"score":0.6694,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":371.9},"6":{"score":0.7988,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":294.4},"9":{"score":0.9773,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":386.3},"10":{"score":0.3445,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":130.5},"11":{"score":0.8453,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":477.7},"12":{"score":0.5911,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":59.3},"20":{"score":0.9542,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":57.5},"21":{"score":0.7508,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":54.5},"22":{"score":0.0155,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":813.2},"23":{"score":0.6586,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":85.6},"24":{"score":0.9186,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":55.7},"25":{"score":0.5153,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":58.9},"26":{"score":0.9945,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":54.2},"27":{"score":0.9821,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":54.3},"28":{"score":0.9755,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":54.1},"29":{"score":0.9855,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":66.3},"30":{"score":0.467,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2306,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":142.8},"32":{"score":0.8617,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":131},"33":{"score":0.3661,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":153.8},"36":{"score":0.3825,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1655.7},"37":{"score":0.5741,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":78.2},"38":{"score":0.2596,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1191.4},"39":{"score":0.971,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":68.3},"40":{"score":0.1973,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":59.6},"41":{"score":0.4961,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":60.4},"42":{"score":0.7857,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":83.3},"43":{"score":0.8649,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":61.2},"44":{"score":0.9766,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":72.6},"50":{"score":0.7178,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":118.3},"56":{"score":0.7505,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":136.6},"57":{"score":0.6523,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":69},"58":{"score":0.6891,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":62},"59":{"score":0.3564,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":99.4},"61":{"score":0.6122,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":74.7},"62":{"score":0.6533,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":98.1},"64":{"score":0.6629,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":63.9}},"benchmarks":{"1":{"raw":0.9876,"skill":0.9833,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9602,"private_eq":0.9701},"2":{"raw":0.6624,"skill":0.6101,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6976,"private_eq":0.7403},"3":{"raw":0.9311,"skill":0.9298,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7648,"private_eq":0.7709},"4":{"raw":0.9083,"skill":0.9071,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6407,"private_eq":0.6483},"5":{"raw":0.6694,"skill":0.6674,"coverage":1,"random":0.006,"tracks":[],"combined":0.2599,"private_eq":0.2717},"6":{"raw":0.7988,"skill":0.5283,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9773,"skill":0.9773,"coverage":1,"random":0,"tracks":[],"combined":0.5661,"private_eq":0.57},"10":{"raw":0.3445,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8453,"skill":0.7763,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5911,"skill":0.3875,"coverage":1,"random":0.3324,"tracks":[],"combined":0.476,"private_eq":0.6535},"20":{"raw":0.9541,"skill":0.9081,"coverage":1,"random":0.5,"tracks":[],"combined":0.8709,"private_eq":0.9357},"21":{"raw":0.7508,"skill":0.5015,"coverage":1,"random":0.5,"tracks":[],"combined":0.2565,"private_eq":0.632},"22":{"raw":0.0145,"skill":0.0068,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0614,"private_eq":0.0783},"23":{"raw":0.6595,"skill":0.319,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.9186,"skill":0.8915,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5153,"skill":0.3537,"coverage":1,"random":0.25,"tracks":[],"combined":0.1671,"private_eq":0.3817},"26":{"raw":0.9945,"skill":0.9927,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9821,"skill":0.9761,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9755,"skill":0.951,"coverage":1,"random":0.5,"tracks":[],"combined":0.4967,"private_eq":0.7507},"29":{"raw":0.9855,"skill":0.9807,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.467,"skill":0.3628,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5951,"random":0.25},{"track":"10-choice","raw":0.3389,"random":0.1}],"combined":0.4761,"private_eq":0.5719},"31":{"raw":0.2306,"skill":0.162,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0669,"private_eq":0.1522},"32":{"raw":0.8617,"skill":0.7801,"coverage":1,"random":0.371,"tracks":[],"combined":0.2042,"private_eq":0.5045},"33":{"raw":0.3661,"skill":0.3577,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3022,"private_eq":0.3182},"36":{"raw":0.3825,"skill":0.3014,"coverage":1,"random":0.116,"tracks":[],"combined":0.3546,"private_eq":0.4351},"37":{"raw":0.5741,"skill":0.4658,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5486,"private_eq":0.6434},"38":{"raw":0.2596,"skill":0.2359,"coverage":1,"random":0.031,"tracks":[],"combined":0.3614,"private_eq":0.3873},"39":{"raw":0.971,"skill":0.9573,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5685,"private_eq":0.7093},"40":{"raw":0.1973,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.8786,"headline":false},{"track":"A · English","score":0.1973,"headline":true},{"track":"C · Arabic pairs","score":0.95,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4961,"skill":0.2441,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0057,"private_eq":0.344},"42":{"raw":0.7857,"skill":0.5831,"coverage":1,"random":0.486,"tracks":[],"combined":0.7143,"private_eq":0.8543},"43":{"raw":0.8649,"skill":0.7857,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2904,"private_eq":0.5572},"44":{"raw":0.9766,"skill":0.9532,"coverage":1,"random":0.5,"tracks":[],"combined":0.8404,"private_eq":0.9206},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.7178,"skill":0.5904,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.7505,"skill":0.501,"coverage":1,"random":0.5,"tracks":[],"combined":0.3607,"private_eq":0.6835},"57":{"raw":0.6523,"skill":0.6089,"coverage":1,"random":0.1109,"tracks":[],"combined":0.2612,"private_eq":0.3497},"58":{"raw":0.6891,"skill":0.5494,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1836,"private_eq":0.4425},"59":{"raw":0.3564,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6122,"skill":0.2244,"coverage":1,"random":0.5,"tracks":[],"combined":0.2804,"private_eq":0.6438},"62":{"raw":0.6533,"skill":0.5377,"coverage":1,"random":0.25,"tracks":[],"combined":0.2275,"private_eq":0.4265},"64":{"raw":0.6629,"skill":0.5786,"coverage":1,"random":0.2,"tracks":[],"combined":0.5064,"private_eq":0.6088}},"coverage":1,"pending":0,"scores":{"balanced_raw":66.53,"balanced_skill":44.43,"breadth_skill":55.17,"public_skill":56.15},"frozen_scores":{"balanced_raw":66.53,"balanced_skill":56.15,"breadth_skill":55.17},"categories":[{"id":"knowledge","raw":0.6089,"skill":0.5298,"combined":0.325,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6174,"skill":0.4722,"combined":0.4049,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6499,"skill":0.5216,"combined":0.4085,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.851,"skill":0.8187,"combined":0.6595,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6266,"skill":0.4841,"combined":0.3927,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"clef-flash","pub":56.15,"same_skills":44.01,"new_domains":35,"v":44.43,"lab":false,"jev":false,"rank_v":113,"rank_pub":95,"name":"Cloudflare clef-flash","eq":{"S":43.46,"O":45.87},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":26.08,"new_domains":18.35}}},{"engine":"decider-4b","name":"Decider 4B","long_name":"Decider 4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":8411558400,"model_url":"https://huggingface.co/Mapika/decider-4b","code_url":"https://github.com/Mapika/decider","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"Mapika/decider-4b","served_checkpoint":null},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.6488,"conf":0.7326,"ece":0.0837,"brier":0.4542,"over95":0.0178,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0381,"conf":0.0703,"acc":0.0666},{"lo":0.1,"w":0.0214,"conf":0.1363,"acc":0.1303},{"lo":0.2,"w":0.0163,"conf":0.2568,"acc":0.2542},{"lo":0.3,"w":0.0444,"conf":0.3544,"acc":0.2521},{"lo":0.4,"w":0.0582,"conf":0.4494,"acc":0.3416},{"lo":0.5,"w":0.1145,"conf":0.5487,"acc":0.471},{"lo":0.6,"w":0.1007,"conf":0.6494,"acc":0.5228},{"lo":0.7,"w":0.101,"conf":0.752,"acc":0.6267},{"lo":0.8,"w":0.12,"conf":0.8523,"acc":0.7324},{"lo":0.9,"w":0.3853,"conf":0.9716,"acc":0.9095}],"areas":{"knowledge":{"ece":0.0964,"acc":0.5764,"conf":0.6706},"language":{"ece":0.0434,"acc":0.7858,"conf":0.8292},"retrieval":{"ece":0.1502,"acc":0.6088,"conf":0.7468},"tools":{"ece":0.0634,"acc":0.8063,"conf":0.8675},"arts":{"ece":0.1048,"acc":0.4994,"conf":0.6026}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":12.6,"p95":188.5,"mean":41},"results":{"1":{"score":0.8796,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":33.7},"2":{"score":0.65,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":288.9},"3":{"score":0.6043,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7472,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":21.9},"5":{"score":0.8655,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":44},"6":{"score":0.7988,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":172.2},"9":{"score":0.1705,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.2003,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":45.2},"11":{"score":0.7234,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5636,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":9.2},"20":{"score":0.8344,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":9.2},"21":{"score":0.5967,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":8.1},"22":{"score":0.1,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":242.7},"23":{"score":0.6048,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.3},"24":{"score":0.7418,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":8.5},"25":{"score":0.4133,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9781,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":8.3},"27":{"score":0.9394,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":8.4},"28":{"score":0.8074,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":7.9},"29":{"score":0.9399,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":9.5},"30":{"score":0.4056,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.105,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.7},"32":{"score":0.5372,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3158,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":127.2},"36":{"score":0.4545,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4238,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.6},"38":{"score":0.1924,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":234.8},"39":{"score":0.8604,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4207,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":8.2,"tracks":[{"track":"A · Arabic","score":0.0829,"headline":false},{"track":"A · English","score":0.4207,"headline":true},{"track":"C · Arabic pairs","score":0.595,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5353,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":9.2},"42":{"score":0.7369,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":22.2},"43":{"score":0.4368,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.633,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":9.2},"45":{"score":0.1038,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4063,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":22.9},"56":{"score":0.578,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":79,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.5165,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":9.4,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6089,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":9.2,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5216,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":21.1,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5627,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":13.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.7588,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":22.6,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.6288,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":56.4,"balanced_skill":36.3,"breadth_skill":40.04,"public_skill":41.19},"frozen_scores":{"balanced_raw":56.4,"balanced_skill":41.19,"breadth_skill":40.04},"categories":[{"id":"knowledge","raw":0.4478,"skill":0.277,"combined":0.2635,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6165,"skill":0.4399,"combined":0.3386,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6129,"skill":0.4467,"combined":0.4223,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6222,"skill":0.5862,"combined":0.4773,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5249,"skill":0.3004,"combined":0.3452,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.8796,"skill":0.8374,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6741,"private_eq":0.7606},"2":{"raw":0.65,"skill":0.5959,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6683,"private_eq":0.7152},"3":{"raw":0.6043,"skill":0.5967,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3397,"private_eq":0.3586},"4":{"raw":0.7472,"skill":0.7439,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7505,"private_eq":0.7555},"5":{"raw":0.8655,"skill":0.8647,"coverage":1,"random":0.006,"tracks":[],"combined":0.5318,"private_eq":0.5389},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.1705,"skill":0.1705,"coverage":1,"random":0,"tracks":[],"combined":0.1842,"private_eq":0.1925},"11":{"raw":0.7234,"skill":0.6,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5636,"skill":0.3463,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3389,"private_eq":0.563},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.833,"skill":0.666,"coverage":1,"random":0.5,"tracks":[],"combined":0.683,"private_eq":0.8428},"21":{"raw":0.5967,"skill":0.1933,"coverage":1,"random":0.5,"tracks":[],"combined":0.176,"private_eq":0.5922},"22":{"raw":0.0934,"skill":0.0863,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1019,"private_eq":0.1181},"23":{"raw":0.5913,"skill":0.1825,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7598,"skill":0.6798,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[],"combined":0.1461,"private_eq":0.3661},"28":{"raw":0.8074,"skill":0.6148,"coverage":1,"random":0.5,"tracks":[],"combined":0.3654,"private_eq":0.6858},"29":{"raw":0.9399,"skill":0.9199,"coverage":1,"random":0.25,"tracks":[],"combined":0.9138,"private_eq":0.9353},"30":{"raw":0.4056,"skill":0.2889,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5444,"random":0.25},{"track":"10-choice","raw":0.2669,"random":0.1}],"combined":0.4953,"private_eq":0.5875},"31":{"raw":0.105,"skill":0.0252,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0398,"private_eq":0.1276},"32":{"raw":0.5372,"skill":0.2642,"coverage":1,"random":0.371,"tracks":[],"combined":0.0262,"private_eq":0.3938},"33":{"raw":0.3158,"skill":0.3067,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3126,"private_eq":0.3283},"36":{"raw":0.4545,"skill":0.3829,"coverage":1,"random":0.116,"tracks":[],"combined":0.3269,"private_eq":0.4109},"37":{"raw":0.4238,"skill":0.2773,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3688,"private_eq":0.5017},"38":{"raw":0.1924,"skill":0.1666,"coverage":1,"random":0.031,"tracks":[],"combined":0.2401,"private_eq":0.2711},"39":{"raw":0.8604,"skill":0.7947,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3402,"private_eq":0.5558},"40":{"raw":0.4207,"skill":0.2548,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0829,"headline":false},{"track":"A · English","score":0.4207,"headline":true},{"track":"C · Arabic pairs","score":0.595,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.5353,"skill":0.3029,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.7369,"skill":0.4881,"coverage":1,"random":0.486,"tracks":[],"combined":0.7924,"private_eq":0.894},"43":{"raw":0.4368,"skill":0.1065,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3697,"private_eq":0.6066},"44":{"raw":0.633,"skill":0.266,"coverage":1,"random":0.5,"tracks":[],"combined":0.3263,"private_eq":0.6665},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4063,"skill":0.1384,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.578,"skill":0.156,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1443,"private_eq":0.5765},"57":{"raw":0.5165,"skill":0.4562,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.4133,"private_eq":0.4834},"58":{"raw":0.6089,"skill":0.4331,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1385,"private_eq":0.4117},"59":{"raw":0.5216,"skill":0.0081,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5627,"skill":0.1254,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3429,"private_eq":0.6747},"62":{"raw":0.7588,"skill":0.6784,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5082,"private_eq":0.6346},"64":{"raw":0.6288,"skill":0.536,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4968,"private_eq":0.6012}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author decider.serve v1.2.1 with DECIDER_FP8=1 and DECIDER_COMPILE=1 on top of CUDA graphs (89 captured), one sequential HTTP client; the board ran graphs only. FP8+compile: 12.6 / 188.5 ms; graphs only: 13.6 / 172.7 ms (kept as latency-v1-full.stock). Chosen on the median","v03":{"engine":"decider-4b","pub":41.19,"same_skills":37.19,"new_domains":26.93,"v":36.3,"lab":false,"jev":false,"rank_v":145,"rank_pub":149,"name":"Decider 4B","eq":{"S":36.57,"O":35.91},"rank_tie":33,"tied":true,"parts":{"public":0,"same_skills":21.94,"new_domains":14.36}}},{"engine":"vllm-pr57250","name":"mmastrac diffusiongemma (vLLM PR 57250)","long_name":"mmastrac diffgemma","variant":null,"meta":{"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/vllm-project/vllm/pull/57250","base_model":"google/diffusiongemma-26B-A4B-it","base_url":"https://huggingface.co/google/diffusiongemma-26B-A4B-it","weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it"},"completed":95679,"counts":{"ok":95679,"unsupported":14522},"lifted":false,"calibration":{"acc":0.6495,"conf":0.8513,"ece":0.2019,"brier":0.535,"over95":0.0907,"n":70144,"benchmarks":28,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0006,"conf":0.1821,"acc":0.2358},{"lo":0.2,"w":0.008,"conf":0.2643,"acc":0.0903},{"lo":0.3,"w":0.0199,"conf":0.3628,"acc":0.2245},{"lo":0.4,"w":0.0327,"conf":0.459,"acc":0.2679},{"lo":0.5,"w":0.0781,"conf":0.5493,"acc":0.3685},{"lo":0.6,"w":0.0803,"conf":0.6521,"acc":0.4727},{"lo":0.7,"w":0.0839,"conf":0.7504,"acc":0.5228},{"lo":0.8,"w":0.1121,"conf":0.8526,"acc":0.5651},{"lo":0.9,"w":0.5845,"conf":0.9804,"acc":0.7895}],"areas":{"knowledge":{"ece":0.2351,"acc":0.5659,"conf":0.8009},"language":{"ece":0.1587,"acc":0.7472,"conf":0.9028},"retrieval":{"ece":0.2883,"acc":0.5026,"conf":0.7909},"tools":{"ece":0.0763,"acc":0.8329,"conf":0.9092},"arts":{"ece":0.2341,"acc":0.6413,"conf":0.8755}}},"gaps":{"share":0.1207,"reasons":[{"reason":"supports at most 26 answer options per question","share":0.1205},{"reason":"structured_server: question 'q_78d37c943d': label 'C' is not a single token","share":0},{"reason":"structured_server: question 'q_c3e60b168d': label 'C' is not a single token","share":0}]},"latency":{"median":126.7,"p95":274.6,"mean":162.8},"results":{"1":{"score":0.9622,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":132},"2":{"score":0.4631,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":664.5},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7988,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":251.1},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":66,"unsupported":22,"errors":0,"median_ms":null},"10":{"score":0.4299,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":96.3},"11":{"score":0.5632,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5574,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":114.8},"20":{"score":0.8854,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123},"21":{"score":0.6084,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":117.3},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.6098,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":141},"24":{"score":0.7862,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":114.5},"25":{"score":0.4898,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9811,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":50.9},"27":{"score":0.9403,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":50.1},"28":{"score":0.708,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":116.4},"29":{"score":0.8173,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":58.5},"30":{"score":0.207,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2173,"jev":null,"metric":"accuracy","requests":5000,"answered":1588,"unsupported":3412,"errors":0,"median_ms":142.9},"32":{"score":0.625,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2564,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":176.5},"36":{"score":0.2259,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5339,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":132.1},"38":{"score":0.1443,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":728.1},"39":{"score":0.8979,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5895,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":119.4,"tracks":[{"track":"A · Arabic","score":0.4853,"headline":false},{"track":"A · English","score":0.5895,"headline":true},{"track":"C · Arabic pairs","score":0.815,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.6086,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":118.2},"42":{"score":0.7446,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":139.5},"43":{"score":0.4526,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6356,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.1},"45":{"score":0.1038,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4666,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":163},"56":{"score":0.824,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":212.5,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.413,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":128.9,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6644,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":122.1,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.6789,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":154.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6913,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":134.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.4874,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":118.6,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5739,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":46.55,"balanced_skill":34.25,"breadth_skill":31.84,"public_skill":32.69},"frozen_scores":{"balanced_raw":46.55,"balanced_skill":32.69,"breadth_skill":31.84},"categories":[{"id":"knowledge","raw":0.425,"skill":0.2499,"combined":0.2321,"coverage":0.9431,"pending":0,"n":11},{"id":"language","raw":0.6252,"skill":0.4648,"combined":0.4771,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3516,"skill":0.2417,"combined":0.2302,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.3898,"skill":0.3399,"combined":0.3658,"coverage":0.7315,"pending":0,"n":5},{"id":"arts","raw":0.5241,"skill":0.317,"combined":0.3321,"coverage":0.8333,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9622,"skill":0.949,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8852,"private_eq":0.9152},"2":{"raw":0.4631,"skill":0.3799,"coverage":1,"random":0.1341,"tracks":[],"combined":0.4563,"private_eq":0.5337},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.7988,"skill":0.5256,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7852,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":0.75,"random":0,"tracks":[],"combined":0.0021,"private_eq":0.0125},"11":{"raw":0.5632,"skill":0.3683,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5574,"skill":0.337,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4978,"private_eq":0.6679},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8896,"skill":0.7791,"coverage":1,"random":0.5,"tracks":[],"combined":0.8204,"private_eq":0.9107},"21":{"raw":0.6084,"skill":0.2169,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.6063,"skill":0.2125,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7911,"skill":0.7215,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4898,"skill":0.3197,"coverage":1,"random":0.25,"tracks":[],"combined":0.1223,"private_eq":0.3485},"28":{"raw":0.708,"skill":0.416,"coverage":1,"random":0.5,"tracks":[],"combined":0.4714,"private_eq":0.7382},"29":{"raw":0.8173,"skill":0.7564,"coverage":1,"random":0.25,"tracks":[],"combined":0.7405,"private_eq":0.8068},"30":{"raw":0.207,"skill":0.0387,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2782,"random":0.25},{"track":"10-choice","raw":0.1357,"random":0.1}],"combined":0.1115,"private_eq":0.2745},"31":{"raw":0.069,"skill":0,"coverage":0.3176,"random":0.0819,"tracks":[],"combined":0.0225,"private_eq":0.1119},"32":{"raw":0.625,"skill":0.4038,"coverage":1,"random":0.371,"tracks":[],"combined":0.1963,"private_eq":0.4996},"33":{"raw":0.2564,"skill":0.2465,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2509,"private_eq":0.2681},"36":{"raw":0.2259,"skill":0.1243,"coverage":1,"random":0.116,"tracks":[],"combined":0.1561,"private_eq":0.2616},"37":{"raw":0.5339,"skill":0.4154,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3874,"private_eq":0.5163},"38":{"raw":0.1443,"skill":0.1169,"coverage":1,"random":0.031,"tracks":[],"combined":0.24,"private_eq":0.271},"39":{"raw":0.8979,"skill":0.8498,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4706,"private_eq":0.6435},"40":{"raw":0.5895,"skill":0.4719,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4853,"headline":false},{"track":"A · English","score":0.5895,"headline":true},{"track":"C · Arabic pairs","score":0.815,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.4994,"private_eq":0.6145},"41":{"raw":0.6086,"skill":0.4128,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4249,"private_eq":0.6203},"42":{"raw":0.7446,"skill":0.5031,"coverage":1,"random":0.486,"tracks":[],"combined":0.7598,"private_eq":0.8774},"43":{"raw":0.4526,"skill":0.1315,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2276,"private_eq":0.5181},"44":{"raw":0.6356,"skill":0.2712,"coverage":1,"random":0.5,"tracks":[],"combined":0.3318,"private_eq":0.6692},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4666,"skill":0.2258,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.824,"skill":0.648,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.5078,"private_eq":0.7562},"57":{"raw":0.413,"skill":0.3398,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.4,"private_eq":0.4717},"58":{"raw":0.6644,"skill":0.5136,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1448,"private_eq":0.416},"59":{"raw":0.6789,"skill":0.3342,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.1271,"private_eq":0.5833},"61":{"raw":0.6913,"skill":0.3826,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.4621,"private_eq":0.7336},"62":{"raw":0.4874,"skill":0.3165,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4672,"private_eq":0.6042},"64":{"raw":0.5739,"skill":0.4674,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4677,"private_eq":0.5782}},"coverage":0.6927,"pending":0.24,"latency_rows":678,"latency_path":"structured_server.py over vLLM PR 57250 nightly, prefix caching, default attention backend, one sequential HTTP client","v03":{"engine":"vllm-pr57250","pub":32.69,"same_skills":33.44,"new_domains":27.38,"v":34.25,"lab":false,"jev":false,"rank_v":155,"rank_pub":180,"name":"mmastrac diffgemma","eq":{"S":32.78,"O":36.47},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":19.67,"new_domains":14.59}}},{"engine":"intelif-qwen3-4b","name":"Intelif Qwen3-4B","long_name":"Intelif Qwen3-4B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4020000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/UserMoonlight/intelif-qwen3-4b","code_url":null,"base_model":"Qwen/Qwen3-4B","base_url":"https://huggingface.co/Qwen/Qwen3-4B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6152,"conf":0.6842,"ece":0.0708,"brier":0.5064,"over95":0.0158,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0234,"conf":0.0758,"acc":0.084},{"lo":0.1,"w":0.0361,"conf":0.1429,"acc":0.1627},{"lo":0.2,"w":0.0199,"conf":0.2505,"acc":0.196},{"lo":0.3,"w":0.0479,"conf":0.3545,"acc":0.3011},{"lo":0.4,"w":0.0712,"conf":0.4498,"acc":0.3479},{"lo":0.5,"w":0.1471,"conf":0.5489,"acc":0.5264},{"lo":0.6,"w":0.1422,"conf":0.6492,"acc":0.6194},{"lo":0.7,"w":0.1282,"conf":0.7509,"acc":0.6787},{"lo":0.8,"w":0.1415,"conf":0.852,"acc":0.7205},{"lo":0.9,"w":0.2424,"conf":0.9626,"acc":0.8653}],"areas":{"knowledge":{"ece":0.0583,"acc":0.5568,"conf":0.6136},"language":{"ece":0.0766,"acc":0.6651,"conf":0.7405},"retrieval":{"ece":0.0657,"acc":0.7317,"conf":0.7954},"tools":{"ece":0.0997,"acc":0.7864,"conf":0.8582},"arts":{"ece":0.103,"acc":0.4583,"conf":0.545}}},"gaps":null,"latency":{"median":22.8,"p95":null,"mean":53.8},"results":{"1":{"score":0.9227,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":43},"2":{"score":0.6772,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":723.2},"3":{"score":0.6457,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":218.6},"4":{"score":0.8378,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.1},"5":{"score":0.8161,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":54.6},"6":{"score":0.7978,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":129.5},"9":{"score":0.0227,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":560.9},"10":{"score":0.5494,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":35.1},"11":{"score":0.5623,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1182.9},"12":{"score":0.4997,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":12.2},"20":{"score":0.7454,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.3},"21":{"score":0.5278,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":11},"22":{"score":0.127,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":183.6},"23":{"score":0.5788,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.9},"24":{"score":0.6447,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":11.6},"25":{"score":0.3214,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":13.3},"26":{"score":0.9609,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":11.3},"27":{"score":0.8882,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":11.4},"28":{"score":0.6504,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":10.8},"29":{"score":0.7619,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":14},"30":{"score":0.3412,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1496,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.1},"32":{"score":0.6051,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":35.2},"33":{"score":0.1661,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":100.8},"36":{"score":0.4048,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1233.3},"37":{"score":0.2907,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.4},"38":{"score":0.1908,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":258.1},"39":{"score":0.8055,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":16},"40":{"score":0.2852,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":11.4},"41":{"score":0.394,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":12.6},"42":{"score":0.69,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":20.9},"43":{"score":0.4035,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":12.4},"44":{"score":0.5758,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.2},"45":{"score":0.1158,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":14.1},"50":{"score":0.327,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":31.3},"56":{"score":0.5,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":64.4},"57":{"score":0.4126,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":13.5},"58":{"score":0.5642,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":12.3},"59":{"score":0.2694,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":30.2},"61":{"score":0.5635,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":20},"62":{"score":0.4455,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":27},"64":{"score":0.5398,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":12.6}},"benchmarks":{"1":{"raw":0.9227,"skill":0.8956,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8171,"private_eq":0.8653},"2":{"raw":0.6772,"skill":0.6272,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6959,"private_eq":0.7388},"3":{"raw":0.6457,"skill":0.6389,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7176,"private_eq":0.7251},"4":{"raw":0.8378,"skill":0.8357,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8385,"private_eq":0.8414},"5":{"raw":0.8161,"skill":0.815,"coverage":1,"random":0.006,"tracks":[],"combined":0.6037,"private_eq":0.6096},"6":{"raw":0.7978,"skill":0.5259,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0227,"skill":0.0227,"coverage":1,"random":0,"tracks":[],"combined":0.0375,"private_eq":0.0475},"10":{"raw":0.5494,"skill":0.2502,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5623,"skill":0.367,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4997,"skill":0.2506,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3077,"private_eq":0.5424},"20":{"raw":0.7494,"skill":0.4988,"coverage":1,"random":0.5,"tracks":[],"combined":0.5816,"private_eq":0.7927},"21":{"raw":0.5278,"skill":0.0556,"coverage":1,"random":0.5,"tracks":[],"combined":0.0352,"private_eq":0.5226},"22":{"raw":0.1249,"skill":0.118,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1596,"private_eq":0.1747},"23":{"raw":0.58,"skill":0.1601,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6447,"skill":0.5263,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[],"combined":0.0691,"private_eq":0.309},"26":{"raw":0.9609,"skill":0.9479,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8882,"skill":0.8509,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6504,"skill":0.3008,"coverage":1,"random":0.5,"tracks":[],"combined":0.3201,"private_eq":0.6634},"29":{"raw":0.7619,"skill":0.6825,"coverage":1,"random":0.25,"tracks":[],"combined":0.6118,"private_eq":0.7114},"30":{"raw":0.3412,"skill":0.2061,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4435,"random":0.25},{"track":"10-choice","raw":0.2388,"random":0.1}],"combined":0.4183,"private_eq":0.5247},"31":{"raw":0.1496,"skill":0.0738,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0593,"private_eq":0.1453},"32":{"raw":0.6051,"skill":0.3722,"coverage":1,"random":0.371,"tracks":[],"combined":0.0726,"private_eq":0.4227},"33":{"raw":0.1661,"skill":0.155,"coverage":1,"random":0.0131,"tracks":[],"combined":0.256,"private_eq":0.2731},"36":{"raw":0.4048,"skill":0.3266,"coverage":1,"random":0.116,"tracks":[],"combined":0.3099,"private_eq":0.396},"37":{"raw":0.2907,"skill":0.1104,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1183,"private_eq":0.3042},"38":{"raw":0.1908,"skill":0.1649,"coverage":1,"random":0.031,"tracks":[],"combined":0.2527,"private_eq":0.2831},"39":{"raw":0.8055,"skill":0.7139,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4965,"private_eq":0.6609},"40":{"raw":0.2852,"skill":0.0804,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2855,"headline":false},{"track":"A · English","score":0.2852,"headline":true},{"track":"C · Arabic pairs","score":0.41,"headline":false},{"track":"C · English pairs","score":0.665,"headline":false}],"combined":0.0716,"private_eq":0.2858},"41":{"raw":0.394,"skill":0.0911,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4885,"private_eq":0.6622},"42":{"raw":0.69,"skill":0.3969,"coverage":1,"random":0.486,"tracks":[],"combined":0.6714,"private_eq":0.8325},"43":{"raw":0.4035,"skill":0.0536,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3279,"private_eq":0.5806},"44":{"raw":0.5758,"skill":0.1516,"coverage":1,"random":0.5,"tracks":[],"combined":0.1817,"private_eq":0.595},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.327,"skill":0.0232,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5038},"57":{"raw":0.4126,"skill":0.3393,"coverage":1,"random":0.1109,"tracks":[],"combined":0.2366,"private_eq":0.3281},"58":{"raw":0.5642,"skill":0.3683,"coverage":1,"random":0.3101,"tracks":[],"combined":0.1019,"private_eq":0.3868},"59":{"raw":0.2694,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5635,"skill":0.127,"coverage":1,"random":0.5,"tracks":[],"combined":0.2011,"private_eq":0.6046},"62":{"raw":0.4455,"skill":0.2607,"coverage":1,"random":0.25,"tracks":[],"combined":0.3919,"private_eq":0.5484},"64":{"raw":0.5398,"skill":0.4247,"coverage":1,"random":0.2,"tracks":[],"combined":0.3706,"private_eq":0.5014}},"coverage":1,"pending":0,"scores":{"balanced_raw":49.72,"balanced_skill":31.33,"breadth_skill":31.02,"public_skill":32.47},"frozen_scores":{"balanced_raw":49.72,"balanced_skill":32.47,"breadth_skill":31.02},"categories":[{"id":"knowledge","raw":0.3932,"skill":0.1947,"combined":0.2027,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5012,"skill":0.3121,"combined":0.3659,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5795,"skill":0.3955,"combined":0.3665,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5606,"skill":0.5096,"combined":0.5494,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4748,"skill":0.2134,"combined":0.2715,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"intelif-qwen3-4b","pub":32.47,"same_skills":35.92,"new_domains":18.43,"v":31.33,"lab":false,"jev":false,"rank_v":163,"rank_pub":181,"name":"Intelif Qwen3-4B","eq":{"S":35.28,"O":25.41},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":21.17,"new_domains":10.16}}},{"engine":"jobe","name":"Jobe","long_name":"Jobe Qwen3.5-4B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/MantisShrimpdev/jobe","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":"Qwen/Qwen3.5-4B"},"completed":110201,"counts":{"ok":110201},"lifted":true,"calibration":{"acc":0.6065,"conf":0.6499,"ece":0.1232,"brier":0.536,"over95":0.0226,"n":35093,"benchmarks":32,"rel":[{"lo":0,"w":0.099,"conf":0.0569,"acc":0.2647},{"lo":0.1,"w":0.0443,"conf":0.1358,"acc":0.4986},{"lo":0.2,"w":0.0211,"conf":0.2494,"acc":0.4015},{"lo":0.3,"w":0.0406,"conf":0.3549,"acc":0.2825},{"lo":0.4,"w":0.0547,"conf":0.4491,"acc":0.3432},{"lo":0.5,"w":0.1407,"conf":0.5488,"acc":0.5012},{"lo":0.6,"w":0.0933,"conf":0.6498,"acc":0.5601},{"lo":0.7,"w":0.1043,"conf":0.7512,"acc":0.6157},{"lo":0.8,"w":0.1017,"conf":0.8525,"acc":0.6877},{"lo":0.9,"w":0.3003,"conf":0.9695,"acc":0.8742}],"areas":{"knowledge":{"ece":0.0978,"acc":0.5389,"conf":0.6363},"language":{"ece":0.1308,"acc":0.6964,"conf":0.8237},"retrieval":{"ece":0.3404,"acc":0.6107,"conf":0.4449},"tools":{"ece":0.1215,"acc":0.7509,"conf":0.6344},"arts":{"ece":0.046,"acc":0.5032,"conf":0.5405}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":53.5,"p95":2910.3,"mean":322.6},"results":{"1":{"score":0.9298,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":108.8},"2":{"score":0.5937,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1662.3},"3":{"score":0.4606,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6237,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1385.1},"5":{"score":0.5441,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":2931.6},"6":{"score":0.7952,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":219.4},"9":{"score":0.0795,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5657,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":58.5},"11":{"score":0.5924,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5453,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":50.9},"20":{"score":0.8516,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.6},"21":{"score":0.6046,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":50.7},"22":{"score":0.0515,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3239.2},"23":{"score":0.5688,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.7},"24":{"score":0.7127,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":51},"25":{"score":0.3724,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9684,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":51},"27":{"score":0.9172,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.6},"28":{"score":0.6606,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":50.2},"29":{"score":0.8351,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":52.2},"30":{"score":0.2418,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.134,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":560},"32":{"score":0.5439,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2018,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":415.9},"36":{"score":0.4118,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3581,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.5},"38":{"score":0.134,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3228.1},"39":{"score":0.6581,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3896,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":51.2,"tracks":[{"track":"A · Arabic","score":0.2774,"headline":false},{"track":"A · English","score":0.3896,"headline":true},{"track":"C · Arabic pairs","score":0.65,"headline":false},{"track":"C · English pairs","score":0.825,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4366,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":51.6},"42":{"score":0.7185,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":53.3},"43":{"score":0.3912,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5638,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.9},"45":{"score":0.1138,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4427,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":58.4},"56":{"score":0.6,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":458.3,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4364,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":54.9,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.5733,"metric":"accuracy","requests":5507,"answered":5332,"unsupported":175,"errors":0,"median_ms":51.9,"edition_added":"v0.2","jev":null},"59":{"score":0.5763,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":54.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6165,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":53.4,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.549,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":57.1,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5814,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":49.65,"balanced_skill":30.35,"breadth_skill":31.67,"public_skill":32.93},"frozen_scores":{"balanced_raw":49.65,"balanced_skill":32.93,"breadth_skill":31.67},"categories":[{"id":"knowledge","raw":0.3869,"skill":0.1845,"combined":0.1621,"coverage":0.9968,"pending":0,"n":11},{"id":"language","raw":0.5491,"skill":0.3485,"combined":0.3601,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5258,"skill":0.3669,"combined":0.375,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5353,"skill":0.4881,"combined":0.4816,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5141,"skill":0.2887,"combined":0.3507,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9298,"skill":0.9052,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8034,"private_eq":0.8553},"2":{"raw":0.5937,"skill":0.5308,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6346,"private_eq":0.6863},"3":{"raw":0.4606,"skill":0.4502,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3869,"private_eq":0.4044},"4":{"raw":0.6237,"skill":0.6189,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6523,"private_eq":0.6596},"5":{"raw":0.5441,"skill":0.5413,"coverage":1,"random":0.006,"tracks":[],"combined":0.407,"private_eq":0.4163},"6":{"raw":0.7953,"skill":0.5167,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7834,"headline":false},{"track":"RouterBench-5shot","score":0.8071,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0795,"skill":0.0795,"coverage":1,"random":0,"tracks":[],"combined":0.0502,"private_eq":0.06},"11":{"raw":0.5924,"skill":0.4105,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5453,"skill":0.3189,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4403,"private_eq":0.6299},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8512,"skill":0.7024,"coverage":1,"random":0.5,"tracks":[],"combined":0.7479,"private_eq":0.8749},"21":{"raw":0.6046,"skill":0.2093,"coverage":1,"random":0.5,"tracks":[],"combined":0.2565,"private_eq":0.632},"22":{"raw":0.0494,"skill":0.042,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0905,"private_eq":0.1069},"23":{"raw":0.5552,"skill":0.1104,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7242,"skill":0.6322,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3724,"skill":0.1633,"coverage":1,"random":0.25,"tracks":[],"combined":0.0943,"private_eq":0.3277},"28":{"raw":0.6606,"skill":0.3212,"coverage":1,"random":0.5,"tracks":[],"combined":0.1133,"private_eq":0.5612},"29":{"raw":0.8351,"skill":0.7801,"coverage":1,"random":0.25,"tracks":[],"combined":0.6514,"private_eq":0.7408},"30":{"raw":0.2418,"skill":0.083,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3283,"random":0.25},{"track":"10-choice","raw":0.1554,"random":0.1}],"combined":0.2733,"private_eq":0.4065},"31":{"raw":0.134,"skill":0.0568,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0799,"private_eq":0.164},"32":{"raw":0.5439,"skill":0.2749,"coverage":1,"random":0.371,"tracks":[],"combined":0.0262,"private_eq":0.3938},"33":{"raw":0.2018,"skill":0.1912,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2098,"private_eq":0.228},"36":{"raw":0.4118,"skill":0.3346,"coverage":1,"random":0.116,"tracks":[],"combined":0.2895,"private_eq":0.3782},"37":{"raw":0.3581,"skill":0.1948,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3868,"private_eq":0.5159},"38":{"raw":0.134,"skill":0.1063,"coverage":1,"random":0.031,"tracks":[],"combined":0.164,"private_eq":0.1982},"39":{"raw":0.6581,"skill":0.4971,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3802,"private_eq":0.5827},"40":{"raw":0.3896,"skill":0.2147,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2774,"headline":false},{"track":"A · English","score":0.3896,"headline":true},{"track":"C · Arabic pairs","score":0.65,"headline":false},{"track":"C · English pairs","score":0.825,"headline":false}],"combined":0.1727,"private_eq":0.3635},"41":{"raw":0.4366,"skill":0.1549,"coverage":1,"random":0.3333,"tracks":[],"combined":0.372,"private_eq":0.5854},"42":{"raw":0.7185,"skill":0.4523,"coverage":1,"random":0.486,"tracks":[],"combined":0.6186,"private_eq":0.8057},"43":{"raw":0.3912,"skill":0.0341,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2819,"private_eq":0.5519},"44":{"raw":0.5638,"skill":0.1276,"coverage":1,"random":0.5,"tracks":[],"combined":0.2727,"private_eq":0.64},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4427,"skill":0.1912,"coverage":1,"random":0.311,"tracks":[],"combined":0.1523,"private_eq":0.4219},"56":{"raw":0.6,"skill":0.2,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1271,"private_eq":0.568},"57":{"raw":0.4364,"skill":0.3661,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2167,"private_eq":0.3106},"58":{"raw":0.5551,"skill":0.3551,"coverage":0.9682,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0816,"private_eq":0.3729},"59":{"raw":0.5763,"skill":0.1215,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.0069,"private_eq":0.526},"61":{"raw":0.6165,"skill":0.233,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3429,"private_eq":0.6747},"62":{"raw":0.549,"skill":0.3987,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4877,"private_eq":0.6194},"64":{"raw":0.5814,"skill":0.4768,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5064,"private_eq":0.6088}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author jobe.score per question with SDPA attention (published default eager; sdpa is faster: median 53.5 vs 55.9 ms, p95 2910 vs 3088 ms); JOBE_LIFT=1 board lifts; venv-q torch 2.14.0+cu130. The eager run is kept as latency-v1-full.eager","v03":{"engine":"jobe","pub":32.93,"same_skills":34.34,"new_domains":18.39,"v":30.35,"lab":false,"jev":false,"rank_v":168,"rank_pub":178,"name":"Jobe Qwen3.5-4B","eq":{"S":33.68,"O":25.36},"rank_tie":38,"tied":true,"parts":{"public":0,"same_skills":20.21,"new_domains":10.14}}},{"engine":"gevva-e4b","name":"Gevva E4B","long_name":"Gevva E4B","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":8000000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/davidburhans/gevva-e4b","code_url":null,"base_model":"google/gemma-4-E4B-it","base_url":"https://huggingface.co/google/gemma-4-E4B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5946,"conf":0.4512,"ece":0.1451,"brier":0.5678,"over95":0,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.1528,"conf":0.0418,"acc":0.4165},{"lo":0.1,"w":0.0279,"conf":0.1414,"acc":0.1115},{"lo":0.2,"w":0.0666,"conf":0.2694,"acc":0.2698},{"lo":0.3,"w":0.1286,"conf":0.354,"acc":0.4358},{"lo":0.4,"w":0.0981,"conf":0.4435,"acc":0.6232},{"lo":0.5,"w":0.2406,"conf":0.5426,"acc":0.6664},{"lo":0.6,"w":0.1337,"conf":0.6461,"acc":0.7538},{"lo":0.7,"w":0.1071,"conf":0.7497,"acc":0.8502},{"lo":0.8,"w":0.0447,"conf":0.8222,"acc":0.9089},{"lo":0.9,"w":0,"conf":null,"acc":null}],"areas":{"knowledge":{"ece":0.1038,"acc":0.5137,"conf":0.4258},"language":{"ece":0.0787,"acc":0.6615,"conf":0.5828},"retrieval":{"ece":0.3747,"acc":0.6461,"conf":0.2714},"tools":{"ece":0.3211,"acc":0.7857,"conf":0.4646},"arts":{"ece":0.1027,"acc":0.4966,"conf":0.406}}},"gaps":null,"latency":{"median":53.1,"p95":null,"mean":524.9},"results":{"1":{"score":0.9268,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":161.7},"2":{"score":0.6082,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1355.3},"3":{"score":0.5512,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":17883.2},"4":{"score":0.6558,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":261.1},"5":{"score":0.7174,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":507.1},"6":{"score":0.7985,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":3152.7},"9":{"score":0.1705,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2205},"10":{"score":0.6516,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":180.4},"11":{"score":0.567,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":6091.8},"12":{"score":0.3101,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31.2},"20":{"score":0.7214,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.5},"21":{"score":0.5982,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":18.6},"22":{"score":0.131,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":577.1},"23":{"score":0.5966,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":49.4},"24":{"score":0.6361,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":38.7},"25":{"score":0.3214,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":40},"26":{"score":0.9407,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":37.8},"27":{"score":0.8695,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":37.7},"28":{"score":0.6038,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":37.7},"29":{"score":0.7374,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":40.4},"30":{"score":0.2733,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1264,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.1},"32":{"score":0.5864,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":84.8},"33":{"score":0.2776,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":395},"36":{"score":0.3859,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2008.3},"37":{"score":0.3787,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.5},"38":{"score":0.1205,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2461.7},"39":{"score":0.7789,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":80.4},"40":{"score":0.3692,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":39.8},"41":{"score":0.3993,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":40.3},"42":{"score":0.6849,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":55.4},"43":{"score":0.3965,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":41},"44":{"score":0.562,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.4},"45":{"score":0.1038,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":55.9},"50":{"score":0.3914,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":80.7},"56":{"score":0.6275,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":245.1},"57":{"score":0.3641,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":42},"58":{"score":0.5965,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":40.3},"59":{"score":0.3668,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":50.8},"61":{"score":0.5777,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":52.1},"62":{"score":0.718,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":65.1},"64":{"score":0.447,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":39.8}},"benchmarks":{"1":{"raw":0.9268,"skill":0.9012,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7354,"private_eq":0.8055},"2":{"raw":0.6082,"skill":0.5475,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6558,"private_eq":0.7045},"3":{"raw":0.5512,"skill":0.5426,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3922,"private_eq":0.4095},"4":{"raw":0.6558,"skill":0.6514,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7145,"private_eq":0.7203},"5":{"raw":0.7174,"skill":0.7157,"coverage":1,"random":0.006,"tracks":[],"combined":0.5276,"private_eq":0.5348},"6":{"raw":0.7985,"skill":0.5275,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1705,"skill":0.1705,"coverage":1,"random":0,"tracks":[],"combined":0.1058,"private_eq":0.115},"10":{"raw":0.6516,"skill":0.4203,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.567,"skill":0.3739,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3101,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.7227,"skill":0.4453,"coverage":1,"random":0.5,"tracks":[],"combined":0.6468,"private_eq":0.8249},"21":{"raw":0.5982,"skill":0.1963,"coverage":1,"random":0.5,"tracks":[],"combined":0.0453,"private_eq":0.5276},"22":{"raw":0.1271,"skill":0.1203,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1636,"private_eq":0.1786},"23":{"raw":0.5849,"skill":0.1697,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6361,"skill":0.5148,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[],"combined":0.0522,"private_eq":0.2965},"26":{"raw":0.9407,"skill":0.9209,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8695,"skill":0.826,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6038,"skill":0.2076,"coverage":1,"random":0.5,"tracks":[],"combined":0.2697,"private_eq":0.6385},"29":{"raw":0.7374,"skill":0.6499,"coverage":1,"random":0.25,"tracks":[],"combined":0.5574,"private_eq":0.6711},"30":{"raw":0.2733,"skill":0.1208,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3541,"random":0.25},{"track":"10-choice","raw":0.1926,"random":0.1}],"combined":0.2952,"private_eq":0.4243},"31":{"raw":0.1264,"skill":0.0485,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0258,"private_eq":0.1149},"32":{"raw":0.5864,"skill":0.3424,"coverage":1,"random":0.371,"tracks":[],"combined":0.0493,"private_eq":0.4082},"33":{"raw":0.2776,"skill":0.268,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2509,"private_eq":0.2681},"36":{"raw":0.3859,"skill":0.3052,"coverage":1,"random":0.116,"tracks":[],"combined":0.2907,"private_eq":0.3792},"37":{"raw":0.3787,"skill":0.2208,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.1205,"skill":0.0924,"coverage":1,"random":0.031,"tracks":[],"combined":0.1808,"private_eq":0.2143},"39":{"raw":0.7789,"skill":0.6748,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5656,"private_eq":0.7073},"40":{"raw":0.3692,"skill":0.1885,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4373,"headline":false},{"track":"A · English","score":0.3692,"headline":true},{"track":"C · Arabic pairs","score":0.655,"headline":false},{"track":"C · English pairs","score":0.845,"headline":false}],"combined":0.1863,"private_eq":0.3739},"41":{"raw":0.3993,"skill":0.099,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1942,"private_eq":0.4682},"42":{"raw":0.6849,"skill":0.387,"coverage":1,"random":0.486,"tracks":[],"combined":0.6059,"private_eq":0.7992},"43":{"raw":0.3965,"skill":0.0425,"coverage":1,"random":0.3697,"tracks":[],"combined":0.353,"private_eq":0.5962},"44":{"raw":0.562,"skill":0.124,"coverage":1,"random":0.5,"tracks":[],"combined":0.2567,"private_eq":0.6321},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3914,"skill":0.1167,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.6275,"skill":0.255,"coverage":1,"random":0.5,"tracks":[],"combined":0.1443,"private_eq":0.5765},"57":{"raw":0.3641,"skill":0.2848,"coverage":1,"random":0.1109,"tracks":[],"combined":0.0566,"private_eq":0.1699},"58":{"raw":0.5965,"skill":0.4151,"coverage":1,"random":0.3101,"tracks":[],"combined":0.098,"private_eq":0.3841},"59":{"raw":0.3668,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5777,"skill":0.1554,"coverage":1,"random":0.5,"tracks":[],"combined":0.1954,"private_eq":0.6018},"62":{"raw":0.718,"skill":0.624,"coverage":1,"random":0.25,"tracks":[],"combined":0.5013,"private_eq":0.6295},"64":{"raw":0.447,"skill":0.3087,"coverage":1,"random":0.2,"tracks":[],"combined":0.3416,"private_eq":0.4785}},"coverage":1,"pending":0,"scores":{"balanced_raw":49.49,"balanced_skill":29.21,"breadth_skill":30.43,"public_skill":32.44},"frozen_scores":{"balanced_raw":49.49,"balanced_skill":32.44,"breadth_skill":30.43},"categories":[{"id":"knowledge","raw":0.3841,"skill":0.1791,"combined":0.1669,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4834,"skill":0.2761,"combined":0.2831,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5598,"skill":0.3997,"combined":0.3285,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6056,"skill":0.5694,"combined":0.4844,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4785,"skill":0.2262,"combined":0.283,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"gevva-e4b","pub":32.44,"same_skills":31.03,"new_domains":20.15,"v":29.21,"lab":false,"jev":false,"rank_v":173,"rank_pub":182,"name":"Gevva E4B","eq":{"S":30.34,"O":27.53},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":18.2,"new_domains":11.01}}},{"engine":"pngwn-space-uncapped-full","name":"open-jev (pngwn)","long_name":"open-jev (pngwn)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":121956232,"model_url":"https://huggingface.co/pngwn/system-one-qwen3.5-4b-scorer-v2b","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"pngwn/system-one-qwen3.5-4b-scorer-v2b","served_checkpoint":null},"completed":110000,"counts":{"ok":110000},"lifted":true,"calibration":{"acc":0.586,"conf":0.5978,"ece":0.0589,"brier":0.5446,"over95":0.0145,"n":72589,"benchmarks":32,"rel":[{"lo":0,"w":0.1012,"conf":0.0513,"acc":0.2456},{"lo":0.1,"w":0.0261,"conf":0.1465,"acc":0.294},{"lo":0.2,"w":0.0283,"conf":0.2455,"acc":0.223},{"lo":0.3,"w":0.0438,"conf":0.3567,"acc":0.3134},{"lo":0.4,"w":0.0789,"conf":0.4535,"acc":0.4291},{"lo":0.5,"w":0.1897,"conf":0.5494,"acc":0.5326},{"lo":0.6,"w":0.1517,"conf":0.6477,"acc":0.6145},{"lo":0.7,"w":0.1188,"conf":0.7489,"acc":0.7098},{"lo":0.8,"w":0.1173,"conf":0.8501,"acc":0.8339},{"lo":0.9,"w":0.1442,"conf":0.9663,"acc":0.8544}],"areas":{"knowledge":{"ece":0.1066,"acc":0.5297,"conf":0.6363},"language":{"ece":0.0192,"acc":0.698,"conf":0.707},"retrieval":{"ece":0.2241,"acc":0.5681,"conf":0.3573},"tools":{"ece":0.1496,"acc":0.7304,"conf":0.6205},"arts":{"ece":0.0824,"acc":0.436,"conf":0.4955}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":112.3,"p95":724.1,"mean":312.7},"results":{"1":{"score":0.8973,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":113.1},"2":{"score":0.6052,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":10281.6},"3":{"score":0.4843,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.5048,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":200.7},"5":{"score":0.5762,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":387},"6":{"score":0.7988,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":153.2},"9":{"score":0.0341,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5904,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":113.7},"11":{"score":0.6368,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5879,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":111.3},"20":{"score":0.519,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111.1},"21":{"score":0.5521,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":110.9},"22":{"score":0.056,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":637.4},"23":{"score":0.5828,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":112.4},"24":{"score":0.4449,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":110.7,"wrong_rows":5303,"contamination":"5,308 contaminated MMLU test items (in its training data) count as wrong"},"25":{"score":0.4388,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9621,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":110.2},"27":{"score":0.9121,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":110.3},"28":{"score":0.708,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":110},"29":{"score":0.7713,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":111.2},"30":{"score":0.4412,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1004,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":747.6},"32":{"score":0.5811,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3073,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":126.3},"36":{"score":0.4057,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3067,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":112.4},"38":{"score":0.1214,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":519.1},"39":{"score":0.7463,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3756,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":110.5,"tracks":[{"track":"A · Arabic","score":0.3459,"headline":false},{"track":"A · English","score":0.3756,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.675,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3828,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":111.1},"42":{"score":0.7434,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":112.6},"43":{"score":0.4825,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6316,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.2},"45":{"score":0.1098,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4206,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":114.4},"56":{"score":0.5025,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":113.8,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1272,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":111.9,"edition_added":"v0.2","wrong_rows":9542,"contamination":"9,542 contaminated MMLU-Pro test items (in its training data) count as wrong","jev":0.8213},"58":{"score":0.5929,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":111.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.0038,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":112.5,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5533,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":112.4,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.4663,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":113.3,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.4564,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":46.59,"balanced_skill":31.1,"breadth_skill":29.4,"public_skill":30.49},"frozen_scores":{"balanced_raw":46.59,"balanced_skill":30.49,"breadth_skill":29.4},"categories":[{"id":"knowledge","raw":0.4097,"skill":0.2279,"combined":0.2387,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4937,"skill":0.3468,"combined":0.3689,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4767,"skill":0.2911,"combined":0.2398,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5118,"skill":0.4573,"combined":0.3667,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4335,"skill":0.1445,"combined":0.2033,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.8973,"skill":0.8613,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7763,"private_eq":0.8354},"2":{"raw":0.6052,"skill":0.5441,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6432,"private_eq":0.6937},"3":{"raw":0.4843,"skill":0.4744,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2767,"private_eq":0.2975},"4":{"raw":0.5048,"skill":0.4984,"coverage":1,"random":0.0127,"tracks":[],"combined":0.4999,"private_eq":0.5109},"5":{"raw":0.5762,"skill":0.5736,"coverage":1,"random":0.006,"tracks":[],"combined":0.4724,"private_eq":0.4805},"6":{"raw":0.7989,"skill":0.5257,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7854,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0341,"skill":0.0341,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.6368,"skill":0.4747,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5879,"skill":0.3827,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2246,"private_eq":0.4876},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5395,"skill":0.079,"coverage":1,"random":0.5,"tracks":[],"combined":0.3793,"private_eq":0.6927},"21":{"raw":0.5521,"skill":0.1043,"coverage":1,"random":0.5,"tracks":[],"combined":0.0755,"private_eq":0.5425},"22":{"raw":0.0535,"skill":0.0461,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0971,"private_eq":0.1134},"23":{"raw":0.579,"skill":0.158,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8357,"skill":0.7809,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4388,"skill":0.2517,"coverage":1,"random":0.25,"tracks":[],"combined":0.177,"private_eq":0.389},"28":{"raw":0.708,"skill":0.416,"coverage":1,"random":0.5,"tracks":[],"combined":0.2697,"private_eq":0.6385},"29":{"raw":0.7713,"skill":0.6951,"coverage":1,"random":0.25,"tracks":[],"combined":0.8346,"private_eq":0.8766},"30":{"raw":0.4412,"skill":0.3255,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5171,"random":0.25},{"track":"10-choice","raw":0.3654,"random":0.1}],"combined":0.5476,"private_eq":0.6302},"31":{"raw":0.1004,"skill":0.0202,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0171,"private_eq":0.107},"32":{"raw":0.5811,"skill":0.334,"coverage":1,"random":0.371,"tracks":[],"combined":0.0726,"private_eq":0.4227},"33":{"raw":0.3073,"skill":0.2981,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2612,"private_eq":0.2782},"36":{"raw":0.4057,"skill":0.3276,"coverage":1,"random":0.116,"tracks":[],"combined":0.3011,"private_eq":0.3883},"37":{"raw":0.3067,"skill":0.1304,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.1214,"skill":0.0933,"coverage":1,"random":0.031,"tracks":[],"combined":0.1874,"private_eq":0.2206},"39":{"raw":0.7463,"skill":0.6269,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5772,"private_eq":0.7151},"40":{"raw":0.3756,"skill":0.1967,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3459,"headline":false},{"track":"A · English","score":0.3756,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.675,"headline":false}],"combined":0.141,"private_eq":0.3391},"41":{"raw":0.3828,"skill":0.0742,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2731,"private_eq":0.5202},"42":{"raw":0.7434,"skill":0.5008,"coverage":1,"random":0.486,"tracks":[],"combined":0.6592,"private_eq":0.8263},"43":{"raw":0.4825,"skill":0.1789,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3238,"private_eq":0.578},"44":{"raw":0.6316,"skill":0.2632,"coverage":1,"random":0.5,"tracks":[],"combined":0.4014,"private_eq":0.7036},"45":{"raw":0.1098,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4206,"skill":0.1592,"coverage":1,"random":0.311,"tracks":[],"combined":0.1234,"private_eq":0.4022},"56":{"raw":0.5025,"skill":0.005,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0059,"private_eq":0.5081},"57":{"raw":0.1272,"skill":0.0183,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2355,"private_eq":0.3271},"58":{"raw":0.5929,"skill":0.4099,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1135,"private_eq":0.3947},"59":{"raw":0.0038,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5533,"skill":0.1066,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0591,"private_eq":0.5344},"62":{"raw":0.4663,"skill":0.2884,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.0838,"private_eq":0.3199},"64":{"raw":0.4564,"skill":0.3205,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.3416,"private_eq":0.4785}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"Space cached scorer (shared prefix, chunked branch forwards), v2b merged LoRA bf16, sdpa, uncapped guards as board; fleet venv torch 2.14.0+cu130 (causal-conv1d absent: torch conv fallback)","contamination":[{"catalog_id":57,"benchmark":"MMLU-Pro","wrong":9542,"flag":"9,542 contaminated MMLU-Pro test items (in its training data) count as wrong"},{"catalog_id":24,"benchmark":"MMLU","wrong":5308,"flag":"5,308 contaminated MMLU test items (in its training data) count as wrong"}],"v03":{"engine":"pngwn-space-uncapped-full","pub":30.49,"same_skills":30.03,"new_domains":25.19,"v":31.1,"lab":false,"jev":false,"rank_v":166,"rank_pub":188,"name":"open-jev (pngwn)","eq":{"S":29.33,"O":33.75},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":17.6,"new_domains":13.5}}},{"engine":"tev1-4b","name":"Tev1-4B-experimental","long_name":"Together Tev1 Qwen3.5-4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9319828096,"model_url":"https://huggingface.co/togethercomputer/Tev1-4B-experimental","code_url":"https://github.com/togethercomputer/tev1","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"togethercomputer/Tev1-4B-experimental","served_checkpoint":null,"org":{"id":"togethercomputer","name":"Together","logo":"data/orgs/togethercomputer.png"}},"completed":95027,"counts":{"ok":95027,"unsupported":14732},"lifted":false,"calibration":{"acc":0.6345,"conf":0.7386,"ece":0.104,"brier":0.4763,"over95":0.0174,"n":32652,"benchmarks":28,"rel":[{"lo":0,"w":0.0002,"conf":0.0919,"acc":0},{"lo":0.1,"w":0.0124,"conf":0.1624,"acc":0.11},{"lo":0.2,"w":0.0224,"conf":0.2506,"acc":0.1198},{"lo":0.3,"w":0.0507,"conf":0.352,"acc":0.1994},{"lo":0.4,"w":0.0557,"conf":0.4561,"acc":0.3715},{"lo":0.5,"w":0.1476,"conf":0.5511,"acc":0.4759},{"lo":0.6,"w":0.1197,"conf":0.6492,"acc":0.5373},{"lo":0.7,"w":0.1334,"conf":0.7497,"acc":0.606},{"lo":0.8,"w":0.1248,"conf":0.8522,"acc":0.7026},{"lo":0.9,"w":0.3331,"conf":0.9674,"acc":0.8904}],"areas":{"knowledge":{"ece":0.1215,"acc":0.5575,"conf":0.6791},"language":{"ece":0.0705,"acc":0.7148,"conf":0.785},"retrieval":{"ece":0.2745,"acc":0.4965,"conf":0.7693},"tools":{"ece":0.0258,"acc":0.9313,"conf":0.9228},"arts":{"ece":0.1059,"acc":0.5864,"conf":0.6903}}},"gaps":{"share":0.1229,"reasons":[{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 151","share":0.0459},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 77","share":0.0257},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'chord' has 129","share":0.0167}]},"latency":{"median":35.8,"p95":166,"mean":67.6},"results":{"1":{"score":0.902,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":83.8},"2":{"score":0.6501,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":452},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.6781,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":163.8},"9":{"score":0.1477,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.407,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":77},"11":{"score":0.6772,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5639,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":34.4},"20":{"score":0.7662,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.3},"21":{"score":0.583,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":33.6},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.5844,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.6},"24":{"score":0.7227,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":34.3},"25":{"score":0.4235,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.976,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.8},"27":{"score":0.9215,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":33.8},"28":{"score":0.6961,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":34.1},"29":{"score":0.8193,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":35},"30":{"score":0.2949,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.22,"jev":null,"metric":"accuracy","requests":5000,"answered":1359,"unsupported":3641,"errors":0,"median_ms":73.7},"32":{"score":0.5798,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.203,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":120.4},"36":{"score":0.4302,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4421,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36},"38":{"score":0.1032,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":433.6},"39":{"score":0.8631,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3594,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":34.4,"tracks":[{"track":"A · Arabic","score":0.3776,"headline":false},{"track":"A · English","score":0.3594,"headline":true},{"track":"C · Arabic pairs","score":0.57,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4766,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":34.7},"42":{"score":0.7734,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":74.7},"43":{"score":0.5018,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.631,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.2},"45":{"score":0.1165,"jev":null,"metric":"accuracy","requests":501,"answered":498,"unsupported":3,"errors":0,"median_ms":null},"50":{"score":0.4159,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":76.4},"56":{"score":0.5065,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":107,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4537,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":35.2,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6348,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":35,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5033,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":75.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.641,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":40.1,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.6054,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":76,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5549,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":45.39,"balanced_skill":33.73,"breadth_skill":28.21,"public_skill":29.51},"frozen_scores":{"balanced_raw":45.39,"balanced_skill":29.51,"breadth_skill":28.21},"categories":[{"id":"knowledge","raw":0.4216,"skill":0.2435,"combined":0.2275,"coverage":0.9387,"pending":0,"n":11},{"id":"language","raw":0.5762,"skill":0.3918,"combined":0.4143,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3191,"skill":0.1548,"combined":0.1847,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4603,"skill":0.4183,"combined":0.442,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.4795,"skill":0.2348,"combined":0.2733,"coverage":0.8333,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.902,"skill":0.8677,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7763,"private_eq":0.8354},"2":{"raw":0.6501,"skill":0.5959,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6694,"private_eq":0.7161},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.6781,"skill":0.2394,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6627,"headline":false},{"track":"RouterBench-5shot","score":0.6935,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.1477,"skill":0.1477,"coverage":1,"random":0,"tracks":[],"combined":0.1943,"private_eq":0.2025},"11":{"raw":0.6772,"skill":0.5332,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5639,"skill":0.3468,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3983,"private_eq":0.6022},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.7699,"skill":0.5399,"coverage":1,"random":0.5,"tracks":[],"combined":0.6757,"private_eq":0.8392},"21":{"raw":0.583,"skill":0.1659,"coverage":1,"random":0.5,"tracks":[],"combined":0.1058,"private_eq":0.5575},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5535,"skill":0.107,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7375,"skill":0.65,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4235,"skill":0.2313,"coverage":1,"random":0.25,"tracks":[],"combined":0.1026,"private_eq":0.3339},"28":{"raw":0.6961,"skill":0.3922,"coverage":1,"random":0.5,"tracks":[],"combined":0.315,"private_eq":0.6609},"29":{"raw":0.8193,"skill":0.7591,"coverage":1,"random":0.25,"tracks":[],"combined":0.6811,"private_eq":0.7628},"30":{"raw":0.2949,"skill":0.1499,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4003,"random":0.25},{"track":"10-choice","raw":0.1895,"random":0.1}],"combined":0.3626,"private_eq":0.4793},"31":{"raw":0.0598,"skill":0,"coverage":0.2718,"random":0.0819,"tracks":[],"combined":0.0019,"private_eq":0.0932},"32":{"raw":0.5798,"skill":0.332,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.203,"skill":0.1924,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3177,"private_eq":0.3333},"36":{"raw":0.4302,"skill":0.3554,"coverage":1,"random":0.116,"tracks":[],"combined":0.3182,"private_eq":0.4033},"37":{"raw":0.4421,"skill":0.3002,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4732,"private_eq":0.584},"38":{"raw":0.1032,"skill":0.0745,"coverage":1,"random":0.031,"tracks":[],"combined":0.1249,"private_eq":0.1607},"39":{"raw":0.8631,"skill":0.7986,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5315,"private_eq":0.6844},"40":{"raw":0.3594,"skill":0.1759,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3776,"headline":false},{"track":"A · English","score":0.3594,"headline":true},{"track":"C · Arabic pairs","score":0.57,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"combined":0.1718,"private_eq":0.3628},"41":{"raw":0.4766,"skill":0.2149,"coverage":1,"random":0.3333,"tracks":[],"combined":0.377,"private_eq":0.5887},"42":{"raw":0.7734,"skill":0.5591,"coverage":1,"random":0.486,"tracks":[],"combined":0.7194,"private_eq":0.8569},"43":{"raw":0.5018,"skill":0.2095,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3782,"private_eq":0.6119},"44":{"raw":0.631,"skill":0.262,"coverage":1,"random":0.5,"tracks":[],"combined":0.3585,"private_eq":0.6824},"45":{"raw":0.1158,"skill":0,"coverage":0.994,"random":0.1641,"tracks":[]},"50":{"raw":0.4159,"skill":0.1522,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.5065,"skill":0.013,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5},"57":{"raw":0.4537,"skill":0.3856,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3223,"private_eq":0.4034},"58":{"raw":0.6348,"skill":0.4706,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1345,"private_eq":0.409},"59":{"raw":0.5033,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.26,"private_eq":0.6467},"61":{"raw":0.641,"skill":0.282,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3996,"private_eq":0.7027},"62":{"raw":0.6054,"skill":0.4739,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.604,"private_eq":0.7056},"64":{"raw":0.5549,"skill":0.4436,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.3998,"private_eq":0.5245}},"coverage":0.6909,"pending":0.24,"latency_rows":677,"latency_path":"in-process vLLM 0.29.0 LLM.generate with all questions of the row in one call, CUDA graphs, gpu util 0.9 (single process)","v03":{"engine":"tev1-4b","pub":29.51,"same_skills":31.78,"new_domains":28.37,"v":33.73,"lab":false,"jev":false,"rank_v":157,"rank_pub":192,"name":"Together Tev1 Qwen3.5-4B","eq":{"S":31.09,"O":37.68},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":18.66,"new_domains":15.07}}},{"engine":"openvons","name":"openvons","long_name":"openvons Qwen3-4B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4022468096,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/genai-craft/openvons","base_model":"Qwen/Qwen3-4B-Instruct-2507","base_url":"https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507","weights_repo":null,"served_checkpoint":"Qwen/Qwen3-4B-Instruct-2507"},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.583,"conf":0.9542,"ece":0.3712,"brier":0.7749,"over95":0.3051,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0,"conf":null,"acc":null},{"lo":0.2,"w":0.0007,"conf":0.2424,"acc":0},{"lo":0.3,"w":0.0026,"conf":0.3383,"acc":0.0431},{"lo":0.4,"w":0.0116,"conf":0.4512,"acc":0.1647},{"lo":0.5,"w":0.0223,"conf":0.5416,"acc":0.3145},{"lo":0.6,"w":0.0276,"conf":0.6423,"acc":0.2856},{"lo":0.7,"w":0.0254,"conf":0.7429,"acc":0.3081},{"lo":0.8,"w":0.0386,"conf":0.8415,"acc":0.2981},{"lo":0.9,"w":0.8712,"conf":0.9949,"acc":0.6276}],"areas":{"knowledge":{"ece":0.3846,"acc":0.5485,"conf":0.933},"language":{"ece":0.3203,"acc":0.6607,"conf":0.981},"retrieval":{"ece":0.4285,"acc":0.54,"conf":0.9685},"tools":{"ece":0.28,"acc":0.7086,"conf":0.9886},"arts":{"ece":0.4422,"acc":0.478,"conf":0.9202}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":21.2,"p95":113.3,"mean":44.1},"results":{"1":{"score":0.8406,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":42.2},"2":{"score":0.6032,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":252.4},"3":{"score":0.435,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.3255,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":78.4},"5":{"score":0.4172,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":107.3},"6":{"score":0.798,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":85.5},"9":{"score":0.0455,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6232,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":46.3},"11":{"score":0.5355,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5379,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":17.6},"20":{"score":0.7622,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.7},"21":{"score":0.5906,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":14.1},"22":{"score":0.056,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":294.3},"23":{"score":0.5326,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.3},"24":{"score":0.6918,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.4},"25":{"score":0.3878,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.963,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":14.7},"27":{"score":0.8951,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":14.6},"28":{"score":0.6259,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":14.4},"29":{"score":0.8076,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":18.2},"30":{"score":0.2896,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.109,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":71.6},"32":{"score":0.6011,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2612,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":58.9},"36":{"score":0.4048,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.33,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.4},"38":{"score":0.1424,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":276.4},"39":{"score":0.6046,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3333,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":15,"tracks":[{"track":"A · Arabic","score":0.3208,"headline":false},{"track":"A · English","score":0.3333,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.87,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4249,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":17.9},"42":{"score":0.7,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":28.2},"43":{"score":0.4263,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5578,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.2},"45":{"score":0.1138,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4588,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":37},"56":{"score":0.5135,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":54.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4318,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":18.1,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.5328,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":17.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5512,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":32.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6095,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":24.3,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5298,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":32.1,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5871,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":46.42,"balanced_skill":28.78,"breadth_skill":27.81,"public_skill":28.66},"frozen_scores":{"balanced_raw":46.42,"balanced_skill":28.66,"breadth_skill":27.81},"categories":[{"id":"knowledge","raw":0.3963,"skill":0.1949,"combined":0.1986,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.522,"skill":0.3073,"combined":0.3173,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4288,"skill":0.2537,"combined":0.3002,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5017,"skill":0.4465,"combined":0.4685,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.493,"skill":0.244,"combined":0.3267,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.8406,"skill":0.7848,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7081,"private_eq":0.7855},"2":{"raw":0.6032,"skill":0.5418,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6468,"private_eq":0.6968},"3":{"raw":0.435,"skill":0.4241,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4552,"private_eq":0.4706},"4":{"raw":0.3255,"skill":0.3168,"coverage":1,"random":0.0127,"tracks":[],"combined":0.4801,"private_eq":0.4916},"5":{"raw":0.4172,"skill":0.4137,"coverage":1,"random":0.006,"tracks":[],"combined":0.2921,"private_eq":0.3034},"6":{"raw":0.798,"skill":0.5234,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7851,"headline":false},{"track":"RouterBench-5shot","score":0.8108,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0455,"skill":0.0455,"coverage":1,"random":0,"tracks":[],"combined":0.0198,"private_eq":0.03},"11":{"raw":0.5355,"skill":0.3282,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5379,"skill":0.3078,"coverage":1,"random":0.3324,"tracks":[],"combined":0.339,"private_eq":0.5631},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.7561,"skill":0.5122,"coverage":1,"random":0.5,"tracks":[],"combined":0.683,"private_eq":0.8428},"21":{"raw":0.5906,"skill":0.1811,"coverage":1,"random":0.5,"tracks":[],"combined":0.136,"private_eq":0.5724},"22":{"raw":0.051,"skill":0.0435,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0786,"private_eq":0.0952},"23":{"raw":0.5143,"skill":0.0287,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7191,"skill":0.6255,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[],"combined":0.0691,"private_eq":0.309},"28":{"raw":0.6259,"skill":0.2518,"coverage":1,"random":0.5,"tracks":[],"combined":0.209,"private_eq":0.6085},"29":{"raw":0.8076,"skill":0.7435,"coverage":1,"random":0.25,"tracks":[],"combined":0.5673,"private_eq":0.6784},"30":{"raw":0.2896,"skill":0.1437,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3973,"random":0.25},{"track":"10-choice","raw":0.182,"random":0.1}],"combined":0.3171,"private_eq":0.4422},"31":{"raw":0.109,"skill":0.0296,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0171,"private_eq":0.107},"32":{"raw":0.6011,"skill":0.3658,"coverage":1,"random":0.371,"tracks":[],"combined":0.2273,"private_eq":0.5189},"33":{"raw":0.2612,"skill":0.2514,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3331,"private_eq":0.3483},"36":{"raw":0.4048,"skill":0.3267,"coverage":1,"random":0.116,"tracks":[],"combined":0.2872,"private_eq":0.3762},"37":{"raw":0.33,"skill":0.1597,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1748,"private_eq":0.3488},"38":{"raw":0.1424,"skill":0.115,"coverage":1,"random":0.031,"tracks":[],"combined":0.2174,"private_eq":0.2493},"39":{"raw":0.6046,"skill":0.4184,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3485,"private_eq":0.5614},"40":{"raw":0.3333,"skill":0.1423,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3208,"headline":false},{"track":"A · English","score":0.3333,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.87,"headline":false}],"combined":0.0819,"private_eq":0.2937},"41":{"raw":0.4249,"skill":0.1374,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1304,"private_eq":0.4262},"42":{"raw":0.7,"skill":0.4163,"coverage":1,"random":0.486,"tracks":[],"combined":0.5944,"private_eq":0.7934},"43":{"raw":0.4263,"skill":0.0898,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2735,"private_eq":0.5467},"44":{"raw":0.5578,"skill":0.1156,"coverage":1,"random":0.5,"tracks":[],"combined":0.1604,"private_eq":0.5845},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4588,"skill":0.2146,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.5135,"skill":0.027,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3,"private_eq":0.6535},"57":{"raw":0.4318,"skill":0.3609,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.3067,"private_eq":0.3897},"58":{"raw":0.5328,"skill":0.3228,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0952,"private_eq":0.3822},"59":{"raw":0.5512,"skill":0.0695,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.2053,"private_eq":0.6206},"61":{"raw":0.6095,"skill":0.219,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2351,"private_eq":0.6214},"62":{"raw":0.5298,"skill":0.3731,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4672,"private_eq":0.6042},"64":{"raw":0.5871,"skill":0.4839,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5259,"private_eq":0.6242}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author client over vLLM 0.29.0 OpenAI server, Qwen3-4B-Instruct-2507 bf16, prefix caching, CUDA graphs, gpu util 0.8","v03":{"engine":"openvons","pub":28.66,"same_skills":32.13,"new_domains":17.92,"v":28.78,"lab":false,"jev":false,"rank_v":175,"rank_pub":195,"name":"openvons Qwen3-4B","eq":{"S":31.45,"O":24.78},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":18.87,"new_domains":9.91}}},{"engine":"gevva-e2b","name":"Gevva E2B","long_name":"Gevva E2B","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":5100000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/davidburhans/gevva-e2b","code_url":null,"base_model":"google/gemma-4-E2B-it","base_url":"https://huggingface.co/google/gemma-4-E2B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5382,"conf":0.4528,"ece":0.0902,"brier":0.6003,"over95":0,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.1533,"conf":0.0375,"acc":0.377},{"lo":0.1,"w":0.0249,"conf":0.1458,"acc":0.0926},{"lo":0.2,"w":0.0691,"conf":0.2672,"acc":0.2782},{"lo":0.3,"w":0.1395,"conf":0.3516,"acc":0.3645},{"lo":0.4,"w":0.1015,"conf":0.4477,"acc":0.5433},{"lo":0.5,"w":0.2204,"conf":0.5442,"acc":0.6213},{"lo":0.6,"w":0.1282,"conf":0.6463,"acc":0.6789},{"lo":0.7,"w":0.0946,"conf":0.7483,"acc":0.7366},{"lo":0.8,"w":0.0685,"conf":0.83,"acc":0.8641},{"lo":0.9,"w":0,"conf":null,"acc":null}],"areas":{"knowledge":{"ece":0.0735,"acc":0.4434,"conf":0.417},"language":{"ece":0.0435,"acc":0.6166,"conf":0.5891},"retrieval":{"ece":0.3663,"acc":0.6165,"conf":0.2503},"tools":{"ece":0.187,"acc":0.6711,"conf":0.5046},"arts":{"ece":0.0728,"acc":0.463,"conf":0.4171}}},"gaps":null,"latency":{"median":42.8,"p95":null,"mean":338.9},"results":{"1":{"score":0.9262,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":88.1},"2":{"score":0.5849,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":794.2},"3":{"score":0.4823,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":11935.7},"4":{"score":0.6335,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":143},"5":{"score":0.6837,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":274.5},"6":{"score":0.7973,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1770},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1205.2},"10":{"score":0.6416,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":99.9},"11":{"score":0.5509,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3483.8},"12":{"score":0.4474,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":18.2},"20":{"score":0.6222,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.3},"21":{"score":0.5833,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":14.9},"22":{"score":0.142,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":9311.6},"23":{"score":0.6018,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.7},"24":{"score":0.5493,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":16.8},"25":{"score":0.2653,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":22.3},"26":{"score":0.8649,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":15.9},"27":{"score":0.7372,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":16.2},"28":{"score":0.5604,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":14.4},"29":{"score":0.568,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":21.7},"30":{"score":0.2183,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1176,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":183.5},"32":{"score":0.5093,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":76.9},"33":{"score":0.2188,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":224.4},"36":{"score":0.3476,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1175.7},"37":{"score":0.3227,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":37.3},"38":{"score":0.1156,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":992.3},"39":{"score":0.7467,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":34.8},"40":{"score":0.309,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":16.2},"41":{"score":0.3737,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":20.1},"42":{"score":0.6894,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":31.3},"43":{"score":0.3596,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":18.7},"44":{"score":0.5636,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.7},"45":{"score":0.1397,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":30.9},"50":{"score":0.3371,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":73.8},"56":{"score":0.6755,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":152.1},"57":{"score":0.2862,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":28.7},"58":{"score":0.5286,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":20.2},"59":{"score":0.1556,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":30.6},"61":{"score":0.5797,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":30.1},"62":{"score":0.7516,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":75.3},"64":{"score":0.3598,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":25.6}},"benchmarks":{"1":{"raw":0.9262,"skill":0.9004,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7831,"private_eq":0.8404},"2":{"raw":0.5849,"skill":0.5206,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6114,"private_eq":0.6665},"3":{"raw":0.4823,"skill":0.4723,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3713,"private_eq":0.3892},"4":{"raw":0.6335,"skill":0.6288,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6752,"private_eq":0.682},"5":{"raw":0.6837,"skill":0.6818,"coverage":1,"random":0.006,"tracks":[],"combined":0.5217,"private_eq":0.529},"6":{"raw":0.7973,"skill":0.5247,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0.0025},"10":{"raw":0.6416,"skill":0.4037,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5509,"skill":0.3506,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4474,"skill":0.1723,"coverage":1,"random":0.3324,"tracks":[],"combined":0.088,"private_eq":0.3974},"20":{"raw":0.6273,"skill":0.2545,"coverage":1,"random":0.5,"tracks":[],"combined":0.0178,"private_eq":0.514},"21":{"raw":0.5833,"skill":0.1667,"coverage":1,"random":0.5,"tracks":[],"combined":0.0856,"private_eq":0.5475},"22":{"raw":0.1354,"skill":0.1287,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1383,"private_eq":0.1538},"23":{"raw":0.6019,"skill":0.2039,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5493,"skill":0.3991,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2653,"skill":0.0204,"coverage":1,"random":0.25,"tracks":[],"combined":0.0453,"private_eq":0.2914},"26":{"raw":0.8649,"skill":0.8198,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7372,"skill":0.6495,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5604,"skill":0.1208,"coverage":1,"random":0.5,"tracks":[],"combined":0.2191,"private_eq":0.6135},"29":{"raw":0.568,"skill":0.424,"coverage":1,"random":0.25,"tracks":[],"combined":0.3396,"private_eq":0.5096},"30":{"raw":0.2183,"skill":0.0527,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2911,"random":0.25},{"track":"10-choice","raw":0.1456,"random":0.1}],"combined":0.2351,"private_eq":0.3753},"31":{"raw":0.1176,"skill":0.0389,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0463,"private_eq":0.1335},"32":{"raw":0.5093,"skill":0.2199,"coverage":1,"random":0.371,"tracks":[],"combined":0.0418,"private_eq":0.4035},"33":{"raw":0.2188,"skill":0.2084,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3177,"private_eq":0.3333},"36":{"raw":0.3476,"skill":0.262,"coverage":1,"random":0.116,"tracks":[],"combined":0.2423,"private_eq":0.3369},"37":{"raw":0.3227,"skill":0.1504,"coverage":1,"random":0.2027,"tracks":[],"combined":0.305,"private_eq":0.4514},"38":{"raw":0.1156,"skill":0.0873,"coverage":1,"random":0.031,"tracks":[],"combined":0.1498,"private_eq":0.1846},"39":{"raw":0.7467,"skill":0.6274,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4965,"private_eq":0.6609},"40":{"raw":0.309,"skill":0.111,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3303,"headline":false},{"track":"A · English","score":0.309,"headline":true},{"track":"C · Arabic pairs","score":0.62,"headline":false},{"track":"C · English pairs","score":0.865,"headline":false}],"combined":0.0832,"private_eq":0.2947},"41":{"raw":0.3737,"skill":0.0606,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2705,"private_eq":0.5185},"42":{"raw":0.6894,"skill":0.3957,"coverage":1,"random":0.486,"tracks":[],"combined":0.2811,"private_eq":0.6342},"43":{"raw":0.3596,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2694,"private_eq":0.5441},"44":{"raw":0.5636,"skill":0.1272,"coverage":1,"random":0.5,"tracks":[],"combined":0.1497,"private_eq":0.5792},"45":{"raw":0.1397,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3371,"skill":0.0379,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.6755,"skill":0.351,"coverage":1,"random":0.5,"tracks":[],"combined":0.2915,"private_eq":0.6493},"57":{"raw":0.2862,"skill":0.1972,"coverage":1,"random":0.1109,"tracks":[],"combined":0.0446,"private_eq":0.1593},"58":{"raw":0.5286,"skill":0.3167,"coverage":1,"random":0.3101,"tracks":[],"combined":0.0593,"private_eq":0.3577},"59":{"raw":0.1556,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5797,"skill":0.1594,"coverage":1,"random":0.5,"tracks":[],"combined":0.2181,"private_eq":0.613},"62":{"raw":0.7516,"skill":0.6688,"coverage":1,"random":0.25,"tracks":[],"combined":0.4397,"private_eq":0.5838},"64":{"raw":0.3598,"skill":0.1997,"coverage":1,"random":0.2,"tracks":[],"combined":0.2057,"private_eq":0.371}},"coverage":1,"pending":0,"scores":{"balanced_raw":45.82,"balanced_skill":24.26,"breadth_skill":25.9,"public_skill":28.53},"frozen_scores":{"balanced_raw":45.82,"balanced_skill":28.53,"breadth_skill":25.9},"categories":[{"id":"knowledge","raw":0.3436,"skill":0.1194,"combined":0.139,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4425,"skill":0.2498,"combined":0.2124,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5418,"skill":0.386,"combined":0.3851,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5605,"skill":0.5253,"combined":0.4497,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4408,"skill":0.1652,"combined":0.1301,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"gevva-e2b","pub":28.53,"same_skills":27.78,"new_domains":14.11,"v":24.26,"lab":false,"jev":false,"rank_v":183,"rank_pub":197,"name":"Gevva E2B","eq":{"S":27.05,"O":20.07},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":16.23,"new_domains":8.03}}},{"engine":"metask-jev-4b","name":"Metask-Jev-4B","long_name":"Metask-Jev-4B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":9078620536,"model_url":"https://huggingface.co/wayfind/metask-jev-4b-policy-mix","code_url":"https://github.com/metask-ai/metask-jev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":"wayfind/metask-jev-4b-policy-mix","served_checkpoint":null,"org":{"id":"wayfind","name":"Metask-AI-Lab","logo":"data/orgs/wayfind.png"}},"completed":95701,"counts":{"ok":95701,"unsupported":14500},"lifted":false,"calibration":{"acc":0.6241,"conf":0.7244,"ece":0.1035,"brier":0.5026,"over95":0.004,"n":32698,"benchmarks":28,"rel":[{"lo":0,"w":0.0101,"conf":0.0831,"acc":0.093},{"lo":0.1,"w":0.0252,"conf":0.1511,"acc":0.1364},{"lo":0.2,"w":0.0293,"conf":0.2593,"acc":0.309},{"lo":0.3,"w":0.048,"conf":0.3518,"acc":0.2295},{"lo":0.4,"w":0.056,"conf":0.4503,"acc":0.3927},{"lo":0.5,"w":0.0939,"conf":0.5523,"acc":0.5137},{"lo":0.6,"w":0.098,"conf":0.6512,"acc":0.5359},{"lo":0.7,"w":0.1444,"conf":0.7533,"acc":0.5933},{"lo":0.8,"w":0.228,"conf":0.8581,"acc":0.6833},{"lo":0.9,"w":0.2669,"conf":0.9362,"acc":0.8817}],"areas":{"knowledge":{"ece":0.091,"acc":0.5421,"conf":0.6256},"language":{"ece":0.1375,"acc":0.6844,"conf":0.8195},"retrieval":{"ece":0.1787,"acc":0.5892,"conf":0.7643},"tools":{"ece":0.0495,"acc":0.9139,"conf":0.9008},"arts":{"ece":0.0894,"acc":0.5862,"conf":0.6742}}},"gaps":{"share":0.1205,"reasons":[{"reason":"metask serve.py HTTP 500: decision: enum choices must be 1–26 nonempty strings.","share":0.1205}]},"latency":{"median":57.3,"p95":369.4,"mean":146.6},"results":{"1":{"score":0.8666,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":116.4},"2":{"score":0.6568,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1778.8},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7962,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":232.7},"9":{"score":0.2045,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5765,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":66},"11":{"score":0.6933,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5625,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":55.3},"20":{"score":0.7848,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.9},"21":{"score":0.5209,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":53.9},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.5788,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":56.4},"24":{"score":0.6954,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":55},"25":{"score":0.4082,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9541,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":54.5},"27":{"score":0.8993,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":54.7},"28":{"score":0.6709,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":53.5},"29":{"score":0.7528,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":56.5},"30":{"score":0.2737,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2305,"jev":null,"metric":"accuracy","requests":5000,"answered":1588,"unsupported":3412,"errors":0,"median_ms":69},"32":{"score":0.5479,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1709,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":445.8},"36":{"score":0.4027,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3488,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.9},"38":{"score":0.0675,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3469.9},"39":{"score":0.793,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4762,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":53.9,"tracks":[{"track":"A · Arabic","score":0.1942,"headline":false},{"track":"A · English","score":0.4762,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.805,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4303,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":55.8},"42":{"score":0.7341,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":57.8},"43":{"score":0.4632,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6172,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.7},"45":{"score":0.0838,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4004,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":62.9},"56":{"score":0.516,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":493.7,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4522,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":59.6,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.6065,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":56,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.6638,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":60.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.624,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":56.8,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.4285,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":62.3,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.553,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":43.89,"balanced_skill":32.17,"breadth_skill":26.04,"public_skill":27.58},"frozen_scores":{"balanced_raw":43.89,"balanced_skill":27.58,"breadth_skill":26.04},"categories":[{"id":"knowledge","raw":0.401,"skill":0.2155,"combined":0.2138,"coverage":0.9431,"pending":0,"n":11},{"id":"language","raw":0.5784,"skill":0.4005,"combined":0.3696,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2988,"skill":0.1291,"combined":0.1409,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4314,"skill":0.3759,"combined":0.3837,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.471,"skill":0.2194,"combined":0.2216,"coverage":0.8333,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.8666,"skill":0.8199,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7285,"private_eq":0.8004},"2":{"raw":0.6568,"skill":0.6036,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6729,"private_eq":0.7191},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.7962,"skill":0.5194,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7825,"headline":false},{"track":"RouterBench-5shot","score":0.8099,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.2045,"skill":0.2045,"coverage":1,"random":0,"tracks":[],"combined":0.0906,"private_eq":0.1},"11":{"raw":0.6933,"skill":0.5565,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5625,"skill":0.3447,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2762,"private_eq":0.5216},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.783,"skill":0.566,"coverage":1,"random":0.5,"tracks":[],"combined":0.5746,"private_eq":0.7892},"21":{"raw":0.5209,"skill":0.0419,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5688,"skill":0.1376,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7027,"skill":0.6036,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4082,"skill":0.2109,"coverage":1,"random":0.25,"tracks":[],"combined":0.1026,"private_eq":0.3339},"28":{"raw":0.6709,"skill":0.3418,"coverage":1,"random":0.5,"tracks":[],"combined":0.2998,"private_eq":0.6534},"29":{"raw":0.7528,"skill":0.6704,"coverage":1,"random":0.25,"tracks":[],"combined":0.592,"private_eq":0.6967},"30":{"raw":0.2737,"skill":0.1232,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3715,"random":0.25},{"track":"10-choice","raw":0.1759,"random":0.1}],"combined":0.343,"private_eq":0.4633},"31":{"raw":0.0732,"skill":0,"coverage":0.3176,"random":0.0819,"tracks":[],"combined":0.0138,"private_eq":0.104},"32":{"raw":0.5479,"skill":0.2812,"coverage":1,"random":0.371,"tracks":[],"combined":0.0881,"private_eq":0.4323},"33":{"raw":0.1709,"skill":0.1599,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1893,"private_eq":0.208},"36":{"raw":0.4027,"skill":0.3242,"coverage":1,"random":0.116,"tracks":[],"combined":0.2769,"private_eq":0.3672},"37":{"raw":0.3488,"skill":0.1832,"coverage":1,"random":0.2027,"tracks":[],"combined":0.2966,"private_eq":0.4448},"38":{"raw":0.0675,"skill":0.0377,"coverage":1,"random":0.031,"tracks":[],"combined":0.0592,"private_eq":0.0978},"39":{"raw":0.793,"skill":0.6955,"coverage":1,"random":0.3201,"tracks":[],"combined":0.53,"private_eq":0.6834},"40":{"raw":0.4762,"skill":0.3261,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1942,"headline":false},{"track":"A · English","score":0.4762,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.805,"headline":false}],"combined":0.2422,"private_eq":0.4169},"41":{"raw":0.4303,"skill":0.1454,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3413,"private_eq":0.5652},"42":{"raw":0.7341,"skill":0.4827,"coverage":1,"random":0.486,"tracks":[],"combined":0.8042,"private_eq":0.9},"43":{"raw":0.4632,"skill":0.1482,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3071,"private_eq":0.5676},"44":{"raw":0.6172,"skill":0.2344,"coverage":1,"random":0.5,"tracks":[],"combined":0.3211,"private_eq":0.6639},"45":{"raw":0.0838,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4004,"skill":0.1297,"coverage":1,"random":0.311,"tracks":[],"combined":0.1162,"private_eq":0.3973},"56":{"raw":0.516,"skill":0.032,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0405,"private_eq":0.5252},"57":{"raw":0.4522,"skill":0.3839,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.35,"private_eq":0.4278},"58":{"raw":0.6065,"skill":0.4296,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.1203,"private_eq":0.3993},"59":{"raw":0.6638,"skill":0.3029,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.086,"private_eq":0.5637},"61":{"raw":0.624,"skill":0.248,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2861,"private_eq":0.6466},"62":{"raw":0.4285,"skill":0.238,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4466,"private_eq":0.5889},"64":{"raw":0.553,"skill":0.4413,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4385,"private_eq":0.5551}},"coverage":0.6927,"pending":0.24,"latency_rows":678,"latency_path":"author serve.py /v1/systemone handler in-process (Flask test client), fla kernels; torch 2.14.0+cu130 / transformers 5.17.0 / fla 0.5.2","v03":{"engine":"metask-jev-4b","pub":27.58,"same_skills":27.82,"new_domains":30.08,"v":32.17,"lab":false,"jev":false,"rank_v":161,"rank_pub":198,"name":"Metask-Jev-4B","eq":{"S":27.09,"O":39.79},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":16.25,"new_domains":15.92}}},{"engine":"semif","name":"SemIf","long_name":"SemIf Qwen3.5-4B Base","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/TheoLeeCJ/openjev","base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null},"completed":94884,"counts":{"ok":94884,"unsupported":15317},"lifted":true,"calibration":{"acc":0.6404,"conf":0.7393,"ece":0.0989,"brier":0.4825,"over95":0.0256,"n":70067,"benchmarks":28,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0004,"conf":0.1774,"acc":0},{"lo":0.2,"w":0.0171,"conf":0.263,"acc":0.1845},{"lo":0.3,"w":0.057,"conf":0.3561,"acc":0.2879},{"lo":0.4,"w":0.0706,"conf":0.4498,"acc":0.324},{"lo":0.5,"w":0.1669,"conf":0.5495,"acc":0.5089},{"lo":0.6,"w":0.1125,"conf":0.6493,"acc":0.5585},{"lo":0.7,"w":0.1206,"conf":0.751,"acc":0.616},{"lo":0.8,"w":0.1155,"conf":0.8532,"acc":0.6777},{"lo":0.9,"w":0.3393,"conf":0.9689,"acc":0.877}],"areas":{"knowledge":{"ece":0.1033,"acc":0.5598,"conf":0.6631},"language":{"ece":0.1291,"acc":0.6952,"conf":0.8215},"retrieval":{"ece":0.1814,"acc":0.6173,"conf":0.7888},"tools":{"ece":0.0383,"acc":0.919,"conf":0.8897},"arts":{"ece":0.0532,"acc":0.611,"conf":0.6642}}},"gaps":{"share":0.1273,"reasons":[{"reason":"supports only 2–16 answer options per question","share":0.1273}]},"latency":{"median":113.1,"p95":214.6,"mean":220.5},"results":{"1":{"score":0.9327,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":119.4},"2":{"score":0.5961,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":9594.1},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7954,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":202.7},"9":{"score":0.0909,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.568,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":119},"11":{"score":0.5951,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5491,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":111.5},"20":{"score":0.8522,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.9},"21":{"score":0.5993,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":110.7},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.566,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":113},"24":{"score":0.7144,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":111.4},"25":{"score":0.3673,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9676,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":111.1},"27":{"score":0.9181,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":111.1},"28":{"score":0.6598,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":110.6},"29":{"score":0.8272,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":112.4},"30":{"score":0.2449,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.3316,"jev":null,"metric":"accuracy","requests":5000,"answered":787,"unsupported":4213,"errors":0,"median_ms":null},"32":{"score":0.5465,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":140.7},"36":{"score":0.4143,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3616,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":114.1},"38":{"score":0.135,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":467.1},"39":{"score":0.6596,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.39,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":110.5,"tracks":[{"track":"A · Arabic","score":0.281,"headline":false},{"track":"A · English","score":0.39,"headline":true},{"track":"C · Arabic pairs","score":0.645,"headline":false},{"track":"C · English pairs","score":0.84,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4396,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":111.6},"42":{"score":0.7196,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":113.3},"43":{"score":0.3895,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5596,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.7},"45":{"score":0.1107,"jev":null,"metric":"accuracy","requests":501,"answered":488,"unsupported":13,"errors":0,"median_ms":null},"50":{"score":0.4433,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":72.6},"56":{"score":0.6005,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":128.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4368,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":115.1,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.5756,"metric":"accuracy","requests":5507,"answered":5332,"unsupported":175,"errors":0,"median_ms":112.3,"edition_added":"v0.2","jev":null},"59":{"score":0.5344,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":115.8,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.6162,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":57.5,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5468,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":118.2,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5777,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":43.19,"balanced_skill":26.71,"breadth_skill":25.35,"public_skill":26.5},"frozen_scores":{"balanced_raw":43.19,"balanced_skill":26.5,"breadth_skill":25.35},"categories":[{"id":"knowledge","raw":0.379,"skill":0.1789,"combined":0.1528,"coverage":0.924,"pending":0,"n":11},{"id":"language","raw":0.5452,"skill":0.3402,"combined":0.3589,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3145,"skill":0.1572,"combined":0.1781,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.4357,"skill":0.391,"combined":0.3965,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.5044,"skill":0.2792,"combined":0.3295,"coverage":0.8333,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.9327,"skill":0.9092,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8034,"private_eq":0.8553},"2":{"raw":0.5961,"skill":0.5335,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6355,"private_eq":0.6871},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.7954,"skill":0.5172,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7828,"headline":false},{"track":"RouterBench-5shot","score":0.8079,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0909,"skill":0.0909,"coverage":1,"random":0,"tracks":[],"combined":0.0527,"private_eq":0.0625},"11":{"raw":0.5951,"skill":0.4144,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5491,"skill":0.3246,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4486,"private_eq":0.6354},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8517,"skill":0.7034,"coverage":1,"random":0.5,"tracks":[],"combined":0.7479,"private_eq":0.8749},"21":{"raw":0.5993,"skill":0.1986,"coverage":1,"random":0.5,"tracks":[],"combined":0.2466,"private_eq":0.6271},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5545,"skill":0.109,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7275,"skill":0.6367,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3673,"skill":0.1565,"coverage":1,"random":0.25,"tracks":[],"combined":0.0971,"private_eq":0.3298},"28":{"raw":0.6598,"skill":0.3196,"coverage":1,"random":0.5,"tracks":[],"combined":0.1133,"private_eq":0.5612},"29":{"raw":0.8272,"skill":0.7696,"coverage":1,"random":0.25,"tracks":[],"combined":0.6218,"private_eq":0.7188},"30":{"raw":0.2449,"skill":0.0871,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3351,"random":0.25},{"track":"10-choice","raw":0.1547,"random":0.1}],"combined":0.2705,"private_eq":0.4042},"31":{"raw":0.0522,"skill":0,"coverage":0.1574,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0834},"32":{"raw":0.5465,"skill":0.279,"coverage":1,"random":0.371,"tracks":[],"combined":0.0339,"private_eq":0.3986},"33":{"raw":0.2,"skill":0.1894,"coverage":1,"random":0.0131,"tracks":[],"combined":0.215,"private_eq":0.2331},"36":{"raw":0.4143,"skill":0.3374,"coverage":1,"random":0.116,"tracks":[],"combined":0.2831,"private_eq":0.3726},"37":{"raw":0.3616,"skill":0.1992,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3854,"private_eq":0.5148},"38":{"raw":0.135,"skill":0.1073,"coverage":1,"random":0.031,"tracks":[],"combined":0.1661,"private_eq":0.2002},"39":{"raw":0.6596,"skill":0.4993,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3762,"private_eq":0.58},"40":{"raw":0.39,"skill":0.2152,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.281,"headline":false},{"track":"A · English","score":0.39,"headline":true},{"track":"C · Arabic pairs","score":0.645,"headline":false},{"track":"C · English pairs","score":0.84,"headline":false}],"combined":0.1549,"private_eq":0.3498},"41":{"raw":0.4396,"skill":0.1595,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4054,"private_eq":0.6074},"42":{"raw":0.7196,"skill":0.4545,"coverage":1,"random":0.486,"tracks":[],"combined":0.6389,"private_eq":0.816},"43":{"raw":0.3895,"skill":0.0313,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2694,"private_eq":0.5441},"44":{"raw":0.5596,"skill":0.1192,"coverage":1,"random":0.5,"tracks":[],"combined":0.278,"private_eq":0.6426},"45":{"raw":0.1078,"skill":0,"coverage":0.9741,"random":0.1641,"tracks":[]},"50":{"raw":0.4433,"skill":0.1921,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.6005,"skill":0.201,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1271,"private_eq":0.568},"57":{"raw":0.4368,"skill":0.3666,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2123,"private_eq":0.3067},"58":{"raw":0.5573,"skill":0.3583,"coverage":0.9682,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0707,"private_eq":0.3655},"59":{"raw":0.5344,"skill":0.0346,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.6162,"skill":0.2324,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3486,"private_eq":0.6775},"62":{"raw":0.5468,"skill":0.3957,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.5013,"private_eq":0.6295},"64":{"raw":0.5777,"skill":0.4721,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.4968,"private_eq":0.6012}},"coverage":0.6863,"pending":0.24,"latency_rows":673,"latency_path":"author score_shared with shared prefix, bf16, causal-conv1d 1.7.0 CUDA kernel (JEV_SEMIF_REQUIRE_CUDA_CONV=1), board lifts; fleet venv torch 2.14.0+cu130","v03":{"engine":"semif","pub":26.5,"same_skills":28.04,"new_domains":18.74,"v":26.71,"lab":false,"jev":false,"rank_v":176,"rank_pub":202,"name":"SemIf Qwen3.5-4B Base","eq":{"S":27.32,"O":25.79},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":16.39,"new_domains":10.32}}},{"engine":"mini-jev","name":"mini-jev","long_name":"mini-jev Qwen3-4B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4022468096,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/r-ms/mini-jev","base_model":"Qwen/Qwen3-4B-Instruct-2507","base_url":"https://huggingface.co/Qwen/Qwen3-4B-Instruct-2507","weights_repo":null,"served_checkpoint":"Qwen/Qwen3-4B-Instruct-2507"},"completed":95701,"counts":{"ok":95701,"unsupported":14500},"lifted":false,"calibration":{"acc":0.604,"conf":0.9497,"ece":0.3457,"brier":0.7257,"over95":0.2734,"n":70199,"benchmarks":28,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0,"conf":null,"acc":null},{"lo":0.2,"w":0.0008,"conf":0.2683,"acc":0},{"lo":0.3,"w":0.0019,"conf":0.3573,"acc":0.081},{"lo":0.4,"w":0.0068,"conf":0.4561,"acc":0.3247},{"lo":0.5,"w":0.0289,"conf":0.5543,"acc":0.3071},{"lo":0.6,"w":0.0302,"conf":0.6506,"acc":0.4033},{"lo":0.7,"w":0.0364,"conf":0.7533,"acc":0.3752},{"lo":0.8,"w":0.0488,"conf":0.8553,"acc":0.3883},{"lo":0.9,"w":0.8463,"conf":0.9936,"acc":0.6475}],"areas":{"knowledge":{"ece":0.3956,"acc":0.5307,"conf":0.9264},"language":{"ece":0.3249,"acc":0.6478,"conf":0.9726},"retrieval":{"ece":0.3419,"acc":0.6439,"conf":0.9857},"tools":{"ece":0.1914,"acc":0.7885,"conf":0.9798},"arts":{"ece":0.3342,"acc":0.5947,"conf":0.9289}}},"gaps":{"share":0.1205,"reasons":[{"reason":"supports only 2–26 answer options per question","share":0.1205},{"reason":"ran out of GPU memory on the longest prompts","share":0}]},"latency":{"median":66.9,"p95":178,"mean":94.7},"results":{"1":{"score":0.7739,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":72.4},"2":{"score":0.4491,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1117.3},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.7938,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":146.6},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5875,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":81.6},"11":{"score":0.5444,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.5194,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":65.4},"20":{"score":0.8098,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":65.9},"21":{"score":0.5822,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":65.4},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.5418,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":66.9},"24":{"score":0.6734,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":65.8},"25":{"score":0.3061,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9579,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":65.8},"27":{"score":0.884,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":65.5},"28":{"score":0.5951,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":65.8},"29":{"score":0.7721,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":66.5},"30":{"score":0.2877,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1688,"jev":null,"metric":"accuracy","requests":5000,"answered":1588,"unsupported":3412,"errors":0,"median_ms":67.8},"32":{"score":0.5957,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2079,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":95},"36":{"score":0.3296,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2921,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":67.4},"38":{"score":0.0912,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":359.2},"39":{"score":0.6397,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3613,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":65.8,"tracks":[{"track":"A · Arabic","score":0.2807,"headline":false},{"track":"A · English","score":0.3613,"headline":true},{"track":"C · Arabic pairs","score":0.75,"headline":false},{"track":"C · English pairs","score":0.865,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4306,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":66.1},"42":{"score":0.7075,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":69.2},"43":{"score":0.3877,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5442,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":66},"45":{"score":0.1198,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4684,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":71.4},"56":{"score":0.5125,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":86.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.4169,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":66.6,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.5141,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":65.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":73.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5723,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":68,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5027,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":69.5,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.5549,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":38.06,"balanced_skill":22.81,"breadth_skill":20.07,"public_skill":21.25},"frozen_scores":{"balanced_raw":38.06,"balanced_skill":21.25,"breadth_skill":20.07},"categories":[{"id":"knowledge","raw":0.3683,"skill":0.1583,"combined":0.1807,"coverage":0.9431,"pending":0,"n":11},{"id":"language","raw":0.4601,"skill":0.2979,"combined":0.2912,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2685,"skill":0.0866,"combined":0.0998,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.3482,"skill":0.2842,"combined":0.296,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.491,"skill":0.2529,"combined":0.3193,"coverage":0.8333,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.7739,"skill":0.6948,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5379,"private_eq":0.6608},"2":{"raw":0.4491,"skill":0.3638,"coverage":1,"random":0.1341,"tracks":[],"combined":0.4816,"private_eq":0.5554},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.7938,"skill":0.5141,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7791,"headline":false},{"track":"RouterBench-5shot","score":0.8085,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.5444,"skill":0.3411,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5194,"skill":0.2801,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3519,"private_eq":0.5716},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.8024,"skill":0.6049,"coverage":1,"random":0.5,"tracks":[],"combined":0.7263,"private_eq":0.8642},"21":{"raw":0.5822,"skill":0.1644,"coverage":1,"random":0.5,"tracks":[],"combined":0.1157,"private_eq":0.5624},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5381,"skill":0.0761,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7019,"skill":0.6026,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3061,"skill":0.0748,"coverage":1,"random":0.25,"tracks":[],"combined":0.0635,"private_eq":0.3049},"28":{"raw":0.5951,"skill":0.1902,"coverage":1,"random":0.5,"tracks":[],"combined":0.2897,"private_eq":0.6484},"29":{"raw":0.7721,"skill":0.6961,"coverage":1,"random":0.25,"tracks":[],"combined":0.5029,"private_eq":0.6307},"30":{"raw":0.2877,"skill":0.142,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4011,"random":0.25},{"track":"10-choice","raw":0.1744,"random":0.1}],"combined":0.271,"private_eq":0.4046},"31":{"raw":0.0536,"skill":0,"coverage":0.3176,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0824},"32":{"raw":0.5957,"skill":0.3572,"coverage":1,"random":0.371,"tracks":[],"combined":0.1731,"private_eq":0.4852},"33":{"raw":0.2079,"skill":0.1974,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2868,"private_eq":0.3032},"36":{"raw":0.3296,"skill":0.2416,"coverage":1,"random":0.116,"tracks":[],"combined":0.2423,"private_eq":0.3369},"37":{"raw":0.2921,"skill":0.112,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0912,"skill":0.0621,"coverage":1,"random":0.031,"tracks":[],"combined":0.116,"private_eq":0.1522},"39":{"raw":0.6397,"skill":0.4701,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3717,"private_eq":0.577},"40":{"raw":0.3613,"skill":0.1783,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2807,"headline":false},{"track":"A · English","score":0.3613,"headline":true},{"track":"C · Arabic pairs","score":0.75,"headline":false},{"track":"C · English pairs","score":0.865,"headline":false}],"combined":0.0797,"private_eq":0.292},"41":{"raw":0.4306,"skill":0.1459,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2298,"private_eq":0.4917},"42":{"raw":0.7075,"skill":0.4309,"coverage":1,"random":0.486,"tracks":[],"combined":0.6332,"private_eq":0.8131},"43":{"raw":0.3877,"skill":0.0286,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2652,"private_eq":0.5415},"44":{"raw":0.5442,"skill":0.0884,"coverage":1,"random":0.5,"tracks":[],"combined":0.0747,"private_eq":0.5421},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4684,"skill":0.2284,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.5125,"skill":0.025,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2482,"private_eq":0.6279},"57":{"raw":0.4169,"skill":0.3442,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.32,"private_eq":0.4014},"58":{"raw":0.5141,"skill":0.2957,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.069,"private_eq":0.3643},"59":{"raw":0,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5723,"skill":0.1446,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1556,"private_eq":0.5821},"62":{"raw":0.5027,"skill":0.3369,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4946,"private_eq":0.6245},"64":{"raw":0.5549,"skill":0.4436,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.5259,"private_eq":0.6242}},"coverage":0.6927,"pending":0.24,"latency_rows":678,"latency_path":"author mini-jev engine in-process, bf16, fleet venv torch 2.14.0+cu130 (same as board)","v03":{"engine":"mini-jev","pub":21.25,"same_skills":23.36,"new_domains":16.61,"v":22.81,"lab":false,"jev":false,"rank_v":188,"rank_pub":210,"name":"mini-jev Qwen3-4B","eq":{"S":22.58,"O":23.16},"rank_tie":43,"tied":true,"parts":{"public":0,"same_skills":13.55,"new_domains":9.27}}},{"engine":"sieve-2b","name":"Sieve 2B","long_name":"Sieve 2B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":2270000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/sieve-2b","code_url":null,"base_model":"Qwen/Qwen3.5-2B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B-Base","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"}},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.5341,"conf":0.6225,"ece":0.0885,"brier":0.5782,"over95":0.009,"n":34833,"benchmarks":32,"rel":[{"lo":0,"w":0.0543,"conf":0.0724,"acc":0.0366},{"lo":0.1,"w":0.0123,"conf":0.1521,"acc":0.1127},{"lo":0.2,"w":0.0381,"conf":0.2543,"acc":0.1937},{"lo":0.3,"w":0.0709,"conf":0.3541,"acc":0.2705},{"lo":0.4,"w":0.0864,"conf":0.4533,"acc":0.3627},{"lo":0.5,"w":0.1885,"conf":0.5476,"acc":0.4964},{"lo":0.6,"w":0.1495,"conf":0.6476,"acc":0.5714},{"lo":0.7,"w":0.1283,"conf":0.7477,"acc":0.6188},{"lo":0.8,"w":0.1195,"conf":0.8504,"acc":0.7045},{"lo":0.9,"w":0.1523,"conf":0.9537,"acc":0.8553}],"areas":{"knowledge":{"ece":0.104,"acc":0.475,"conf":0.579},"language":{"ece":0.1067,"acc":0.5994,"conf":0.7062},"retrieval":{"ece":0.0972,"acc":0.5443,"conf":0.6188},"tools":{"ece":0.1088,"acc":0.6558,"conf":0.7572},"arts":{"ece":0.0453,"acc":0.4651,"conf":0.4898}}},"gaps":null,"latency":{"median":62.5,"p95":null,"mean":136.4},"results":{"1":{"score":0.781,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":275.6},"2":{"score":0.5783,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1476.2},"3":{"score":0.4567,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":691.8},"4":{"score":0.7528,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":277.8},"5":{"score":0.5129,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":439.3},"6":{"score":0.7957,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":548.5},"9":{"score":0.0114,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1309.3},"10":{"score":0.3893,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":172.5},"11":{"score":0.3929,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2519.9},"12":{"score":0.4076,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":119},"20":{"score":0.7012,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.8},"21":{"score":0.5746,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":115.8},"22":{"score":0.0105,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2807.6},"23":{"score":0.5872,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":131.9},"24":{"score":0.6026,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":114.7},"25":{"score":0.2959,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":113.7},"26":{"score":0.9339,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":113.1},"27":{"score":0.8353,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":113.1},"28":{"score":0.5541,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":109.6},"29":{"score":0.6256,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":118.2},"30":{"score":0.2127,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0936,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":203.1},"32":{"score":0.516,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":178.2},"33":{"score":0.0848,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":513.9},"36":{"score":0.3341,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2000.2},"37":{"score":0.2447,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":146.9},"38":{"score":0.0635,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":912.6},"39":{"score":0.681,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":134.6},"40":{"score":0.2853,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":117.3},"41":{"score":0.4154,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":108.1},"42":{"score":0.6544,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":129.2},"43":{"score":0.3053,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":116.6},"44":{"score":0.5454,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":106.3},"45":{"score":0.1218,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":129.8},"50":{"score":0.4278,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":168.7},"56":{"score":0.5,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":224.6},"57":{"score":0.3517,"jev":null,"metric":"accuracy","requests":12032,"answered":12031,"unsupported":1,"errors":0,"median_ms":124.8},"58":{"score":0.4649,"jev":null,"metric":"accuracy","requests":5507,"answered":5506,"unsupported":1,"errors":0,"median_ms":114.6},"59":{"score":0.0695,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":150.9},"61":{"score":0.5092,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":129.5},"62":{"score":0.1889,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":146.6},"64":{"score":0.4394,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":129.6}},"benchmarks":{"1":{"raw":0.781,"skill":0.7043,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5855,"private_eq":0.6957},"2":{"raw":0.5783,"skill":0.513,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6159,"private_eq":0.6703},"3":{"raw":0.4567,"skill":0.4462,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3082,"private_eq":0.328},"4":{"raw":0.7528,"skill":0.7496,"coverage":1,"random":0.0127,"tracks":[],"combined":0.72,"private_eq":0.7257},"5":{"raw":0.5129,"skill":0.51,"coverage":1,"random":0.006,"tracks":[],"combined":0.4484,"private_eq":0.457},"6":{"raw":0.7957,"skill":0.521,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0114,"skill":0.0114,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.3893,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.3929,"skill":0.1221,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4076,"skill":0.1126,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3716,"private_eq":0.5846},"20":{"raw":0.7035,"skill":0.4071,"coverage":1,"random":0.5,"tracks":[],"combined":0.5816,"private_eq":0.7927},"21":{"raw":0.5746,"skill":0.1492,"coverage":1,"random":0.5,"tracks":[],"combined":0.136,"private_eq":0.5724},"22":{"raw":0.0108,"skill":0.0031,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5696,"skill":0.1392,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6026,"skill":0.4701,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2959,"skill":0.0612,"coverage":1,"random":0.25,"tracks":[],"combined":0.0312,"private_eq":0.2809},"26":{"raw":0.9339,"skill":0.9118,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8353,"skill":0.7803,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5541,"skill":0.1082,"coverage":1,"random":0.5,"tracks":[],"combined":0.2242,"private_eq":0.616},"29":{"raw":0.6256,"skill":0.5008,"coverage":1,"random":0.25,"tracks":[],"combined":0.4188,"private_eq":0.5683},"30":{"raw":0.2127,"skill":0.0474,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3002,"random":0.25},{"track":"10-choice","raw":0.1251,"random":0.1}],"combined":0.2453,"private_eq":0.3836},"31":{"raw":0.0936,"skill":0.0128,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0138,"private_eq":0.104},"32":{"raw":0.516,"skill":0.2305,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.3746},"33":{"raw":0.0848,"skill":0.0727,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0042,"private_eq":0.0275},"36":{"raw":0.3341,"skill":0.2467,"coverage":1,"random":0.116,"tracks":[],"combined":0.2228,"private_eq":0.3199},"37":{"raw":0.2447,"skill":0.0526,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1738,"private_eq":0.348},"38":{"raw":0.0635,"skill":0.0335,"coverage":1,"random":0.031,"tracks":[],"combined":0.0514,"private_eq":0.0903},"39":{"raw":0.681,"skill":0.5308,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4523,"private_eq":0.6312},"40":{"raw":0.2853,"skill":0.0805,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2182,"headline":false},{"track":"A · English","score":0.2853,"headline":true},{"track":"C · Arabic pairs","score":0.59,"headline":false},{"track":"C · English pairs","score":0.49,"headline":false}],"combined":0.0255,"private_eq":0.2504},"41":{"raw":0.4154,"skill":0.1232,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3104,"private_eq":0.5448},"42":{"raw":0.6544,"skill":0.3276,"coverage":1,"random":0.486,"tracks":[],"combined":0.044,"private_eq":0.5137},"43":{"raw":0.3053,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2569,"private_eq":0.5363},"44":{"raw":0.5454,"skill":0.0908,"coverage":1,"random":0.5,"tracks":[],"combined":0.1764,"private_eq":0.5924},"45":{"raw":0.1218,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4278,"skill":0.1696,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5038},"57":{"raw":0.3516,"skill":0.2707,"coverage":0.9999,"random":0.1109,"tracks":[],"combined":0.2344,"private_eq":0.3262},"58":{"raw":0.4649,"skill":0.2244,"coverage":0.9998,"random":0.3101,"tracks":[],"combined":0.0531,"private_eq":0.3535},"59":{"raw":0.0695,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5092,"skill":0.0184,"coverage":1,"random":0.5,"tracks":[],"combined":0.0081,"private_eq":0.5092},"62":{"raw":0.1889,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0.0359,"private_eq":0.2844},"64":{"raw":0.4394,"skill":0.2992,"coverage":1,"random":0.2,"tracks":[],"combined":0.2543,"private_eq":0.4094}},"coverage":1,"pending":0,"scores":{"balanced_raw":40.7,"balanced_skill":21.9,"breadth_skill":20.97,"public_skill":22.16},"frozen_scores":{"balanced_raw":40.7,"balanced_skill":22.16,"breadth_skill":20.97},"categories":[{"id":"knowledge","raw":0.3253,"skill":0.1043,"combined":0.1244,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4045,"skill":0.2078,"combined":0.2169,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4809,"skill":0.2846,"combined":0.2801,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4192,"skill":0.3528,"combined":0.3173,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4543,"skill":0.1946,"combined":0.2358,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-2b","pub":22.16,"same_skills":23.66,"new_domains":14.41,"v":21.9,"lab":false,"jev":false,"rank_v":191,"rank_pub":207,"name":"Sieve 2B","eq":{"S":22.88,"O":20.44},"rank_tie":44,"tied":true,"parts":{"public":0,"same_skills":13.73,"new_domains":8.18}}},{"engine":"decider-2b-fp8-http","name":"Decider 2B","long_name":"Decider 2B","variant":"FP8","meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":3763692048,"model_url":"https://huggingface.co/Mapika/decider-2b","code_url":null,"base_model":"Qwen/Qwen3.5-2B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B-Base","weights_repo":"Mapika/decider-2b","served_checkpoint":"Mapika/decider-2b"},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.5859,"conf":0.663,"ece":0.0771,"brier":0.52,"over95":0.0097,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.0516,"conf":0.0575,"acc":0.0288},{"lo":0.1,"w":0.0298,"conf":0.1499,"acc":0.1282},{"lo":0.2,"w":0.0273,"conf":0.2553,"acc":0.1883},{"lo":0.3,"w":0.059,"conf":0.3525,"acc":0.2861},{"lo":0.4,"w":0.0691,"conf":0.455,"acc":0.3442},{"lo":0.5,"w":0.1369,"conf":0.5491,"acc":0.4997},{"lo":0.6,"w":0.1238,"conf":0.6498,"acc":0.5915},{"lo":0.7,"w":0.1297,"conf":0.7494,"acc":0.6396},{"lo":0.8,"w":0.1325,"conf":0.8508,"acc":0.742},{"lo":0.9,"w":0.2403,"conf":0.9603,"acc":0.8816}],"areas":{"knowledge":{"ece":0.0802,"acc":0.5128,"conf":0.593},"language":{"ece":0.0597,"acc":0.7007,"conf":0.7604},"retrieval":{"ece":0.1054,"acc":0.6542,"conf":0.7537},"tools":{"ece":0.0671,"acc":0.6702,"conf":0.7337},"arts":{"ece":0.0917,"acc":0.4347,"conf":0.5263}}},"gaps":{"share":0,"reasons":[{"reason":"runtime error","share":0}]},"latency":{"median":8.1,"p95":134.9,"mean":40.5},"results":{"1":{"score":0.8294,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":21.6},"2":{"score":0.6312,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":150.3},"3":{"score":0.3169,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7277,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":13.1},"5":{"score":0.7369,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":36.3},"6":{"score":0.7988,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":132.3},"9":{"score":0.0341,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4885,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":40.4},"11":{"score":0.634,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4937,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":6.3},"20":{"score":0.6492,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.4},"21":{"score":0.5993,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":5.8},"22":{"score":0.0025,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":154.8},"23":{"score":0.5696,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.8},"24":{"score":0.619,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":6.1},"25":{"score":0.3622,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9377,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6},"27":{"score":0.8515,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":5.9},"28":{"score":0.6772,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":5.8},"29":{"score":0.8581,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":7},"30":{"score":0.2309,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0988,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.7},"32":{"score":0.5492,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2467,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":75.9},"36":{"score":0.3711,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3468,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":8.2},"38":{"score":0.0786,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":110.2},"39":{"score":0.7612,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2243,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":6,"tracks":[{"track":"A · Arabic","score":0.2174,"headline":false},{"track":"A · English","score":0.2243,"headline":true},{"track":"C · Arabic pairs","score":0.375,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.5104,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":6.7},"42":{"score":0.6547,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.7},"43":{"score":0.3298,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5726,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.3},"45":{"score":0.1198,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4218,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":15},"56":{"score":0.5005,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":38.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.3758,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":6.8,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.5097,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.2835,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":14.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5907,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":10.4,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5471,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":14.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.447,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":47.01,"balanced_skill":23.65,"breadth_skill":28.46,"public_skill":29.75},"frozen_scores":{"balanced_raw":47.01,"balanced_skill":29.75,"breadth_skill":28.46},"categories":[{"id":"knowledge","raw":0.3735,"skill":0.1718,"combined":0.1541,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5073,"skill":0.3221,"combined":0.2736,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5516,"skill":0.3728,"combined":0.3669,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4793,"skill":0.4245,"combined":0.3566,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4439,"skill":0.1757,"combined":0.2443,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.8294,"skill":0.7697,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6876,"private_eq":0.7705},"2":{"raw":0.6312,"skill":0.5741,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6504,"private_eq":0.6999},"3":{"raw":0.3169,"skill":0.3037,"coverage":1,"random":0.0189,"tracks":[],"combined":0.1035,"private_eq":0.1295},"4":{"raw":0.7277,"skill":0.7242,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7253,"private_eq":0.7309},"5":{"raw":0.7369,"skill":0.7353,"coverage":1,"random":0.006,"tracks":[],"combined":0.4768,"private_eq":0.4849},"6":{"raw":0.7988,"skill":0.5256,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7853,"headline":false},{"track":"RouterBench-5shot","score":0.8123,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0341,"skill":0.0341,"coverage":1,"random":0,"tracks":[],"combined":0.0299,"private_eq":0.04},"11":{"raw":0.634,"skill":0.4706,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4937,"skill":0.2416,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3445,"private_eq":0.5667},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.6481,"skill":0.2962,"coverage":1,"random":0.5,"tracks":[],"combined":0.4661,"private_eq":0.7356},"21":{"raw":0.5993,"skill":0.1986,"coverage":1,"random":0.5,"tracks":[],"combined":0.136,"private_eq":0.5724},"22":{"raw":0.002,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0266,"private_eq":0.0442},"23":{"raw":0.5449,"skill":0.0897,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6439,"skill":0.5253,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3622,"skill":0.1497,"coverage":1,"random":0.25,"tracks":[],"combined":0.0467,"private_eq":0.2924},"28":{"raw":0.6772,"skill":0.3544,"coverage":1,"random":0.5,"tracks":[],"combined":0.2393,"private_eq":0.6235},"29":{"raw":0.8581,"skill":0.8108,"coverage":1,"random":0.25,"tracks":[],"combined":0.6366,"private_eq":0.7298},"30":{"raw":0.2309,"skill":0.0684,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3071,"random":0.25},{"track":"10-choice","raw":0.1547,"random":0.1}],"combined":0.2424,"private_eq":0.3813},"31":{"raw":0.0988,"skill":0.0185,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0019,"private_eq":0.0932},"32":{"raw":0.5492,"skill":0.2833,"coverage":1,"random":0.371,"tracks":[],"combined":0.0803,"private_eq":0.4275},"33":{"raw":0.2467,"skill":0.2367,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2509,"private_eq":0.2681},"36":{"raw":0.3711,"skill":0.2885,"coverage":1,"random":0.116,"tracks":[],"combined":0.2344,"private_eq":0.33},"37":{"raw":0.3468,"skill":0.1808,"coverage":1,"random":0.2027,"tracks":[],"combined":0.2742,"private_eq":0.4271},"38":{"raw":0.0786,"skill":0.0491,"coverage":1,"random":0.031,"tracks":[],"combined":0.0754,"private_eq":0.1133},"39":{"raw":0.7612,"skill":0.6488,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4105,"private_eq":0.6031},"40":{"raw":0.2243,"skill":0.0021,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2174,"headline":false},{"track":"A · English","score":0.2243,"headline":true},{"track":"C · Arabic pairs","score":0.375,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.5104,"skill":0.2656,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1362,"private_eq":0.43},"42":{"raw":0.6547,"skill":0.3282,"coverage":1,"random":0.486,"tracks":[],"combined":0.4466,"private_eq":0.7183},"43":{"raw":0.3298,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2904,"private_eq":0.5572},"44":{"raw":0.5726,"skill":0.1452,"coverage":1,"random":0.5,"tracks":[],"combined":0.1712,"private_eq":0.5898},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4218,"skill":0.1609,"coverage":1,"random":0.311,"tracks":[],"combined":0.2902,"private_eq":0.5158},"56":{"raw":0.5005,"skill":0.001,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0751,"private_eq":0.5423},"57":{"raw":0.3758,"skill":0.2979,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.1889,"private_eq":0.2862},"58":{"raw":0.5097,"skill":0.2893,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0484,"private_eq":0.3503},"59":{"raw":0.2835,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.0629,"private_eq":0.5527},"61":{"raw":0.5907,"skill":0.1814,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3486,"private_eq":0.6775},"62":{"raw":0.5471,"skill":0.3961,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.296,"private_eq":0.4773},"64":{"raw":0.447,"skill":0.3087,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.3028,"private_eq":0.4478}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author decider.serve through decider_memory_app (DECIDER_COMPILE=1, DECIDER_FP8=1, CUDA graphs, DECIDER_MAX_FWD_TOKENS 65536), same server as board; one sequential HTTP client (run_decider_http_suite.py --concurrency 1) instead of concurrency 8","v03":{"engine":"decider-2b-fp8-http","pub":29.75,"same_skills":28.47,"new_domains":12.04,"v":23.65,"lab":false,"jev":false,"rank_v":185,"rank_pub":190,"name":"Decider 2B","eq":{"S":27.75,"O":17.51},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":16.65,"new_domains":7}}},{"engine":"decision-2.0-sol-2b","name":"vLLM-SR Decision 2.0 Sol 2B","long_name":"vLLM-SR Decision 2.0 Sol 2B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2270000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/Decision-2.0-Sol-2B","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.5777,"conf":0.6105,"ece":0.0366,"brier":0.5332,"over95":0.006,"n":34833,"benchmarks":32,"rel":[{"lo":0,"w":0.0557,"conf":0.0405,"acc":0.0326},{"lo":0.1,"w":0.0153,"conf":0.153,"acc":0.1074},{"lo":0.2,"w":0.0451,"conf":0.2616,"acc":0.2543},{"lo":0.3,"w":0.0908,"conf":0.3534,"acc":0.371},{"lo":0.4,"w":0.1025,"conf":0.4493,"acc":0.4483},{"lo":0.5,"w":0.1607,"conf":0.5477,"acc":0.5375},{"lo":0.6,"w":0.1266,"conf":0.6489,"acc":0.6511},{"lo":0.7,"w":0.1241,"conf":0.7505,"acc":0.686},{"lo":0.8,"w":0.1316,"conf":0.8493,"acc":0.7455},{"lo":0.9,"w":0.1475,"conf":0.9549,"acc":0.8884}],"areas":{"knowledge":{"ece":0.0473,"acc":0.4872,"conf":0.5269},"language":{"ece":0.0735,"acc":0.643,"conf":0.7097},"retrieval":{"ece":0.0591,"acc":0.6808,"conf":0.6756},"tools":{"ece":0.0925,"acc":0.7999,"conf":0.7086},"arts":{"ece":0.0666,"acc":0.4433,"conf":0.505}}},"gaps":null,"latency":{"median":25.7,"p95":null,"mean":51.4},"results":{"1":{"score":0.8217,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":28.6},"2":{"score":0.6203,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":220.5},"3":{"score":0.6791,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":94},"4":{"score":0.7831,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":33},"5":{"score":0.7864,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":51.7},"6":{"score":0.734,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":55.4},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":191.3},"10":{"score":0.6201,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":25.9},"11":{"score":0.6048,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":374},"12":{"score":0.444,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":26.7},"20":{"score":0.6126,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.7},"21":{"score":0.5529,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":25.6},"22":{"score":0.0135,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":89.2},"23":{"score":0.5668,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.6},"24":{"score":0.5028,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":23.3},"25":{"score":0.3163,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":23.6},"26":{"score":0.9141,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":22.9},"27":{"score":0.7398,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":23.2},"28":{"score":0.7009,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":22},"29":{"score":0.6331,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":23.3},"30":{"score":0.365,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0674,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.7},"32":{"score":0.5612,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":24.4},"33":{"score":0.1297,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":47.5},"36":{"score":0.3923,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":403.4},"37":{"score":0.3376,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.1},"38":{"score":0.1008,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":126.4},"39":{"score":0.6095,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":26},"40":{"score":0.2341,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":24.9},"41":{"score":0.4496,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":25.1},"42":{"score":0.6847,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":26.3},"43":{"score":0.3737,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":25.1},"44":{"score":0.5904,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":25},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":26.1},"50":{"score":0.3813,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":27.1},"56":{"score":0.5085,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":38.2},"57":{"score":0.2519,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":25.4},"58":{"score":0.4965,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":24.7},"59":{"score":0.102,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":26.9},"61":{"score":0.7843,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":26},"62":{"score":0.517,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":26.1},"64":{"score":0.4659,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":24.8}},"benchmarks":{"1":{"raw":0.8217,"skill":0.7593,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.7206,"combined":0.6195},"2":{"raw":0.6194,"skill":0.5605,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.6615,"private_eq":0.7094},"3":{"raw":0.6791,"skill":0.6729,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5496,"private_eq":0.5622},"4":{"raw":0.7831,"skill":0.7803,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.7073,"combined":0.7011},"5":{"raw":0.7864,"skill":0.7851,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.4352,"combined":0.4263},"6":{"raw":0.734,"skill":0.3763,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0.0025},"10":{"raw":0.6201,"skill":0.3679,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6048,"skill":0.4284,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.444,"skill":0.1672,"coverage":1,"random":0.3324,"tracks":[],"combined":0.1159,"private_eq":0.4158},"20":{"raw":0.6122,"skill":0.2245,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6712,"combined":0.3358},"21":{"raw":0.5529,"skill":0.1058,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0128,"skill":0.0051,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.002,"private_eq":0.0201},"23":{"raw":0.5474,"skill":0.0949,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5028,"skill":0.3371,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3163,"skill":0.0884,"coverage":1,"random":0.25,"tracks":[],"combined":0.0228,"private_eq":0.2747},"26":{"raw":0.9141,"skill":0.8854,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7398,"skill":0.653,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7009,"skill":0.4018,"coverage":1,"random":0.5,"tracks":[],"combined":0.214,"private_eq":0.611},"29":{"raw":0.6331,"skill":0.5108,"coverage":1,"random":0.25,"tracks":[],"combined":0.3891,"private_eq":0.5463},"30":{"raw":0.365,"skill":0.2359,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4723,"random":0.25},{"track":"10-choice","raw":0.2578,"random":0.1}],"private_eq":0.5458,"combined":0.4441},"31":{"raw":0.0674,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0903},"32":{"raw":0.5612,"skill":0.3024,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1297,"skill":0.1181,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0475,"combined":0.0247},"36":{"raw":0.3923,"skill":0.3125,"coverage":1,"random":0.116,"tracks":[],"combined":0.2655,"private_eq":0.3572},"37":{"raw":0.3376,"skill":0.1692,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3126,"private_eq":0.4574},"38":{"raw":0.1008,"skill":0.072,"coverage":1,"random":0.031,"tracks":[],"combined":0.126,"private_eq":0.1618},"39":{"raw":0.6095,"skill":0.4257,"coverage":1,"random":0.3201,"tracks":[],"combined":0.2963,"private_eq":0.5263},"40":{"raw":0.2341,"skill":0.0147,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2973,"headline":false},{"track":"A · English","score":0.2341,"headline":true},{"track":"C · Arabic pairs","score":0.855,"headline":false},{"track":"C · English pairs","score":0.73,"headline":false}],"private_eq":0.248,"combined":0.0224},"41":{"raw":0.4496,"skill":0.1744,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2799,"private_eq":0.5247},"42":{"raw":0.6847,"skill":0.3866,"coverage":1,"random":0.486,"tracks":[],"combined":0.6964,"private_eq":0.8452},"43":{"raw":0.3737,"skill":0.0063,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1816,"private_eq":0.4894},"44":{"raw":0.5904,"skill":0.1808,"coverage":1,"random":0.5,"tracks":[],"combined":0.353,"private_eq":0.6797},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3813,"skill":0.102,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.5085,"skill":0.017,"coverage":1,"random":0.5,"tracks":[],"combined":0.0923,"private_eq":0.5508},"57":{"raw":0.2519,"skill":0.1586,"coverage":1,"random":0.1109,"tracks":[],"combined":0.0766,"private_eq":0.1875},"58":{"raw":0.4965,"skill":0.2702,"coverage":1,"random":0.3101,"tracks":[],"combined":0.0279,"private_eq":0.3363},"59":{"raw":0.102,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.7843,"skill":0.5686,"coverage":1,"random":0.5,"tracks":[],"combined":0.5924,"private_eq":0.798},"62":{"raw":0.517,"skill":0.356,"coverage":1,"random":0.25,"tracks":[],"combined":0.4603,"private_eq":0.5991},"64":{"raw":0.4659,"skill":0.3324,"coverage":1,"random":0.2,"tracks":[],"combined":0.3222,"private_eq":0.4631}},"coverage":0.9999,"pending":0.0001,"scores":{"balanced_raw":46.89,"balanced_skill":23.64,"breadth_skill":27.75,"public_skill":30.05},"frozen_scores":{"balanced_raw":46.89,"balanced_skill":30.05,"breadth_skill":27.75},"categories":[{"id":"knowledge","raw":0.3619,"skill":0.1622,"combined":0.1298,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4338,"skill":0.2463,"combined":0.24,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6037,"skill":0.4558,"combined":0.4044,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.544,"skill":0.488,"combined":0.4661,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.4287,"skill":0.1441,"combined":0.1633,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"decision-2.0-sol-2b","pub":30.05,"same_skills":29.25,"new_domains":11.06,"v":23.64,"lab":false,"jev":false,"rank_v":186,"rank_pub":189,"name":"vLLM-SR Decision 2.0 Sol 2B","eq":{"S":28.54,"O":16.3},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":17.12,"new_domains":6.52}}},{"engine":"intern-decision-2b","name":"Intern-Decision-2B","long_name":"InternLM Intern-Decision Qwen3.5-2B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":4426562832,"model_url":"https://huggingface.co/internlm/Intern-Decision-2B","code_url":"https://github.com/internlm/Intern-Decision","base_model":"Qwen/Qwen3.5-2B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B-Base","weights_repo":"internlm/Intern-Decision-2B","served_checkpoint":null,"org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"}},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.4884,"conf":0.5492,"ece":0.0608,"brier":0.6076,"over95":0.0002,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0269,"conf":0.0845,"acc":0.027},{"lo":0.1,"w":0.0567,"conf":0.145,"acc":0.0571},{"lo":0.2,"w":0.0583,"conf":0.246,"acc":0.1486},{"lo":0.3,"w":0.0778,"conf":0.3538,"acc":0.2459},{"lo":0.4,"w":0.115,"conf":0.4514,"acc":0.4273},{"lo":0.5,"w":0.2544,"conf":0.5451,"acc":0.5007},{"lo":0.6,"w":0.175,"conf":0.6452,"acc":0.5615},{"lo":0.7,"w":0.1054,"conf":0.7486,"acc":0.6834},{"lo":0.8,"w":0.0864,"conf":0.8488,"acc":0.8061},{"lo":0.9,"w":0.0441,"conf":0.933,"acc":0.9125}],"areas":{"knowledge":{"ece":0.0634,"acc":0.4866,"conf":0.5355},"language":{"ece":0.0864,"acc":0.5476,"conf":0.6279},"retrieval":{"ece":0.1024,"acc":0.3515,"conf":0.4539},"tools":{"ece":0.0813,"acc":0.5643,"conf":0.577},"arts":{"ece":0.052,"acc":0.4502,"conf":0.4975}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":33.5,"p95":58,"mean":39.2},"results":{"1":{"score":0.7102,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":35.3},"2":{"score":0.4904,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":128.5},"3":{"score":0.2854,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.1291,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.6},"5":{"score":0.0426,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.3},"6":{"score":0.7934,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":46.3},"9":{"score":0.0114,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5929,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":35.9},"11":{"score":0.4477,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4694,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":32.7},"20":{"score":0.589,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.7},"21":{"score":0.578,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.5},"22":{"score":0.0085,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":110},"23":{"score":0.5582,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.5},"24":{"score":0.5727,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":32.5},"25":{"score":0.3418,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9057,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":32.5},"27":{"score":0.7816,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":32.5},"28":{"score":0.5754,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":32.4},"29":{"score":0.6043,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":33.1},"30":{"score":0.2202,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.106,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.8},"32":{"score":0.5439,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1121,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":35.9},"36":{"score":0.2137,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3463,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.5},"38":{"score":0.0402,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":123.2},"39":{"score":0.6317,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2867,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":32.5,"tracks":[{"track":"A · Arabic","score":0.2334,"headline":false},{"track":"A · English","score":0.2867,"headline":true},{"track":"C · Arabic pairs","score":0.515,"headline":false},{"track":"C · English pairs","score":0.585,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3933,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":32.7},"42":{"score":0.6628,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":33.8},"43":{"score":0.3386,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5378,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.6},"45":{"score":0.1277,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.457,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":34.6},"56":{"score":0.7,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":35.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.33,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":33.1,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.4732,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.2969,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5675,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":34.1,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.4907,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":35.1,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.4034,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":37.89,"balanced_skill":20.93,"breadth_skill":18.44,"public_skill":19.26},"frozen_scores":{"balanced_raw":37.89,"balanced_skill":19.26,"breadth_skill":18.44},"categories":[{"id":"knowledge","raw":0.3397,"skill":0.1201,"combined":0.095,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4306,"skill":0.2126,"combined":0.2877,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3146,"skill":0.1565,"combined":0.1771,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.405,"skill":0.3333,"combined":0.3689,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4277,"skill":0.1435,"combined":0.2137,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.7102,"skill":0.6087,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.576,"combined":0.4221},"2":{"raw":0.4904,"skill":0.4114,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6006,"private_eq":0.6572},"3":{"raw":0.2854,"skill":0.2716,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4709,"private_eq":0.4858},"4":{"raw":0.1291,"skill":0.1179,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.1437,"combined":0.1237},"5":{"raw":0.0426,"skill":0.0368,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.0644,"combined":0.0489},"6":{"raw":0.7934,"skill":0.5119,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.7836,"headline":false},{"track":"RouterBench-5shot","score":0.8032,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0114,"skill":0.0114,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.4477,"skill":0.2013,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4694,"skill":0.2052,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3477,"private_eq":0.5688},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5851,"skill":0.1701,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7356,"combined":0.4661},"21":{"raw":0.578,"skill":0.156,"coverage":1,"random":0.5,"tracks":[],"combined":0.136,"private_eq":0.5724},"22":{"raw":0.0085,"skill":0.0008,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0478,"private_eq":0.065},"23":{"raw":0.5341,"skill":0.0682,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5921,"skill":0.4561,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3418,"skill":0.1224,"coverage":1,"random":0.25,"tracks":[],"combined":0.0635,"private_eq":0.3049},"28":{"raw":0.5754,"skill":0.1508,"coverage":1,"random":0.5,"tracks":[],"combined":0.0174,"private_eq":0.5138},"29":{"raw":0.6043,"skill":0.4724,"coverage":1,"random":0.25,"tracks":[],"combined":0.394,"private_eq":0.5499},"30":{"raw":0.2202,"skill":0.0569,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3101,"random":0.25},{"track":"10-choice","raw":0.1304,"random":0.1}],"private_eq":0.3592,"combined":0.2153},"31":{"raw":0.106,"skill":0.0263,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0246,"private_eq":0.1138},"32":{"raw":0.5439,"skill":0.2749,"coverage":1,"random":0.371,"tracks":[],"combined":0.0339,"private_eq":0.3986},"33":{"raw":0.1121,"skill":0.1003,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0877,"combined":0.0659},"36":{"raw":0.2137,"skill":0.1104,"coverage":1,"random":0.116,"tracks":[],"combined":0.1322,"private_eq":0.2407},"37":{"raw":0.3463,"skill":0.1801,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3338,"private_eq":0.4741},"38":{"raw":0.0402,"skill":0.0095,"coverage":1,"random":0.031,"tracks":[],"combined":0.0109,"private_eq":0.0515},"39":{"raw":0.6317,"skill":0.4583,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5441,"private_eq":0.6929},"40":{"raw":0.2867,"skill":0.0824,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2334,"headline":false},{"track":"A · English","score":0.2867,"headline":true},{"track":"C · Arabic pairs","score":0.515,"headline":false},{"track":"C · English pairs","score":0.585,"headline":false}],"private_eq":0.2565,"combined":0.0335},"41":{"raw":0.3933,"skill":0.0899,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2002,"private_eq":0.4722},"42":{"raw":0.6628,"skill":0.344,"coverage":1,"random":0.486,"tracks":[],"combined":0.6086,"private_eq":0.8006},"43":{"raw":0.3386,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2151,"private_eq":0.5103},"44":{"raw":0.5378,"skill":0.0756,"coverage":1,"random":0.5,"tracks":[],"combined":0.021,"private_eq":0.5156},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.457,"skill":0.212,"coverage":1,"random":0.311,"tracks":[],"combined":0.174,"private_eq":0.4367},"56":{"raw":0.7,"skill":0.4,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2569,"private_eq":0.6322},"57":{"raw":0.33,"skill":0.2464,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2556,"private_eq":0.3448},"58":{"raw":0.4732,"skill":0.2364,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0388,"private_eq":0.3437},"59":{"raw":0.2969,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0.1294,"private_eq":0.5844},"61":{"raw":0.5675,"skill":0.135,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2124,"private_eq":0.6102},"62":{"raw":0.4907,"skill":0.3209,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.3303,"private_eq":0.5027},"64":{"raw":0.4034,"skill":0.2542,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.2446,"private_eq":0.4018}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.predict on the whole row, SDPA, max_length 262144 and the board lifts; transformers fast path enabled with causal-conv1d 1.7.0 (fla 0.4.2 gated delta + CUDA conv). The board ran the torch fallback because causal-conv1d was absent; that run is kept as latency-v1-full.torch-fallback. venv-intern torch 2.9.1+cu128","v03":{"engine":"intern-decision-2b","pub":19.26,"same_skills":23.28,"new_domains":12.91,"v":20.93,"lab":false,"jev":false,"rank_v":193,"rank_pub":217,"name":"InternLM Intern-Decision Qwen3.5-2B","eq":{"S":22.5,"O":18.58},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":13.5,"new_domains":7.43}}},{"engine":"jpt-0.8b","name":"JPT-0.8B","long_name":"JPT-0.8B Qwen3.5-0.8B LoRA","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1706035272,"model_url":"https://huggingface.co/kirp/jpt-0.8b","code_url":"https://github.com/tic-top/llm2jev","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"kirp/jpt-0.8b","served_checkpoint":"5921ae03fbaff98fcbe781c1ff3f1200dc897fd1"},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.4947,"conf":0.5626,"ece":0.0688,"brier":0.6121,"over95":0.0035,"n":35093,"benchmarks":32,"rel":[{"lo":0,"w":0.0641,"conf":0.0755,"acc":0.0506},{"lo":0.1,"w":0.0356,"conf":0.1462,"acc":0.1591},{"lo":0.2,"w":0.0491,"conf":0.257,"acc":0.2267},{"lo":0.3,"w":0.0922,"conf":0.3546,"acc":0.3448},{"lo":0.4,"w":0.1158,"conf":0.4482,"acc":0.3676},{"lo":0.5,"w":0.1915,"conf":0.5457,"acc":0.504},{"lo":0.6,"w":0.1441,"conf":0.6459,"acc":0.5451},{"lo":0.7,"w":0.1183,"conf":0.7474,"acc":0.6251},{"lo":0.8,"w":0.1019,"conf":0.8508,"acc":0.7403},{"lo":0.9,"w":0.0875,"conf":0.9452,"acc":0.868}],"areas":{"knowledge":{"ece":0.072,"acc":0.4448,"conf":0.5167},"language":{"ece":0.0734,"acc":0.609,"conf":0.6798},"retrieval":{"ece":0.0916,"acc":0.3913,"conf":0.459},"tools":{"ece":0.0472,"acc":0.6257,"conf":0.6647},"arts":{"ece":0.0766,"acc":0.403,"conf":0.4739}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":117.5,"p95":341.2,"mean":118.6},"results":{"1":{"score":0.7674,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":197.5},"2":{"score":0.5285,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":424.1},"3":{"score":0.3012,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.3541,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":119},"5":{"score":0.1046,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":122.6},"6":{"score":0.6413,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":271.2},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4914,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":122.4},"11":{"score":0.7274,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3856,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":114.4},"20":{"score":0.5006,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44},"21":{"score":0.5221,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":113.6},"22":{"score":0.024,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":174.6},"23":{"score":0.5288,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":115},"24":{"score":0.5017,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":42.5},"25":{"score":0.301,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8481,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":41.3},"27":{"score":0.6809,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":40.8},"28":{"score":0.532,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":41.4},"29":{"score":0.486,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":114.5},"30":{"score":0.21,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0846,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":117.7},"32":{"score":0.4761,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0394,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":291.5},"36":{"score":0.3277,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.301,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.9},"38":{"score":0.0732,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":460.2},"39":{"score":0.7806,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2655,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":113.6,"tracks":[{"track":"A · Arabic","score":0.1504,"headline":false},{"track":"A · English","score":0.2655,"headline":true},{"track":"C · Arabic pairs","score":0.485,"headline":false},{"track":"C · English pairs","score":0.395,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4399,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43},"42":{"score":0.6283,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":119},"43":{"score":0.3386,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.544,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.9},"45":{"score":0.1717,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3944,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":120.9},"56":{"score":0.5205,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":284,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.2795,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":44.1,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.4567,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":43.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5534,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":120,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5485,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":118.2,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.4995,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":120.8,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.4015,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":39.36,"balanced_skill":15.46,"breadth_skill":17.65,"public_skill":19.37},"frozen_scores":{"balanced_raw":39.36,"balanced_skill":19.37,"breadth_skill":17.65},"categories":[{"id":"knowledge","raw":0.3151,"skill":0.0816,"combined":0.0663,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4802,"skill":0.2517,"combined":0.1925,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3506,"skill":0.164,"combined":0.1459,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4278,"skill":0.3623,"combined":0.3402,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3964,"skill":0.0844,"combined":0.0782,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.7674,"skill":0.686,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.6807,"combined":0.565},"2":{"raw":0.5285,"skill":0.4555,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.6691,"combined":0.6145},"3":{"raw":0.3012,"skill":0.2877,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.2669,"combined":0.2452},"4":{"raw":0.3541,"skill":0.3458,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.3053,"combined":0.2893},"5":{"raw":0.1046,"skill":0.0992,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.0124,"combined":0},"6":{"raw":0.6413,"skill":0.1574,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6057,"headline":false},{"track":"RouterBench-5shot","score":0.677,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0.0125,"combined":0.0021},"11":{"raw":0.7274,"skill":0.6058,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3856,"skill":0.0797,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.5188,"combined":0.2719},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.4996,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5497,"combined":0.09},"21":{"raw":0.5221,"skill":0.0441,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"22":{"raw":0.025,"skill":0.0174,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0297,"combined":0.0118},"23":{"raw":0.536,"skill":0.0721,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5137,"skill":0.3517,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.301,"skill":0.068,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2996,"combined":0.0564},"28":{"raw":0.532,"skill":0.064,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5237,"combined":0.0374},"29":{"raw":0.486,"skill":0.3147,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4619,"combined":0.2753},"30":{"raw":0.21,"skill":0.0423,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2805,"random":0.25},{"track":"10-choice","raw":0.1395,"random":0.1}],"private_eq":0.3052,"combined":0.1491},"31":{"raw":0.0846,"skill":0.003,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0932,"combined":0.0019},"32":{"raw":0.4761,"skill":0.1671,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0.0394,"skill":0.0266,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0425,"combined":0.0196},"36":{"raw":0.3277,"skill":0.2394,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.2937,"combined":0.1928},"37":{"raw":0.301,"skill":0.1233,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2853,"combined":0.0943},"38":{"raw":0.0732,"skill":0.0436,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.1212,"combined":0.0836},"39":{"raw":0.7806,"skill":0.6773,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6544,"combined":0.4868},"40":{"raw":0.2655,"skill":0.0551,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1504,"headline":false},{"track":"A · English","score":0.2655,"headline":true},{"track":"C · Arabic pairs","score":0.485,"headline":false},{"track":"C · English pairs","score":0.395,"headline":false}],"private_eq":0.2227,"combined":0},"41":{"raw":0.4399,"skill":0.1599,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3333,"combined":0},"42":{"raw":0.6283,"skill":0.2768,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.6227,"combined":0.2585},"43":{"raw":0.3386,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.5103,"combined":0.2151},"44":{"raw":0.544,"skill":0.088,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5712,"combined":0.1335},"45":{"raw":0.1717,"skill":0.0091,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3944,"skill":0.1211,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4417,"combined":0.1814},"56":{"raw":0.5205,"skill":0.041,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","private_eq":0.5937,"combined":0.179},"57":{"raw":0.2795,"skill":0.1896,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","private_eq":0.1358,"combined":0.0178},"58":{"raw":0.4567,"skill":0.2125,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","private_eq":0.3592,"combined":0.0615},"59":{"raw":0.5534,"skill":0.074,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","private_eq":0.5954,"combined":0.1524},"61":{"raw":0.5485,"skill":0.097,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","private_eq":0.5625,"combined":0.1159},"62":{"raw":0.4995,"skill":0.3327,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","private_eq":0.4418,"combined":0.2482},"64":{"raw":0.4015,"skill":0.2519,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","private_eq":0.302,"combined":0.1185}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"llm2jev 0.6.1 (chat prompt, thinking off, T=1.140) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","v03":{"engine":"jpt-0.8b","pub":19.37,"same_skills":17.82,"new_domains":8.55,"v":15.46,"lab":false,"jev":false,"rank_v":214,"rank_pub":216,"name":"JPT-0.8B Qwen3.5-0.8B LoRA","eq":{"S":16.97,"O":13.2},"rank_tie":50,"tied":true,"parts":{"public":0,"same_skills":10.18,"new_domains":5.28}}},{"engine":"this-that-1.2","name":"this-that 1.2","long_name":"FLock this-that 1.2","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":1881825088,"params_estimated":false,"params_from_name":false,"trained_bytes":3763692048,"model_url":"https://huggingface.co/flock-io/this-that-model-1.2","code_url":"https://github.com/FLock-io/this-that-model","base_model":"Mapika/decider-2b","base_url":"https://huggingface.co/Mapika/decider-2b","weights_repo":"flock-io/this-that-model-1.2","served_checkpoint":null,"org":{"id":"flock-io","name":"FLock.io","logo":"data/orgs/flock-io.png"}},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.5882,"conf":0.7865,"ece":0.1983,"brier":0.5907,"over95":0.0785,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0119,"conf":0.0848,"acc":0.0291},{"lo":0.1,"w":0.0334,"conf":0.1451,"acc":0.0451},{"lo":0.2,"w":0.018,"conf":0.2502,"acc":0.1757},{"lo":0.3,"w":0.027,"conf":0.3546,"acc":0.1461},{"lo":0.4,"w":0.0385,"conf":0.4552,"acc":0.3002},{"lo":0.5,"w":0.0951,"conf":0.5488,"acc":0.4085},{"lo":0.6,"w":0.0846,"conf":0.649,"acc":0.4596},{"lo":0.7,"w":0.0916,"conf":0.7513,"acc":0.5311},{"lo":0.8,"w":0.1132,"conf":0.8547,"acc":0.5823},{"lo":0.9,"w":0.4868,"conf":0.9786,"acc":0.771}],"areas":{"knowledge":{"ece":0.2136,"acc":0.5171,"conf":0.7307},"language":{"ece":0.1613,"acc":0.7028,"conf":0.8627},"retrieval":{"ece":0.2394,"acc":0.6524,"conf":0.8918},"tools":{"ece":0.1689,"acc":0.6815,"conf":0.8496},"arts":{"ece":0.2192,"acc":0.4308,"conf":0.65}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":44.3,"p95":57.2,"mean":92.5},"results":{"1":{"score":0.8235,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":47.3},"2":{"score":0.5401,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":185.3},"3":{"score":0.5236,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7157,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.6},"5":{"score":0.6608,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":52.1},"6":{"score":0.6848,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":53.7},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5428,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":47.8},"11":{"score":0.6468,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4966,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":43.4},"20":{"score":0.5526,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"21":{"score":0.6043,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":43},"22":{"score":0.0135,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":109.8},"23":{"score":0.5808,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.7},"24":{"score":0.5958,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":43.2},"25":{"score":0.3316,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9213,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":43.1},"27":{"score":0.8319,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":43.1},"28":{"score":0.674,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":43},"29":{"score":0.8067,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":43.7},"30":{"score":0.2271,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0868,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.4},"32":{"score":0.5465,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3127,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":47.5},"36":{"score":0.1951,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.3783,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.5},"38":{"score":0.0728,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":144},"39":{"score":0.747,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1942,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":43.2,"tracks":[{"track":"A · Arabic","score":0.0372,"headline":false},{"track":"A · English","score":0.1942,"headline":true},{"track":"C · Arabic pairs","score":0.33,"headline":false},{"track":"C · English pairs","score":0.51,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4996,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.4},"42":{"score":0.6757,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":46},"43":{"score":0.3193,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5642,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.3},"45":{"score":0.1238,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4057,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":47.4},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":47,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.3718,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":43.6,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.4868,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":43.6,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.1417,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":47.6,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.597,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":45.4,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.5709,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":46.3,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.4356,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":45.76,"balanced_skill":24.85,"breadth_skill":27.18,"public_skill":28.65},"frozen_scores":{"balanced_raw":45.76,"balanced_skill":28.65,"breadth_skill":27.18},"categories":[{"id":"knowledge","raw":0.3702,"skill":0.1658,"combined":0.1427,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4832,"skill":0.3154,"combined":0.2336,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5093,"skill":0.3283,"combined":0.3788,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5051,"skill":0.4497,"combined":0.3835,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4269,"skill":0.1419,"combined":0.1908,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.8235,"skill":0.7617,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7421,"private_eq":0.8104},"2":{"raw":0.5401,"skill":0.4689,"coverage":1,"random":0.1341,"tracks":[],"combined":0.5415,"private_eq":0.6066},"3":{"raw":0.5236,"skill":0.5144,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2032,"private_eq":0.2262},"4":{"raw":0.7157,"skill":0.712,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7071,"private_eq":0.7131},"5":{"raw":0.6608,"skill":0.6588,"coverage":1,"random":0.006,"tracks":[],"combined":0.3945,"private_eq":0.404},"6":{"raw":0.6848,"skill":0.2606,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6501,"headline":false},{"track":"RouterBench-5shot","score":0.7196,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.6468,"skill":0.4892,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4966,"skill":0.246,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3265,"private_eq":0.5548},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5526,"skill":0.1052,"coverage":1,"random":0.5,"tracks":[],"combined":0.278,"private_eq":0.6426},"21":{"raw":0.6043,"skill":0.2085,"coverage":1,"random":0.5,"tracks":[],"combined":0.156,"private_eq":0.5823},"22":{"raw":0.0126,"skill":0.0049,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0129},"23":{"raw":0.5505,"skill":0.101,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.617,"skill":0.4893,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3316,"skill":0.1088,"coverage":1,"random":0.25,"tracks":[],"combined":0.055,"private_eq":0.2986},"28":{"raw":0.674,"skill":0.348,"coverage":1,"random":0.5,"tracks":[],"combined":0.1182,"private_eq":0.5636},"29":{"raw":0.8067,"skill":0.7423,"coverage":1,"random":0.25,"tracks":[],"combined":0.5574,"private_eq":0.6711},"30":{"raw":0.2271,"skill":0.066,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3237,"random":0.25},{"track":"10-choice","raw":0.1304,"random":0.1}],"combined":0.2109,"private_eq":0.3556},"31":{"raw":0.0868,"skill":0.0054,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0892},"32":{"raw":0.5465,"skill":0.279,"coverage":1,"random":0.371,"tracks":[],"combined":0.0493,"private_eq":0.4082},"33":{"raw":0.3127,"skill":0.3036,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2868,"private_eq":0.3032},"36":{"raw":0.1951,"skill":0.0895,"coverage":1,"random":0.116,"tracks":[],"combined":0.1564,"private_eq":0.2619},"37":{"raw":0.3783,"skill":0.2201,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4035,"private_eq":0.529},"38":{"raw":0.0728,"skill":0.0431,"coverage":1,"random":0.031,"tracks":[],"combined":0.0736,"private_eq":0.1116},"39":{"raw":0.747,"skill":0.6279,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4114,"private_eq":0.6037},"40":{"raw":0.1942,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0372,"headline":false},{"track":"A · English","score":0.1942,"headline":true},{"track":"C · Arabic pairs","score":0.33,"headline":false},{"track":"C · English pairs","score":0.51,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4996,"skill":0.2494,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3338},"42":{"raw":0.6757,"skill":0.3691,"coverage":1,"random":0.486,"tracks":[],"combined":0.447,"private_eq":0.7185},"43":{"raw":0.3193,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3238,"private_eq":0.578},"44":{"raw":0.5642,"skill":0.1284,"coverage":1,"random":0.5,"tracks":[],"combined":0.1175,"private_eq":0.5633},"45":{"raw":0.1238,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4057,"skill":0.1375,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1875,"private_eq":0.5979},"57":{"raw":0.3718,"skill":0.2934,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.2556,"private_eq":0.3448},"58":{"raw":0.4868,"skill":0.2561,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0342,"private_eq":0.3406},"59":{"raw":0.1417,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.597,"skill":0.194,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.3996,"private_eq":0.7027},"62":{"raw":0.5709,"skill":0.4279,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.4056,"private_eq":0.5585},"64":{"raw":0.4356,"skill":0.2945,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.264,"private_eq":0.4171}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"TypedDecider.decide one forward per row, bf16, fla chunk_gated_delta_rule + causal-conv1d 1.7.0 CUDA kernel (board used the torch conv fallback), sdpa full attention, torch 2.14.0+cu130","v03":{"engine":"this-that-1.2","pub":28.65,"same_skills":27.51,"new_domains":15.65,"v":24.85,"lab":false,"jev":false,"rank_v":182,"rank_pub":196,"name":"FLock this-that 1.2","eq":{"S":26.78,"O":21.97},"rank_tie":41,"tied":true,"parts":{"public":0,"same_skills":16.07,"new_domains":8.79}}},{"engine":"bosun-v3.1-1.7b","name":"Bosun v3.1 1.7B","long_name":"Bosun v3.1 1.7B LoRA","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":1720574976,"params_estimated":false,"params_from_name":false,"trained_bytes":71879720,"model_url":"https://huggingface.co/Hanno-Labs/bosun-v3.1-1.7b","code_url":"https://github.com/Hanno-Labs/jev-compatible-server","base_model":"Qwen/Qwen3-1.7B-Base","base_url":"https://huggingface.co/Qwen/Qwen3-1.7B-Base","weights_repo":"Hanno-Labs/bosun-v3.1-1.7b","served_checkpoint":null,"org":{"id":"Hanno-Labs","name":"Hanno Labs","logo":"data/orgs/Hanno-Labs.png"}},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.5237,"conf":0.6534,"ece":0.1302,"brier":0.6136,"over95":0.0271,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.052,"conf":0.0413,"acc":0.0455},{"lo":0.1,"w":0.0217,"conf":0.1502,"acc":0.1487},{"lo":0.2,"w":0.024,"conf":0.2561,"acc":0.1817},{"lo":0.3,"w":0.0582,"conf":0.3536,"acc":0.2656},{"lo":0.4,"w":0.0887,"conf":0.4497,"acc":0.3267},{"lo":0.5,"w":0.1502,"conf":0.5499,"acc":0.4454},{"lo":0.6,"w":0.1267,"conf":0.6495,"acc":0.5456},{"lo":0.7,"w":0.1314,"conf":0.7507,"acc":0.6153},{"lo":0.8,"w":0.1399,"conf":0.8501,"acc":0.666},{"lo":0.9,"w":0.2071,"conf":0.9604,"acc":0.7687}],"areas":{"knowledge":{"ece":0.1726,"acc":0.3791,"conf":0.5517},"language":{"ece":0.1211,"acc":0.6198,"conf":0.7409},"retrieval":{"ece":0.1299,"acc":0.7441,"conf":0.8238},"tools":{"ece":0.0725,"acc":0.6822,"conf":0.7122},"arts":{"ece":0.1509,"acc":0.3973,"conf":0.5481}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":40.3,"p95":173.5,"mean":68.1},"results":{"1":{"score":0.6021,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":45},"2":{"score":0.5699,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1117},"3":{"score":0.6083,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8561,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":41.6},"5":{"score":0.9088,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":62.4},"6":{"score":0.6827,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":137.6},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5472,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":40.9},"11":{"score":0.4436,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.4048,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":39.8},"20":{"score":0.5522,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.1},"21":{"score":0.5384,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":39.7},"22":{"score":0.015,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":163.8},"23":{"score":0.5546,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.9},"24":{"score":0.4235,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":39.5},"25":{"score":0.25,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.6654,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":39.4},"27":{"score":0.5358,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":39.6},"28":{"score":0.5178,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":39.6},"29":{"score":0.3488,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":39.8},"30":{"score":0.1888,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.091,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.7},"32":{"score":0.4176,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0394,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":127.2},"36":{"score":0.3289,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4784,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.4},"38":{"score":0.0879,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":416.7},"39":{"score":0.7229,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1538,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":39.9,"tracks":[{"track":"A · Arabic","score":0.0181,"headline":false},{"track":"A · English","score":0.1538,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.445,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3642,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":40},"42":{"score":0.5324,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":40.5},"43":{"score":0.4018,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.532,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.9},"45":{"score":0.1337,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.361,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":41.3},"56":{"score":0.502,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":76.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.2463,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.9,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.3684,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":39.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.2212,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":40.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5363,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":40.4,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.3237,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":40.7,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.3598,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":41.06,"balanced_skill":16.54,"breadth_skill":17.12,"public_skill":20.55},"frozen_scores":{"balanced_raw":41.06,"balanced_skill":20.55,"breadth_skill":17.12},"categories":[{"id":"knowledge","raw":0.2908,"skill":0.0475,"combined":0.03,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3649,"skill":0.1352,"combined":0.1469,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6105,"skill":0.4282,"combined":0.3793,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4345,"skill":0.3478,"combined":0.319,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.394,"skill":0.0896,"combined":0.0851,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.6021,"skill":0.4629,"coverage":1,"random":0.2592,"tracks":[],"combined":0.3811,"private_eq":0.546},"2":{"raw":0.5699,"skill":0.5033,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6399,"private_eq":0.6909},"3":{"raw":0.6083,"skill":0.6008,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5129,"private_eq":0.5266},"4":{"raw":0.8561,"skill":0.8542,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6924,"private_eq":0.6988},"5":{"raw":0.9088,"skill":0.9082,"coverage":1,"random":0.006,"tracks":[],"combined":0.5282,"private_eq":0.5354},"6":{"raw":0.6827,"skill":0.2594,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6344,"headline":false},{"track":"RouterBench-5shot","score":0.7311,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0.005},"11":{"raw":0.4436,"skill":0.1953,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4048,"skill":0.1084,"coverage":1,"random":0.3324,"tracks":[],"combined":0.1116,"private_eq":0.413},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5533,"skill":0.1066,"coverage":1,"random":0.5,"tracks":[],"combined":0.1408,"private_eq":0.5748},"21":{"raw":0.5384,"skill":0.0769,"coverage":1,"random":0.5,"tracks":[],"combined":0.0251,"private_eq":0.5176},"22":{"raw":0.013,"skill":0.0053,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0056,"private_eq":0.0236},"23":{"raw":0.5383,"skill":0.0766,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4371,"skill":0.2494,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.25,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0.0173,"private_eq":0.2706},"28":{"raw":0.5178,"skill":0.0356,"coverage":1,"random":0.5,"tracks":[],"combined":0.0425,"private_eq":0.5262},"29":{"raw":0.3488,"skill":0.1317,"coverage":1,"random":0.25,"tracks":[],"combined":0.2208,"private_eq":0.4215},"30":{"raw":0.1888,"skill":0.0174,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2684,"random":0.25},{"track":"10-choice","raw":0.1092,"random":0.1}],"combined":0.0204,"private_eq":0.2002},"31":{"raw":0.091,"skill":0.01,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0062,"private_eq":0.0971},"32":{"raw":0.4176,"skill":0.0741,"coverage":1,"random":0.371,"tracks":[],"combined":0.0107,"private_eq":0.3842},"33":{"raw":0.0394,"skill":0.0266,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0146,"private_eq":0.0376},"36":{"raw":0.3289,"skill":0.2408,"coverage":1,"random":0.116,"tracks":[],"combined":0.1713,"private_eq":0.2749},"37":{"raw":0.4784,"skill":0.3457,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4162,"private_eq":0.539},"38":{"raw":0.0879,"skill":0.0587,"coverage":1,"random":0.031,"tracks":[],"combined":0.0461,"private_eq":0.0852},"39":{"raw":0.7229,"skill":0.5924,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4431,"private_eq":0.625},"40":{"raw":0.1538,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0181,"headline":false},{"track":"A · English","score":0.1538,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.445,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.3642,"skill":0.0463,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1007,"private_eq":0.4066},"42":{"raw":0.5324,"skill":0.0903,"coverage":1,"random":0.486,"tracks":[],"combined":0.2662,"private_eq":0.6266},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0479,"private_eq":0.4061},"44":{"raw":0.532,"skill":0.064,"coverage":1,"random":0.5,"tracks":[],"combined":0.0372,"private_eq":0.5236},"45":{"raw":0.1337,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.361,"skill":0.0726,"coverage":1,"random":0.311,"tracks":[],"combined":0.1162,"private_eq":0.3973},"56":{"raw":0.502,"skill":0.004,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1703,"private_eq":0.5894},"57":{"raw":0.2463,"skill":0.1523,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.0934,"private_eq":0.2022},"58":{"raw":0.3684,"skill":0.0845,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0028,"private_eq":0.3192},"59":{"raw":0.2212,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5363,"skill":0.0726,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2464,"private_eq":0.627},"62":{"raw":0.3237,"skill":0.0983,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.0154,"private_eq":0.2692},"64":{"raw":0.3598,"skill":0.1997,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.1379,"private_eq":0.3174}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author RegistryRuntime.decide with the row-batched backend (all questions of a row scored together, prompts rendered exactly as model.predict); board used exact per-question mode; bf16, torch 2.14.0+cu130","v03":{"engine":"bosun-v3.1-1.7b","pub":20.55,"same_skills":20.41,"new_domains":7.54,"v":16.54,"lab":false,"jev":false,"rank_v":209,"rank_pub":213,"name":"Bosun v3.1 1.7B LoRA","eq":{"S":19.6,"O":11.95},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":11.76,"new_domains":4.78}}},{"engine":"kodiak-v0.2-1b-accuracy","name":"Kodiak v0.2 1B (accuracy)","long_name":"Kodiak v0.2 1B (accuracy)","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":3000000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/cortex-agent-llc/kodiak-v0.2-1b-accuracy","code_url":null,"base_model":"jhu-clsp/ettin-encoder-1b","base_url":"https://huggingface.co/jhu-clsp/ettin-encoder-1b","weights_repo":null,"served_checkpoint":null,"org":{"id":"cortex-agent-llc","name":"Cortex Agent LLC","logo":"data/orgs/cortex-agent-llc.png"}},"completed":110201,"counts":{"ok":110073,"unsupported":128,"error":0},"lifted":false,"calibration":{"acc":0.5043,"conf":0.5816,"ece":0.0773,"brier":0.606,"over95":0.0079,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0568,"conf":0.0408,"acc":0.0368},{"lo":0.1,"w":0.0474,"conf":0.1388,"acc":0.0645},{"lo":0.2,"w":0.0272,"conf":0.2547,"acc":0.2013},{"lo":0.3,"w":0.0709,"conf":0.3543,"acc":0.3162},{"lo":0.4,"w":0.0979,"conf":0.4514,"acc":0.3829},{"lo":0.5,"w":0.2047,"conf":0.5473,"acc":0.5074},{"lo":0.6,"w":0.1602,"conf":0.6489,"acc":0.5782},{"lo":0.7,"w":0.135,"conf":0.7493,"acc":0.6268},{"lo":0.8,"w":0.1063,"conf":0.848,"acc":0.7379},{"lo":0.9,"w":0.0936,"conf":0.9521,"acc":0.7927}],"areas":{"knowledge":{"ece":0.1391,"acc":0.4149,"conf":0.5512},"language":{"ece":0.0543,"acc":0.6399,"conf":0.6942},"retrieval":{"ece":0.0612,"acc":0.6523,"conf":0.6013},"tools":{"ece":0.167,"acc":0.3543,"conf":0.5087},"arts":{"ece":0.0528,"acc":0.4285,"conf":0.4738}}},"gaps":null,"latency":{"median":142,"p95":null,"mean":236},"results":{"1":{"score":0.5012,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":197.4},"2":{"score":0.42,"jev":null,"metric":"nDCG@10","requests":685,"answered":618,"unsupported":67,"errors":0,"median_ms":3267.2},"3":{"score":0.0551,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2171.9},"4":{"score":0.6307,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":190.7},"5":{"score":0.916,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":186.5},"6":{"score":0.4936,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":447.6},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":368.9},"10":{"score":0.3641,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":286.1},"11":{"score":0.5227,"jev":null,"metric":"macro-F1","requests":123,"answered":121,"unsupported":2,"errors":0,"median_ms":772},"12":{"score":0.4081,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":129},"20":{"score":0.7346,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":117.1},"21":{"score":0.5559,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":107.6},"22":{"score":0.0005,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":1174.2},"23":{"score":0.5208,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":149.6},"24":{"score":0.4531,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":113.4},"25":{"score":0.2704,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":134.6},"26":{"score":0.8068,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":108.8},"27":{"score":0.698,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":112.2},"28":{"score":0.7845,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":112.4},"29":{"score":0.6148,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":137.1},"30":{"score":0.1975,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1164,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":200.6},"32":{"score":0.5878,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":502.6},"33":{"score":0.1176,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":207.9},"36":{"score":0.336,"jev":null,"metric":"nDCG@10","requests":220,"answered":120,"unsupported":100,"errors":0,"median_ms":3241.3},"37":{"score":0.396,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":142.9},"38":{"score":0.0892,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1774.5},"39":{"score":0.359,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":121.4},"40":{"score":0.1891,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":107.9},"41":{"score":0.4387,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":133.9},"42":{"score":0.5668,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":154},"43":{"score":0.3439,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":114.9},"44":{"score":0.5482,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.6},"45":{"score":0.1489,"jev":null,"metric":"accuracy","requests":501,"answered":497,"unsupported":4,"errors":0,"median_ms":139.2},"50":{"score":0.3467,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":227.2},"56":{"score":0.5,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":170.1},"57":{"score":0.2295,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":122},"58":{"score":0.3861,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":117.5},"59":{"score":0.4004,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":207.7},"61":{"score":0.6022,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":150.8},"62":{"score":0.3237,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":200},"64":{"score":0.4148,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":119.5}},"benchmarks":{"1":{"raw":0.5012,"skill":0.3266,"coverage":1,"random":0.2592,"tracks":[],"combined":0.3334,"private_eq":0.5111},"2":{"raw":0.3789,"skill":0.2827,"coverage":0.9022,"random":0.1341,"tracks":[],"combined":0.2808,"private_eq":0.3835},"3":{"raw":0.0551,"skill":0.0369,"coverage":1,"random":0.0189,"tracks":[],"combined":0.0247,"private_eq":0.0531},"4":{"raw":0.6307,"skill":0.6259,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6195,"private_eq":0.6276},"5":{"raw":0.916,"skill":0.9155,"coverage":1,"random":0.006,"tracks":[],"combined":0.4881,"private_eq":0.496},"6":{"raw":0.4936,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.3641,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5068,"skill":0.2867,"coverage":0.9837,"random":0.3085,"tracks":[]},"12":{"raw":0.4081,"skill":0.1134,"coverage":1,"random":0.3324,"tracks":[],"combined":0.1686,"private_eq":0.4506},"20":{"raw":0.7371,"skill":0.4742,"coverage":1,"random":0.5,"tracks":[],"combined":0.3721,"private_eq":0.6891},"21":{"raw":0.5559,"skill":0.1119,"coverage":1,"random":0.5,"tracks":[],"combined":0.0554,"private_eq":0.5326},"22":{"raw":0.0013,"skill":0,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5236,"skill":0.0471,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4531,"skill":0.2708,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2704,"skill":0.0272,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"26":{"raw":0.8068,"skill":0.7423,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.698,"skill":0.5972,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7845,"skill":0.569,"coverage":1,"random":0.5,"tracks":[],"combined":0.6075,"private_eq":0.8055},"29":{"raw":0.6148,"skill":0.4864,"coverage":1,"random":0.25,"tracks":[],"combined":0.5723,"private_eq":0.6821},"30":{"raw":0.1975,"skill":0.0291,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2873,"random":0.25},{"track":"10-choice","raw":0.1077,"random":0.1}],"combined":0.0519,"private_eq":0.2259},"31":{"raw":0.1164,"skill":0.0376,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0084,"private_eq":0.0991},"32":{"raw":0.5878,"skill":0.3447,"coverage":1,"random":0.371,"tracks":[],"combined":0.0339,"private_eq":0.3986},"33":{"raw":0.1176,"skill":0.1059,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.1832,"skill":0.076,"coverage":0.5455,"random":0.116,"tracks":[],"combined":0.1953,"private_eq":0.2959},"37":{"raw":0.396,"skill":0.2424,"coverage":1,"random":0.2027,"tracks":[],"combined":0.2147,"private_eq":0.3802},"38":{"raw":0.0892,"skill":0.0601,"coverage":1,"random":0.031,"tracks":[],"combined":0.0531,"private_eq":0.0919},"39":{"raw":0.359,"skill":0.0572,"coverage":1,"random":0.3201,"tracks":[],"combined":0,"private_eq":0.3201},"40":{"raw":0.1891,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1313,"headline":false},{"track":"A · English","score":0.1891,"headline":true},{"track":"C · Arabic pairs","score":0.465,"headline":false},{"track":"C · English pairs","score":0.505,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4387,"skill":0.158,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5668,"skill":0.1572,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3439,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0394,"private_eq":0.4008},"44":{"raw":0.5482,"skill":0.0964,"coverage":1,"random":0.5,"tracks":[],"combined":0.1066,"private_eq":0.5579},"45":{"raw":0.1477,"skill":0,"coverage":0.992,"random":0.1641,"tracks":[]},"50":{"raw":0.3467,"skill":0.0518,"coverage":1,"random":0.311,"tracks":[],"combined":0.2322,"private_eq":0.4763},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.2295,"skill":0.1334,"coverage":1,"random":0.1109,"tracks":[],"combined":0.0344,"private_eq":0.1504},"58":{"raw":0.3861,"skill":0.1102,"coverage":1,"random":0.3101,"tracks":[],"combined":0.0194,"private_eq":0.3305},"59":{"raw":0.4004,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6022,"skill":0.2044,"coverage":1,"random":0.5,"tracks":[],"combined":0.2068,"private_eq":0.6074},"62":{"raw":0.3237,"skill":0.0983,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"64":{"raw":0.4148,"skill":0.2685,"coverage":1,"random":0.2,"tracks":[],"combined":0.2154,"private_eq":0.3787}},"coverage":0.9794,"pending":0.0206,"scores":{"balanced_raw":38.98,"balanced_skill":13.86,"breadth_skill":18.07,"public_skill":19.05},"frozen_scores":{"balanced_raw":38.98,"balanced_skill":19.05,"breadth_skill":18.07},"categories":[{"id":"knowledge","raw":0.3411,"skill":0.1351,"combined":0.0936,"coverage":0.9992,"pending":0,"n":11},{"id":"language","raw":0.4019,"skill":0.1531,"combined":0.1059,"coverage":0.9983,"pending":0,"n":9},{"id":"retrieval","raw":0.5415,"skill":0.3618,"combined":0.2991,"coverage":0.9174,"pending":0,"n":6},{"id":"tools","raw":0.2537,"skill":0.1513,"combined":0.1277,"coverage":0.9819,"pending":0,"n":5},{"id":"arts","raw":0.4299,"skill":0.1589,"combined":0.1729,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"kodiak-v0.2-1b-accuracy","pub":19.05,"same_skills":16.33,"new_domains":7.13,"v":13.86,"lab":false,"jev":false,"rank_v":219,"rank_pub":219,"name":"Kodiak v0.2 1B (accuracy)","eq":{"S":15.47,"O":11.45},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":9.28,"new_domains":4.58}}},{"engine":"decision-2.0-kai-0.6b","name":"vLLM-SR Decision 2.0 Kai 0.6B","long_name":"vLLM-SR Decision 2.0 Kai 0.6B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":600000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/Decision-2.0-Kai-0.6B","code_url":null,"base_model":"Qwen/Qwen3-0.6B-Base","base_url":"https://huggingface.co/Qwen/Qwen3-0.6B-Base","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110188,"unsupported":13,"error":0},"lifted":false,"calibration":{"acc":0.4676,"conf":0.4815,"ece":0.0376,"brier":0.615,"over95":0,"n":34831,"benchmarks":32,"rel":[{"lo":0,"w":0.0708,"conf":0.0395,"acc":0.0532},{"lo":0.1,"w":0.0426,"conf":0.1415,"acc":0.2447},{"lo":0.2,"w":0.0782,"conf":0.267,"acc":0.3042},{"lo":0.3,"w":0.1618,"conf":0.3514,"acc":0.339},{"lo":0.4,"w":0.1211,"conf":0.4451,"acc":0.4704},{"lo":0.5,"w":0.2185,"conf":0.5473,"acc":0.5396},{"lo":0.6,"w":0.1559,"conf":0.645,"acc":0.5794},{"lo":0.7,"w":0.085,"conf":0.7467,"acc":0.649},{"lo":0.8,"w":0.0489,"conf":0.8459,"acc":0.7733},{"lo":0.9,"w":0.0173,"conf":0.9341,"acc":0.961}],"areas":{"knowledge":{"ece":0.0423,"acc":0.4133,"conf":0.412},"language":{"ece":0.0564,"acc":0.5133,"conf":0.5571},"retrieval":{"ece":0.0902,"acc":0.6097,"conf":0.5197},"tools":{"ece":0.1715,"acc":0.4885,"conf":0.5168},"arts":{"ece":0.0743,"acc":0.3781,"conf":0.4468}}},"gaps":null,"latency":{"median":18.5,"p95":null,"mean":37},"results":{"1":{"score":0.4156,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":22.2},"2":{"score":0.6184,"jev":null,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":154.6},"3":{"score":0.3878,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":99.2},"4":{"score":0.7156,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":27.3},"5":{"score":0.6518,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":43.3},"6":{"score":0.7121,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":52.4},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":142.1},"10":{"score":0.5896,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":22},"11":{"score":0.3099,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":342.1},"12":{"score":0.3054,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":17.8},"20":{"score":0.492,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18},"21":{"score":0.4947,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":17.6},"22":{"score":0.0232,"jev":null,"metric":"accuracy","requests":2000,"answered":1985,"unsupported":15,"errors":0,"median_ms":96.1},"23":{"score":0.5168,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.8},"24":{"score":0.4097,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.2},"25":{"score":0.2551,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":17.7},"26":{"score":0.7845,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":17.6},"27":{"score":0.5316,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":16.7},"28":{"score":0.5746,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":17.3},"29":{"score":0.4454,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":17.5},"30":{"score":0.2521,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0708,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20},"32":{"score":0.4481,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":20.2},"33":{"score":0.137,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":35.6},"36":{"score":0.3427,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":333.2},"37":{"score":0.2825,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.4},"38":{"score":0.0486,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":71.9},"39":{"score":0.6798,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":17.4},"40":{"score":0.1632,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":17.3},"41":{"score":0.3422,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":17},"42":{"score":0.5949,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":17.5},"43":{"score":0.3316,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":17.4},"44":{"score":0.5102,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.2},"45":{"score":0.12,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":17.4},"50":{"score":0.3795,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":19.6},"56":{"score":0.5,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":26.4},"57":{"score":0.1845,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":17.3},"58":{"score":0.3793,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":17.4},"59":{"score":0.01,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":19.3},"61":{"score":0.534,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":18.5},"62":{"score":0.3861,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":18.9},"64":{"score":0.4545,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":17.8}},"benchmarks":{"1":{"raw":0.4156,"skill":0.2111,"coverage":1,"random":0.2592,"tracks":[],"combined":0.2177,"private_eq":0.4263},"2":{"raw":0.6165,"skill":0.5572,"coverage":0.9971,"random":0.1341,"tracks":[],"combined":0.6474,"private_eq":0.6973},"3":{"raw":0.3878,"skill":0.376,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2399,"private_eq":0.2618},"4":{"raw":0.7156,"skill":0.7119,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6416,"private_eq":0.6492},"5":{"raw":0.6518,"skill":0.6497,"coverage":1,"random":0.006,"tracks":[],"combined":0.417,"private_eq":0.4261},"6":{"raw":0.7121,"skill":0.325,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.5896,"skill":0.3171,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.3099,"skill":0.0019,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3054,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.4915,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5033},"21":{"raw":0.4947,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0231,"skill":0.0155,"coverage":0.9925,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5067,"skill":0.0135,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4097,"skill":0.2129,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2551,"skill":0.0068,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"26":{"raw":0.7845,"skill":0.7126,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5316,"skill":0.3753,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5746,"skill":0.1492,"coverage":1,"random":0.5,"tracks":[],"combined":0.1384,"private_eq":0.5736},"29":{"raw":0.4454,"skill":0.2605,"coverage":1,"random":0.25,"tracks":[],"combined":0.1613,"private_eq":0.3774},"30":{"raw":0.2521,"skill":0.0969,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.351,"random":0.25},{"track":"10-choice","raw":0.1531,"random":0.1}],"combined":0.2446,"private_eq":0.3831},"31":{"raw":0.0708,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.4481,"skill":0.1226,"coverage":1,"random":0.371,"tracks":[],"combined":0.0262,"private_eq":0.3938},"33":{"raw":0.137,"skill":0.1255,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0864,"private_eq":0.1077},"36":{"raw":0.3427,"skill":0.2565,"coverage":1,"random":0.116,"tracks":[],"combined":0.2178,"private_eq":0.3155},"37":{"raw":0.2825,"skill":0.1001,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1578,"private_eq":0.3354},"38":{"raw":0.0486,"skill":0.0182,"coverage":1,"random":0.031,"tracks":[],"combined":0.0231,"private_eq":0.0632},"39":{"raw":0.6798,"skill":0.529,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3733,"private_eq":0.5781},"40":{"raw":0.1632,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1811,"headline":false},{"track":"A · English","score":0.1632,"headline":true},{"track":"C · Arabic pairs","score":0.47,"headline":false},{"track":"C · English pairs","score":0.29,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.3422,"skill":0.0133,"coverage":1,"random":0.3333,"tracks":[],"combined":0.422,"private_eq":0.6184},"42":{"raw":0.5949,"skill":0.2119,"coverage":1,"random":0.486,"tracks":[],"combined":0.1198,"private_eq":0.5522},"43":{"raw":0.3316,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0813,"private_eq":0.4269},"44":{"raw":0.5102,"skill":0.0204,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5051},"45":{"raw":0.1198,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.3795,"skill":0.0994,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1845,"skill":0.0828,"coverage":1,"random":0.1109,"tracks":[],"combined":0.0577,"private_eq":0.1709},"58":{"raw":0.3793,"skill":0.1003,"coverage":1,"random":0.3101,"tracks":[],"combined":0,"private_eq":0.3169},"59":{"raw":0.01,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.534,"skill":0.068,"coverage":1,"random":0.5,"tracks":[],"combined":0.0876,"private_eq":0.5485},"62":{"raw":0.3861,"skill":0.1815,"coverage":1,"random":0.25,"tracks":[],"combined":0.1318,"private_eq":0.3555},"64":{"raw":0.4545,"skill":0.3181,"coverage":1,"random":0.2,"tracks":[],"combined":0.2252,"private_eq":0.3864}},"coverage":0.9996,"pending":0.0004,"scores":{"balanced_raw":36.86,"balanced_skill":16.43,"breadth_skill":14.86,"public_skill":16.68},"frozen_scores":{"balanced_raw":36.86,"balanced_skill":16.68,"breadth_skill":14.86},"categories":[{"id":"knowledge","raw":0.2971,"skill":0.0644,"combined":0.0602,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.3244,"skill":0.1156,"combined":0.1307,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5104,"skill":0.3197,"combined":0.2676,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.3642,"skill":0.2673,"combined":0.244,"coverage":0.9995,"pending":0,"n":5},{"id":"arts","raw":0.3917,"skill":0.0744,"combined":0.0821,"coverage":0.9987,"pending":0,"n":6}],"v03":{"engine":"decision-2.0-kai-0.6b","pub":16.68,"same_skills":16.9,"new_domains":11.63,"v":16.43,"lab":false,"jev":false,"rank_v":210,"rank_pub":223,"name":"vLLM-SR Decision 2.0 Kai 0.6B","eq":{"S":16.05,"O":17.01},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":9.63,"new_domains":6.81}}},{"engine":"decision-2.0-eos-0.8b","name":"vLLM-SR Decision 2.0 Eos 0.8B","long_name":"vLLM-SR Decision 2.0 Eos 0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":870000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/Decision-2.0-Eos-0.8B","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.5015,"conf":0.6161,"ece":0.1295,"brier":0.6429,"over95":0.0282,"n":34833,"benchmarks":32,"rel":[{"lo":0,"w":0.0568,"conf":0.042,"acc":0.0724},{"lo":0.1,"w":0.0434,"conf":0.1504,"acc":0.2139},{"lo":0.2,"w":0.0372,"conf":0.2558,"acc":0.3199},{"lo":0.3,"w":0.079,"conf":0.3517,"acc":0.359},{"lo":0.4,"w":0.0983,"conf":0.4492,"acc":0.3758},{"lo":0.5,"w":0.1423,"conf":0.548,"acc":0.4879},{"lo":0.6,"w":0.1182,"conf":0.6495,"acc":0.5402},{"lo":0.7,"w":0.1187,"conf":0.7503,"acc":0.5533},{"lo":0.8,"w":0.1135,"conf":0.8491,"acc":0.6121},{"lo":0.9,"w":0.1926,"conf":0.9637,"acc":0.7399}],"areas":{"knowledge":{"ece":0.1015,"acc":0.4258,"conf":0.5273},"language":{"ece":0.2001,"acc":0.5359,"conf":0.7356},"retrieval":{"ece":0.1308,"acc":0.6546,"conf":0.6217},"tools":{"ece":0.1274,"acc":0.6416,"conf":0.6975},"arts":{"ece":0.1695,"acc":0.3998,"conf":0.5431}}},"gaps":null,"latency":{"median":25.1,"p95":null,"mean":50.2},"results":{"1":{"score":0.5791,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":25.1},"2":{"score":0.5966,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":132.8},"3":{"score":0.4961,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":63},"4":{"score":0.7955,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":24.5},"5":{"score":0.6309,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":34.9},"6":{"score":0.7654,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":37.9},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":117.1},"10":{"score":0.6088,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":25.4},"11":{"score":0.4448,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":249.7},"12":{"score":0.3653,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":23.5},"20":{"score":0.5148,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.5},"21":{"score":0.5453,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":23},"22":{"score":0.079,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":62.1},"23":{"score":0.5116,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.1},"24":{"score":0.3984,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":23.5},"25":{"score":0.2551,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":22.4},"26":{"score":0.7803,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":24.2},"27":{"score":0.57,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":25},"28":{"score":0.5178,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":21.8},"29":{"score":0.4373,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":22.2},"30":{"score":0.207,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0854,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.4},"32":{"score":0.5,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":23.4},"33":{"score":0.1079,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":32.7},"36":{"score":0.2728,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":256.6},"37":{"score":0.3503,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":25},"38":{"score":0.0715,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":80.6},"39":{"score":0.5976,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":25.7},"40":{"score":0.2121,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":25.3},"41":{"score":0.4258,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":24.4},"42":{"score":0.6364,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":25.9},"43":{"score":0.407,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":25.5},"44":{"score":0.5454,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.6},"45":{"score":0.1198,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":25.2},"50":{"score":0.3634,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":27.7},"56":{"score":0.5005,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":32.6},"57":{"score":0.1825,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":25.4},"58":{"score":0.4265,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":24.5},"59":{"score":0.0453,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":28.6},"61":{"score":0.7153,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":27.2},"62":{"score":0.4017,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":26.8},"64":{"score":0.3996,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":25.4}},"benchmarks":{"1":{"raw":0.5791,"skill":0.4318,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.5061,"combined":0.3266},"2":{"raw":0.5958,"skill":0.5332,"coverage":0.9985,"random":0.1341,"tracks":[],"private_eq":0.7051,"combined":0.6565},"3":{"raw":0.4961,"skill":0.4864,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.3433,"combined":0.324},"4":{"raw":0.7955,"skill":0.7929,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.6687,"combined":0.6616},"5":{"raw":0.6309,"skill":0.6287,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.3311,"combined":0.3203},"6":{"raw":0.7654,"skill":0.4499,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0.005,"combined":0},"10":{"raw":0.6088,"skill":0.3491,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.4448,"skill":0.1971,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3653,"skill":0.0493,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.4301,"combined":0.1375},"20":{"raw":0.5097,"skill":0.0195,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5426,"combined":0.0757},"21":{"raw":0.5453,"skill":0.0906,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5077,"combined":0.0051},"22":{"raw":0.0767,"skill":0.0695,"coverage":0.9995,"random":0.0078,"tracks":[],"private_eq":0.105,"combined":0.0886},"23":{"raw":0.514,"skill":0.0279,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3984,"skill":0.1979,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2551,"skill":0.0068,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2737,"combined":0.0215},"26":{"raw":0.7803,"skill":0.707,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.57,"skill":0.4265,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5178,"skill":0.0356,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.606,"combined":0.2039},"29":{"raw":0.4373,"skill":0.2497,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4545,"combined":0.2653},"30":{"raw":0.207,"skill":0.039,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2813,"random":0.25},{"track":"10-choice","raw":0.1327,"random":0.1}],"private_eq":0.3212,"combined":0.1687},"31":{"raw":0.0854,"skill":0.0039,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0824,"combined":0},"32":{"raw":0.5,"skill":0.2051,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.4131,"combined":0.0572},"33":{"raw":0.1079,"skill":0.0961,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.1779,"combined":0.1584},"36":{"raw":0.2728,"skill":0.1773,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.2746,"combined":0.171},"37":{"raw":0.3503,"skill":0.1851,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.5003,"combined":0.3671},"38":{"raw":0.0715,"skill":0.0418,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.1163,"combined":0.0785},"39":{"raw":0.5976,"skill":0.4081,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6334,"combined":0.4556},"40":{"raw":0.2121,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2893,"headline":false},{"track":"A · English","score":0.2121,"headline":true},{"track":"C · Arabic pairs","score":0.805,"headline":false},{"track":"C · English pairs","score":0.74,"headline":false}],"private_eq":0.2227,"combined":0},"41":{"raw":0.4258,"skill":0.1387,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.5524,"combined":0.3219},"42":{"raw":0.6364,"skill":0.2926,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.6606,"combined":0.3331},"43":{"raw":0.407,"skill":0.0592,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.3878,"combined":0.0185},"44":{"raw":0.5454,"skill":0.0908,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5553,"combined":0.1014},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3634,"skill":0.076,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.3973,"combined":0.1162},"56":{"raw":0.5005,"skill":0.001,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5295,"combined":0.0492},"57":{"raw":0.1825,"skill":0.0805,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1661,"combined":0.0523},"58":{"raw":0.4265,"skill":0.1687,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3118,"combined":0},"59":{"raw":0.0453,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.7153,"skill":0.4306,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6943,"combined":0.3826},"62":{"raw":0.4017,"skill":0.2023,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3656,"combined":0.1454},"64":{"raw":0.3996,"skill":0.2495,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.3634,"combined":0.1961}},"coverage":0.9999,"pending":0.0001,"scores":{"balanced_raw":39.91,"balanced_skill":18.37,"breadth_skill":18.04,"public_skill":20.57},"frozen_scores":{"balanced_raw":39.91,"balanced_skill":20.57,"breadth_skill":18.04},"categories":[{"id":"knowledge","raw":0.3046,"skill":0.0703,"combined":0.0757,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3613,"skill":0.1529,"combined":0.1966,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5462,"skill":0.3841,"combined":0.3306,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4237,"skill":0.3402,"combined":0.2921,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.4014,"skill":0.0888,"combined":0.0963,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"decision-2.0-eos-0.8b","pub":20.57,"same_skills":21.31,"new_domains":10.16,"v":18.37,"lab":false,"jev":false,"rank_v":200,"rank_pub":212,"name":"vLLM-SR Decision 2.0 Eos 0.8B","eq":{"S":20.5,"O":15.19},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":12.3,"new_domains":6.07}}},{"engine":"tev1-0.8b","name":"Tev1-0.8B-experimental","long_name":"Together Tev1 Qwen3.5-0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1746942600,"model_url":"https://huggingface.co/togethercomputer/Tev1-0.8B-experimental","code_url":"https://github.com/togethercomputer/tev1","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"togethercomputer/Tev1-0.8B-experimental","served_checkpoint":null,"org":{"id":"togethercomputer","name":"Together","logo":"data/orgs/togethercomputer.png"}},"completed":95017,"counts":{"ok":95017,"unsupported":14732},"lifted":false,"calibration":{"acc":0.5022,"conf":0.6279,"ece":0.1257,"brier":0.6067,"over95":0.012,"n":32652,"benchmarks":28,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0025,"conf":0.1873,"acc":0.0958},{"lo":0.2,"w":0.0409,"conf":0.2605,"acc":0.1139},{"lo":0.3,"w":0.0888,"conf":0.3533,"acc":0.269},{"lo":0.4,"w":0.1141,"conf":0.4501,"acc":0.354},{"lo":0.5,"w":0.2462,"conf":0.5522,"acc":0.463},{"lo":0.6,"w":0.1575,"conf":0.6467,"acc":0.525},{"lo":0.7,"w":0.1341,"conf":0.7483,"acc":0.6048},{"lo":0.8,"w":0.0969,"conf":0.8523,"acc":0.6472},{"lo":0.9,"w":0.1189,"conf":0.9528,"acc":0.7778}],"areas":{"knowledge":{"ece":0.1353,"acc":0.4149,"conf":0.5502},"language":{"ece":0.1351,"acc":0.5196,"conf":0.6547},"retrieval":{"ece":0.1907,"acc":0.6214,"conf":0.8078},"tools":{"ece":0.0341,"acc":0.7799,"conf":0.7678},"arts":{"ece":0.1187,"acc":0.5044,"conf":0.6213}}},"gaps":{"share":0.1229,"reasons":[{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 151","share":0.0459},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 77","share":0.0257},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'chord' has 129","share":0.0167}]},"latency":{"median":28.5,"p95":133.9,"mean":50.1},"results":{"1":{"score":0.6612,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":63.9},"2":{"score":0.5723,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":195.4},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.5868,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":131.6},"9":{"score":0.0114,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5968,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":61.5},"11":{"score":0.4607,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3761,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":27.6},"20":{"score":0.5072,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.4},"21":{"score":0.5126,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":27.3},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.537,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":59.3},"24":{"score":0.4591,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":27.6},"25":{"score":0.2755,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8199,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":27.4},"27":{"score":0.6195,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.5},"28":{"score":0.5225,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":27.4},"29":{"score":0.4481,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":28},"30":{"score":0.1994,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.195,"jev":null,"metric":"accuracy","requests":5000,"answered":1359,"unsupported":3641,"errors":0,"median_ms":59.2},"32":{"score":0.5199,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0582,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":78.9},"36":{"score":0.2936,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2181,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.9},"38":{"score":0.0538,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":171},"39":{"score":0.5778,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2126,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":27.7,"tracks":[{"track":"A · Arabic","score":0.2317,"headline":false},{"track":"A · English","score":0.2126,"headline":true},{"track":"C · Arabic pairs","score":0.505,"headline":false},{"track":"C · English pairs","score":0.465,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4464,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":27.8},"42":{"score":0.613,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":59.9},"43":{"score":0.3807,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5052,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.5},"45":{"score":0.1486,"jev":null,"metric":"accuracy","requests":501,"answered":498,"unsupported":3,"errors":0,"median_ms":null},"50":{"score":0.3998,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":61.2},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":70.5,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.2288,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":28,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.4362,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":27.9,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.1825,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":60.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5308,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":31.9,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.6681,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":60.5,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.3068,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":33.12,"balanced_skill":12.47,"breadth_skill":11.48,"public_skill":13.1},"frozen_scores":{"balanced_raw":33.12,"balanced_skill":13.1,"breadth_skill":11.48},"categories":[{"id":"knowledge","raw":0.3041,"skill":0.0678,"combined":0.0444,"coverage":0.9387,"pending":0,"n":11},{"id":"language","raw":0.3762,"skill":0.1526,"combined":0.1438,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2426,"skill":0.0488,"combined":0.0341,"coverage":0.6364,"pending":0,"n":6},{"id":"tools","raw":0.3787,"skill":0.3197,"combined":0.3057,"coverage":0.7778,"pending":0,"n":5},{"id":"arts","raw":0.3753,"skill":0.0587,"combined":0.0569,"coverage":0.8333,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.6612,"skill":0.5426,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.6858,"combined":0.572},"2":{"raw":0.5723,"skill":0.5061,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.6449,"combined":0.5862},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"private_eq":0.0189,"combined":0},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"private_eq":0.0127,"combined":0},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"private_eq":0.006,"combined":0},"6":{"raw":0.5869,"skill":0.0362,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5589,"headline":false},{"track":"RouterBench-5shot","score":0.6148,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0.0114,"skill":0.0114,"coverage":1,"random":0,"tracks":[],"private_eq":0.0025,"combined":0},"11":{"raw":0.4607,"skill":0.2201,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3761,"skill":0.0655,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.4685,"combined":0.1957},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5108,"skill":0.0215,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5354,"combined":0.0611},"21":{"raw":0.5126,"skill":0.0251,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"private_eq":0.0078,"combined":0},"23":{"raw":0.5216,"skill":0.0431,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4774,"skill":0.3032,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2924,"combined":0.0467},"28":{"raw":0.5225,"skill":0.045,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5512,"combined":0.0931},"29":{"raw":0.4481,"skill":0.2641,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4839,"combined":0.305},"30":{"raw":0.1994,"skill":0.0299,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2752,"random":0.25},{"track":"10-choice","raw":0.1236,"random":0.1}],"private_eq":0.2946,"combined":0.1361},"31":{"raw":0.053,"skill":0,"coverage":0.2718,"random":0.0819,"tracks":[],"private_eq":0.0819,"combined":0},"32":{"raw":0.5199,"skill":0.2367,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0.0582,"skill":0.0457,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0877,"combined":0.0659},"36":{"raw":0.2936,"skill":0.2009,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.2881,"combined":0.1864},"37":{"raw":0.2181,"skill":0.0192,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2027,"combined":0},"38":{"raw":0.0538,"skill":0.0235,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.0816,"combined":0.0423},"39":{"raw":0.5778,"skill":0.379,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.4975,"combined":0.2534},"40":{"raw":0.2126,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2317,"headline":false},{"track":"A · English","score":0.2126,"headline":true},{"track":"C · Arabic pairs","score":0.505,"headline":false},{"track":"C · English pairs","score":0.465,"headline":false}],"private_eq":0.2227,"combined":0},"41":{"raw":0.4464,"skill":0.1696,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.451,"combined":0.1681},"42":{"raw":0.613,"skill":0.2471,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.575,"combined":0.1646},"43":{"raw":0.3807,"skill":0.0174,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.4608,"combined":0.1357},"44":{"raw":0.5052,"skill":0.0104,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"45":{"raw":0.1477,"skill":0,"coverage":0.994,"random":0.1641,"tracks":[]},"50":{"raw":0.3998,"skill":0.1289,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.417,"combined":0.1451},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","private_eq":0.5,"combined":0},"57":{"raw":0.2288,"skill":0.1326,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","private_eq":0.124,"combined":0.0044},"58":{"raw":0.4362,"skill":0.1828,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","private_eq":0.3177,"combined":0.0006},"59":{"raw":0.1825,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","private_eq":0.5177,"combined":0},"61":{"raw":0.5308,"skill":0.0616,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","private_eq":0.5288,"combined":0.0478},"62":{"raw":0.6681,"skill":0.5575,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","private_eq":0.5534,"combined":0.3987},"64":{"raw":0.3068,"skill":0.1335,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","private_eq":0.2867,"combined":0.0991}},"coverage":0.6909,"pending":0.24,"latency_rows":677,"latency_path":"in-process vLLM 0.29.0 LLM.generate with all questions of the row in one call, CUDA graphs, gpu util 0.9 (single process)","v03":{"engine":"tev1-0.8b","pub":13.1,"same_skills":12.84,"new_domains":8.6,"v":12.47,"lab":false,"jev":false,"rank_v":222,"rank_pub":227,"name":"Together Tev1 Qwen3.5-0.8B","eq":{"S":11.94,"O":13.27},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":7.16,"new_domains":5.31}}},{"engine":"intern-decision-0.8b","name":"Intern-Decision-0.8B","long_name":"InternLM Intern-Decision Qwen3.5-0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":1706034736,"model_url":"https://huggingface.co/internlm/Intern-Decision-0.8B","code_url":"https://github.com/internlm/Intern-Decision","base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":"internlm/Intern-Decision-0.8B","served_checkpoint":null,"org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"}},"completed":109749,"counts":{"ok":109749},"lifted":false,"calibration":{"acc":0.4516,"conf":0.4473,"ece":0.0255,"brier":0.6309,"over95":0,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0489,"conf":0.0817,"acc":0.0267},{"lo":0.1,"w":0.0995,"conf":0.1444,"acc":0.0845},{"lo":0.2,"w":0.0566,"conf":0.2589,"acc":0.2427},{"lo":0.3,"w":0.1324,"conf":0.3555,"acc":0.3671},{"lo":0.4,"w":0.1541,"conf":0.4474,"acc":0.4403},{"lo":0.5,"w":0.3364,"conf":0.5396,"acc":0.5602},{"lo":0.6,"w":0.1191,"conf":0.6397,"acc":0.6686},{"lo":0.7,"w":0.0387,"conf":0.7381,"acc":0.7769},{"lo":0.8,"w":0.0132,"conf":0.8369,"acc":0.9399},{"lo":0.9,"w":0.0012,"conf":0.9138,"acc":1}],"areas":{"knowledge":{"ece":0.0262,"acc":0.4155,"conf":0.4195},"language":{"ece":0.0463,"acc":0.543,"conf":0.5182},"retrieval":{"ece":0.0943,"acc":0.3738,"conf":0.3365},"tools":{"ece":0.0559,"acc":0.4619,"conf":0.4587},"arts":{"ece":0.0562,"acc":0.4224,"conf":0.463}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":33.7,"p95":39.2,"mean":36.6},"results":{"1":{"score":0.6446,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":35.4},"2":{"score":0.4742,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":83.9},"3":{"score":0.2106,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.1185,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.6},"5":{"score":0.049,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38},"6":{"score":0.566,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":37.9},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5742,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":35.9},"11":{"score":0.3545,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3409,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":33.2},"20":{"score":0.5036,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.2},"21":{"score":0.5369,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.8},"22":{"score":0.0155,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":71},"23":{"score":0.5674,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.2},"24":{"score":0.469,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":33},"25":{"score":0.2806,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8123,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33},"27":{"score":0.6288,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":32.9},"28":{"score":0.5312,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":32.7},"29":{"score":0.4356,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":33.2},"30":{"score":0.1857,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.088,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.1},"32":{"score":0.5133,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0339,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":36.1},"36":{"score":0.2237,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2436,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.5},"38":{"score":0.0464,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":82.8},"39":{"score":0.5708,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2628,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":32.8,"tracks":[{"track":"A · Arabic","score":0.2461,"headline":false},{"track":"A · English","score":0.2628,"headline":true},{"track":"C · Arabic pairs","score":0.545,"headline":false},{"track":"C · English pairs","score":0.615,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2798,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":33.3},"42":{"score":0.5765,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":34.2},"43":{"score":0.3281,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.505,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.1},"45":{"score":0.1357,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4463,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":35.1},"56":{"score":0.4585,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":35.6,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.2654,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":33.3,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.4267,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33.2,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5326,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.8,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5235,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":34.1,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.4608,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":35.5,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.2708,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":33.22,"balanced_skill":12.71,"breadth_skill":10.72,"public_skill":11.81},"frozen_scores":{"balanced_raw":33.22,"balanced_skill":11.81,"breadth_skill":10.72},"categories":[{"id":"knowledge","raw":0.3018,"skill":0.0677,"combined":0.0654,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3782,"skill":0.1087,"combined":0.0843,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2568,"skill":0.0644,"combined":0.0849,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.3632,"skill":0.2838,"combined":0.2922,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3862,"skill":0.0772,"combined":0.0815,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.6446,"skill":0.5203,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.571,"combined":0.4152},"2":{"raw":0.4742,"skill":0.3927,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.6493,"combined":0.5913},"3":{"raw":0.2106,"skill":0.1954,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.2262,"combined":0.2032},"4":{"raw":0.1185,"skill":0.1072,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.1315,"combined":0.1112},"5":{"raw":0.049,"skill":0.0433,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.0723,"combined":0.057},"6":{"raw":0.566,"skill":0.0174,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5411,"headline":false},{"track":"RouterBench-5shot","score":0.5909,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0,"combined":0},"11":{"raw":0.3545,"skill":0.0665,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3409,"skill":0.0127,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.3324,"combined":0},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5026,"skill":0.0052,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5783,"combined":0.1479},"21":{"raw":0.5369,"skill":0.0738,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5326,"combined":0.0554},"22":{"raw":0.0148,"skill":0.0071,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.036,"combined":0.0182},"23":{"raw":0.5461,"skill":0.0923,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4827,"skill":0.3103,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2806,"skill":0.0408,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2924,"combined":0.0467},"28":{"raw":0.5312,"skill":0.0624,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"29":{"raw":0.4356,"skill":0.2475,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4325,"combined":0.2356},"30":{"raw":0.1857,"skill":0.0131,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2608,"random":0.25},{"track":"10-choice","raw":0.1107,"random":0.1}],"private_eq":0.2648,"combined":0.0996},"31":{"raw":0.088,"skill":0.0067,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.107,"combined":0.0171},"32":{"raw":0.5133,"skill":0.2262,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0.0339,"skill":0.0211,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0325,"combined":0.0094},"36":{"raw":0.2237,"skill":0.1218,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.2715,"combined":0.1674},"37":{"raw":0.2436,"skill":0.0513,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2031,"combined":0},"38":{"raw":0.0464,"skill":0.0159,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.0723,"combined":0.0326},"39":{"raw":0.5708,"skill":0.3687,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.5638,"combined":0.3521},"40":{"raw":0.2628,"skill":0.0516,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2461,"headline":false},{"track":"A · English","score":0.2628,"headline":true},{"track":"C · Arabic pairs","score":0.545,"headline":false},{"track":"C · English pairs","score":0.615,"headline":false}],"private_eq":0.2414,"combined":0.0138},"41":{"raw":0.2798,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3333,"combined":0},"42":{"raw":0.5765,"skill":0.1761,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.5117,"combined":0.0401},"43":{"raw":0.3281,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.5154,"combined":0.2233},"44":{"raw":0.505,"skill":0.01,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"45":{"raw":0.1357,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4463,"skill":0.1964,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.3775,"combined":0.0871},"56":{"raw":0.4585,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","private_eq":0.5252,"combined":0.0405},"57":{"raw":0.2654,"skill":0.1738,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","private_eq":0.3233,"combined":0.2311},"58":{"raw":0.4267,"skill":0.169,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","private_eq":0.3499,"combined":0.0478},"59":{"raw":0.5326,"skill":0.0309,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","private_eq":0.5273,"combined":0.0096},"61":{"raw":0.5235,"skill":0.047,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","private_eq":0.5681,"combined":0.1273},"62":{"raw":0.4608,"skill":0.2811,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","private_eq":0.4468,"combined":0.2549},"64":{"raw":0.2708,"skill":0.0885,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","private_eq":0.2867,"combined":0.0991}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author DecisionEngine.predict on the whole row, SDPA, max_length 262144 and the board lifts; transformers fast path enabled with causal-conv1d 1.7.0 (fla 0.4.2 gated delta + CUDA conv). The board ran the torch fallback because causal-conv1d was absent; that run is kept as latency-v1-full.torch-fallback. venv-intern torch 2.9.1+cu128","v03":{"engine":"intern-decision-0.8b","pub":11.81,"same_skills":12.93,"new_domains":8.97,"v":12.71,"lab":false,"jev":false,"rank_v":220,"rank_pub":229,"name":"InternLM Intern-Decision Qwen3.5-0.8B","eq":{"S":12.03,"O":13.73},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":7.22,"new_domains":5.49}}},{"engine":"kev-0.8b-r15","name":"Kev 0.8B r15","long_name":"Kev 0.8B r15","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":870000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/jaredpalmer/kev-0.8b","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5047,"conf":0.5188,"ece":0.0402,"brier":0.6042,"over95":0.001,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0609,"conf":0.0401,"acc":0.0473},{"lo":0.1,"w":0.0627,"conf":0.149,"acc":0.1964},{"lo":0.2,"w":0.0558,"conf":0.2622,"acc":0.3237},{"lo":0.3,"w":0.1116,"conf":0.3507,"acc":0.3491},{"lo":0.4,"w":0.1003,"conf":0.4469,"acc":0.5048},{"lo":0.5,"w":0.2129,"conf":0.5479,"acc":0.5497},{"lo":0.6,"w":0.1746,"conf":0.6473,"acc":0.5935},{"lo":0.7,"w":0.1156,"conf":0.7476,"acc":0.6735},{"lo":0.8,"w":0.0715,"conf":0.8485,"acc":0.7727},{"lo":0.9,"w":0.0341,"conf":0.9308,"acc":0.8236}],"areas":{"knowledge":{"ece":0.0347,"acc":0.4517,"conf":0.4593},"language":{"ece":0.0769,"acc":0.5573,"conf":0.6277},"retrieval":{"ece":0.1144,"acc":0.5609,"conf":0.4499},"tools":{"ece":0.0894,"acc":0.67,"conf":0.5904},"arts":{"ece":0.0843,"acc":0.3823,"conf":0.4658}}},"gaps":null,"latency":{"median":45,"p95":null,"mean":50.8},"results":{"1":{"score":0.7048,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.5166,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.437,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7765,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.1496,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7309,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5365,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.4086,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3356,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.4774,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.5091,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.0145,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.5538,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.4574,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.3316,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8215,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.6331,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.528,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.4287,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.1895,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0792,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.4894,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0897,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.3175,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2287,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.0987,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.7141,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1993,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.406,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.5594,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.3772,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5048,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1377,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3335,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.5165,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.2229,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.4142,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.4615,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.5098,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.4324,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.2784,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.7048,"skill":0.6015,"coverage":1,"random":0.2592,"tracks":[],"combined":0.4288,"private_eq":0.5809},"2":{"raw":0.5166,"skill":0.4418,"coverage":1,"random":0.1341,"tracks":[],"combined":0.5947,"private_eq":0.6522},"3":{"raw":0.437,"skill":0.4262,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2399,"private_eq":0.2618},"4":{"raw":0.7765,"skill":0.7736,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6399,"private_eq":0.6475},"5":{"raw":0.1496,"skill":0.1445,"coverage":1,"random":0.006,"tracks":[],"combined":0.0186,"private_eq":0.0346},"6":{"raw":0.7309,"skill":0.3691,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.5365,"skill":0.2288,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.4086,"skill":0.1448,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3356,"skill":0.0048,"coverage":1,"random":0.3324,"tracks":[],"combined":0.226,"private_eq":0.4885},"20":{"raw":0.4696,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.5091,"skill":0.0183,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.014,"skill":0.0063,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5364,"skill":0.0728,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4574,"skill":0.2765,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3316,"skill":0.1088,"coverage":1,"random":0.25,"tracks":[],"combined":0.0522,"private_eq":0.2965},"26":{"raw":0.8215,"skill":0.7619,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.6331,"skill":0.5107,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.528,"skill":0.056,"coverage":1,"random":0.5,"tracks":[],"combined":0.2545,"private_eq":0.631},"29":{"raw":0.4287,"skill":0.2383,"coverage":1,"random":0.25,"tracks":[],"combined":0.2505,"private_eq":0.4435},"30":{"raw":0.1895,"skill":0.0188,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2737,"random":0.25},{"track":"10-choice","raw":0.1054,"random":0.1}],"combined":0.1058,"private_eq":0.2699},"31":{"raw":0.0792,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0214,"private_eq":0.1109},"32":{"raw":0.4894,"skill":0.1882,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0897,"skill":0.0776,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1533,"private_eq":0.1729},"36":{"raw":0.3175,"skill":0.2279,"coverage":1,"random":0.116,"tracks":[],"combined":0.1895,"private_eq":0.2908},"37":{"raw":0.2287,"skill":0.0326,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0987,"skill":0.0699,"coverage":1,"random":0.031,"tracks":[],"combined":0.104,"private_eq":0.1407},"39":{"raw":0.7141,"skill":0.5795,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4062,"private_eq":0.6002},"40":{"raw":0.1993,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1036,"headline":false},{"track":"A · English","score":0.1993,"headline":true},{"track":"C · Arabic pairs","score":0.445,"headline":false},{"track":"C · English pairs","score":0.425,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.406,"skill":0.109,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5594,"skill":0.1428,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3772,"skill":0.0119,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1232,"private_eq":0.453},"44":{"raw":0.5048,"skill":0.0096,"coverage":1,"random":0.5,"tracks":[],"combined":0.0372,"private_eq":0.5236},"45":{"raw":0.1377,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3335,"skill":0.0327,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.5165,"skill":0.033,"coverage":1,"random":0.5,"tracks":[],"combined":0.0666,"private_eq":0.5381},"57":{"raw":0.2229,"skill":0.126,"coverage":1,"random":0.1109,"tracks":[],"combined":0,"private_eq":0.1109},"58":{"raw":0.4142,"skill":0.1509,"coverage":1,"random":0.3101,"tracks":[],"combined":0.0103,"private_eq":0.3243},"59":{"raw":0.4615,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5098,"skill":0.0196,"coverage":1,"random":0.5,"tracks":[],"combined":0.0308,"private_eq":0.5204},"62":{"raw":0.4324,"skill":0.2432,"coverage":1,"random":0.25,"tracks":[],"combined":0.2687,"private_eq":0.457},"64":{"raw":0.2784,"skill":0.098,"coverage":1,"random":0.2,"tracks":[],"combined":0.07,"private_eq":0.2637}},"coverage":1,"pending":0,"scores":{"balanced_raw":38.06,"balanced_skill":16.3,"breadth_skill":14.88,"public_skill":16.78},"frozen_scores":{"balanced_raw":38.06,"balanced_skill":16.78,"breadth_skill":14.88},"categories":[{"id":"knowledge","raw":0.3076,"skill":0.0697,"combined":0.0739,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4005,"skill":0.1423,"combined":0.1238,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4163,"skill":0.2213,"combined":0.1673,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4295,"skill":0.3552,"combined":0.3065,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3568,"skill":0.038,"combined":0.0469,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"kev-0.8b-r15","pub":16.78,"same_skills":15.84,"new_domains":12.67,"v":16.3,"lab":false,"jev":false,"rank_v":212,"rank_pub":222,"name":"Kev 0.8B r15","eq":{"S":14.97,"O":18.29},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":8.98,"new_domains":7.32}},"notes":["Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."]},{"engine":"von-1.3","name":"Von 1.3","long_name":"Von 1.3","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":400000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/wfzyx/von","code_url":null,"base_model":"answerdotai/ModernBERT-large","base_url":"https://huggingface.co/answerdotai/ModernBERT-large","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110196,"unsupported":5,"error":0},"lifted":false,"calibration":{"acc":0.4189,"conf":0.6347,"ece":0.2157,"brier":0.7709,"over95":0.0974,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0323,"conf":0.0467,"acc":0.0222},{"lo":0.1,"w":0.0158,"conf":0.1551,"acc":0.0549},{"lo":0.2,"w":0.0419,"conf":0.2648,"acc":0.1776},{"lo":0.3,"w":0.1003,"conf":0.3569,"acc":0.291},{"lo":0.4,"w":0.071,"conf":0.45,"acc":0.4392},{"lo":0.5,"w":0.2511,"conf":0.544,"acc":0.454},{"lo":0.6,"w":0.1289,"conf":0.6442,"acc":0.401},{"lo":0.7,"w":0.0618,"conf":0.7484,"acc":0.4372},{"lo":0.8,"w":0.0491,"conf":0.8539,"acc":0.4283},{"lo":0.9,"w":0.2477,"conf":0.985,"acc":0.5482}],"areas":{"knowledge":{"ece":0.2626,"acc":0.3993,"conf":0.6619},"language":{"ece":0.279,"acc":0.4517,"conf":0.7239},"retrieval":{"ece":0.1745,"acc":0.5379,"conf":0.6479},"tools":{"ece":0.2291,"acc":0.309,"conf":0.5381},"arts":{"ece":0.0974,"acc":0.374,"conf":0.4614}}},"gaps":null,"latency":{"median":17.7,"p95":null,"mean":103.3},"results":{"1":{"score":0.3695,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":144.9},"2":{"score":0.2731,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":919.4},"3":{"score":0.0039,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1802.1},"4":{"score":0.7573,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":49.2},"5":{"score":0.5989,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":45.6},"6":{"score":0.5488,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":527.5},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1562.1},"10":{"score":0.2683,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":424.5},"11":{"score":0.2904,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":4084.4},"12":{"score":0.4664,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":23.9},"20":{"score":0.5372,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.1},"21":{"score":0.4703,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":22.9},"22":{"score":0.0275,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":918.2},"23":{"score":0.5226,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.5},"24":{"score":0.3456,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":23.1},"25":{"score":0.2449,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":23.5},"26":{"score":0.5791,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":23},"27":{"score":0.4215,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":23},"28":{"score":0.5399,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":22.9},"29":{"score":0.3751,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":24.4},"30":{"score":0.166,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.074,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.3},"32":{"score":0.5652,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":85.9},"33":{"score":0.0145,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":341.6},"36":{"score":0.1771,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1426.4},"37":{"score":0.2628,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.7},"38":{"score":0.0351,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1384.4},"39":{"score":0.6119,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":46.8},"40":{"score":0.1892,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":23.2},"41":{"score":0.3738,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":23.9},"42":{"score":0.5675,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.2},"43":{"score":0.4333,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":24.2},"44":{"score":0.686,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.6},"45":{"score":0.1916,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":24.6},"50":{"score":0.2894,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":73.8},"56":{"score":0.5,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":313.5},"57":{"score":0.1628,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":24.6},"58":{"score":0.3819,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":24.5},"59":{"score":0.411,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":120.1},"61":{"score":0.6048,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":414.9},"62":{"score":0.4729,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":75.4},"64":{"score":0.2462,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":24.4}},"benchmarks":{"1":{"raw":0.3695,"skill":0.1489,"coverage":1,"random":0.2592,"tracks":[],"combined":0.1087,"private_eq":0.3465},"2":{"raw":0.2711,"skill":0.1582,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.0679,"private_eq":0.2012},"3":{"raw":0.0039,"skill":0,"coverage":1,"random":0.0189,"tracks":[],"combined":0.0458,"private_eq":0.0735},"4":{"raw":0.7573,"skill":0.7542,"coverage":1,"random":0.0127,"tracks":[],"combined":0.597,"private_eq":0.6057},"5":{"raw":0.5989,"skill":0.5965,"coverage":1,"random":0.006,"tracks":[],"combined":0.3428,"private_eq":0.3532},"6":{"raw":0.5488,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.2683,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.2852,"skill":0,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.4664,"skill":0.2007,"coverage":1,"random":0.3324,"tracks":[],"combined":0.1543,"private_eq":0.4412},"20":{"raw":0.5314,"skill":0.0629,"coverage":1,"random":0.5,"tracks":[],"combined":0.0973,"private_eq":0.5533},"21":{"raw":0.4703,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0254,"skill":0.0178,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.0269,"private_eq":0.0445},"23":{"raw":0.512,"skill":0.0239,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3456,"skill":0.1275,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0.0173,"private_eq":0.2706},"26":{"raw":0.5791,"skill":0.4387,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.4215,"skill":0.2285,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5399,"skill":0.0798,"coverage":1,"random":0.5,"tracks":[],"combined":0.1637,"private_eq":0.5861},"29":{"raw":0.3751,"skill":0.1668,"coverage":1,"random":0.25,"tracks":[],"combined":0.1168,"private_eq":0.3444},"30":{"raw":0.166,"skill":0,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2418,"random":0.25},{"track":"10-choice","raw":0.0902,"random":0.1}],"combined":0.0119,"private_eq":0.1933},"31":{"raw":0.074,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0883},"32":{"raw":0.5652,"skill":0.3087,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0145,"skill":0.0014,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.1771,"skill":0.0691,"coverage":1,"random":0.116,"tracks":[],"combined":0.1025,"private_eq":0.2148},"37":{"raw":0.2628,"skill":0.0753,"coverage":1,"random":0.2027,"tracks":[],"combined":0.066,"private_eq":0.263},"38":{"raw":0.0351,"skill":0.0042,"coverage":1,"random":0.031,"tracks":[],"combined":0.0021,"private_eq":0.0431},"39":{"raw":0.6119,"skill":0.4292,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4376,"private_eq":0.6213},"40":{"raw":0.1892,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2645,"headline":false},{"track":"A · English","score":0.1892,"headline":true},{"track":"C · Arabic pairs","score":0.53,"headline":false},{"track":"C · English pairs","score":0.21,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.3738,"skill":0.0606,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3032,"private_eq":0.5401},"42":{"raw":0.5675,"skill":0.1586,"coverage":1,"random":0.486,"tracks":[],"combined":0.1902,"private_eq":0.588},"43":{"raw":0.4333,"skill":0.1009,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1021,"private_eq":0.4399},"44":{"raw":0.686,"skill":0.372,"coverage":1,"random":0.5,"tracks":[],"combined":0.3745,"private_eq":0.6903},"45":{"raw":0.1916,"skill":0.0329,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.2894,"skill":0,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1628,"skill":0.0584,"coverage":1,"random":0.1109,"tracks":[],"combined":0,"private_eq":0.1109},"58":{"raw":0.3819,"skill":0.1041,"coverage":1,"random":0.3101,"tracks":[],"combined":0,"private_eq":0.3157},"59":{"raw":0.411,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6048,"skill":0.2096,"coverage":1,"random":0.5,"tracks":[],"combined":0.0706,"private_eq":0.5401},"62":{"raw":0.4729,"skill":0.2972,"coverage":1,"random":0.25,"tracks":[],"combined":0.1659,"private_eq":0.3808},"64":{"raw":0.2462,"skill":0.0577,"coverage":1,"random":0.2,"tracks":[],"combined":0.0991,"private_eq":0.2867}},"coverage":0.9996,"pending":0.0004,"scores":{"balanced_raw":34.91,"balanced_skill":11.65,"breadth_skill":12.61,"public_skill":13.86},"frozen_scores":{"balanced_raw":34.91,"balanced_skill":13.86,"breadth_skill":12.61},"categories":[{"id":"knowledge","raw":0.3137,"skill":0.0928,"combined":0.0616,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3706,"skill":0.1163,"combined":0.1473,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.486,"skill":0.3013,"combined":0.2086,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.2208,"skill":0.1174,"combined":0.0757,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.3458,"skill":0.027,"combined":0.0803,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"von-1.3","pub":13.86,"same_skills":12.8,"new_domains":6.99,"v":11.65,"lab":false,"jev":false,"rank_v":226,"rank_pub":226,"name":"Von 1.3","eq":{"S":11.9,"O":11.28},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":7.14,"new_domains":4.51}}},{"engine":"jevk5-lite","name":"JevK5 Lite","long_name":"JevK5 Lite","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":300000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/alibiserikbay/JevK5-Lite","code_url":null,"base_model":"microsoft/deberta-v3-large","base_url":"https://huggingface.co/microsoft/deberta-v3-large","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":76007,"unsupported":34194,"error":0},"lifted":false,"calibration":{"acc":0.4475,"conf":0.5895,"ece":0.149,"brier":0.6428,"over95":0.0075,"n":28900,"benchmarks":27,"rel":[{"lo":0,"w":0.0032,"conf":0.0796,"acc":0.0224},{"lo":0.1,"w":0.0341,"conf":0.1554,"acc":0.2161},{"lo":0.2,"w":0.058,"conf":0.2669,"acc":0.2911},{"lo":0.3,"w":0.1091,"conf":0.3495,"acc":0.3073},{"lo":0.4,"w":0.1253,"conf":0.4494,"acc":0.3237},{"lo":0.5,"w":0.2499,"conf":0.5492,"acc":0.3624},{"lo":0.6,"w":0.1094,"conf":0.6455,"acc":0.5326},{"lo":0.7,"w":0.0947,"conf":0.751,"acc":0.6602},{"lo":0.8,"w":0.1074,"conf":0.8537,"acc":0.5661},{"lo":0.9,"w":0.1088,"conf":0.9493,"acc":0.7063}],"areas":{"knowledge":{"ece":0.0753,"acc":0.4286,"conf":0.4953},"language":{"ece":0.2539,"acc":0.4257,"conf":0.6773},"retrieval":{"ece":0.1397,"acc":0.5559,"conf":0.6252},"tools":{"ece":0.1314,"acc":0.916,"conf":0.7846},"arts":{"ece":0.2532,"acc":0.3176,"conf":0.5708}}},"gaps":null,"latency":{"median":19.3,"p95":null,"mean":44.2},"results":{"1":{"score":0.9331,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":658,"unsupported":1036,"errors":0,"median_ms":null},"2":{"score":0.1933,"jev":null,"metric":"nDCG@10","requests":685,"answered":76,"unsupported":609,"errors":0,"median_ms":null},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":0.867,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.711,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":null,"jev":null,"metric":null,"requests":10000,"answered":0,"unsupported":10000,"errors":0,"median_ms":null},"9":{"score":null,"jev":null,"metric":null,"requests":88,"answered":0,"unsupported":88,"errors":0,"median_ms":null},"10":{"score":null,"jev":null,"metric":null,"requests":2500,"answered":0,"unsupported":2500,"errors":0,"median_ms":null},"11":{"score":0.0801,"jev":null,"metric":"macro-F1","requests":123,"answered":2,"unsupported":121,"errors":0,"median_ms":null},"12":{"score":0.2621,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.5056,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.481,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0,"jev":null,"metric":"accuracy","requests":5000,"answered":1,"unsupported":4999,"errors":0,"median_ms":null},"24":{"score":0.387,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.2653,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.7315,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.4497,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.5438,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.4653,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.1774,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.3206,"jev":null,"metric":"accuracy","requests":5000,"answered":708,"unsupported":4292,"errors":0,"median_ms":null},"32":{"score":0.5106,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.1752,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":919,"unsupported":731,"errors":0,"median_ms":null},"36":{"score":0.1303,"jev":null,"metric":"nDCG@10","requests":220,"answered":9,"unsupported":211,"errors":0,"median_ms":null},"37":{"score":0.067,"jev":null,"metric":"macro-F1","requests":5000,"answered":3409,"unsupported":1591,"errors":0,"median_ms":null},"38":{"score":0.031,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.6135,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1163,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.3823,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.4814,"jev":null,"metric":"macro-F1","requests":5500,"answered":3243,"unsupported":2257,"errors":0,"median_ms":null},"43":{"score":0.355,"jev":null,"metric":"accuracy","requests":570,"answered":569,"unsupported":1,"errors":0,"median_ms":null},"44":{"score":0.5108,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1417,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.6923,"jev":null,"metric":"accuracy","requests":1676,"answered":13,"unsupported":1663,"errors":0,"median_ms":null},"56":{"score":0.5018,"jev":null,"metric":"accuracy","requests":2000,"answered":1997,"unsupported":3,"errors":0,"median_ms":null},"57":{"score":0.1789,"jev":null,"metric":"accuracy","requests":12032,"answered":11825,"unsupported":207,"errors":0,"median_ms":null},"58":{"score":0.3642,"jev":null,"metric":"accuracy","requests":5507,"answered":5499,"unsupported":8,"errors":0,"median_ms":null},"59":{"score":0.0233,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":478,"unsupported":2222,"errors":0,"median_ms":null},"61":{"score":0.511,"jev":null,"metric":"accuracy","requests":4000,"answered":2998,"unsupported":1002,"errors":0,"median_ms":null},"62":{"score":0.2042,"jev":null,"metric":"accuracy","requests":3652,"answered":862,"unsupported":2790,"errors":0,"median_ms":null},"64":{"score":0.3125,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.3625,"skill":0.1393,"coverage":0.3884,"random":0.2592,"tracks":[],"combined":0.0201,"private_eq":0.2816},"2":{"raw":0.0214,"skill":0,"coverage":0.1109,"random":0.1341,"tracks":[],"combined":0,"private_eq":0.1341},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0.867,"skill":0.8653,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7582,"private_eq":0.763},"5":{"raw":0.711,"skill":0.7093,"coverage":1,"random":0.006,"tracks":[],"combined":0.287,"private_eq":0.2984},"6":{"raw":0,"skill":0,"coverage":0,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":0,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0,"skill":0,"coverage":0,"random":0.399,"tracks":[]},"11":{"raw":0.0013,"skill":0,"coverage":0.0163,"random":0.3085,"tracks":[]},"12":{"raw":0.2621,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.5063,"skill":0.0125,"coverage":1,"random":0.5,"tracks":[],"combined":0.0322,"private_eq":0.5211},"21":{"raw":0.481,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0152,"private_eq":0.5127},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0,"skill":0,"coverage":0.0002,"random":0.5,"tracks":[]},"24":{"raw":0.387,"skill":0.1827,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2653,"skill":0.0204,"coverage":1,"random":0.25,"tracks":[],"combined":0.0004,"private_eq":0.2581},"26":{"raw":0.7315,"skill":0.6419,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.4497,"skill":0.2661,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5438,"skill":0.0876,"coverage":1,"random":0.5,"tracks":[],"combined":0.1839,"private_eq":0.5961},"29":{"raw":0.4653,"skill":0.2871,"coverage":1,"random":0.25,"tracks":[],"combined":0.3693,"private_eq":0.5316},"30":{"raw":0.1774,"skill":0.0037,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2555,"random":0.25},{"track":"10-choice","raw":0.0993,"random":0.1}],"combined":0.0389,"private_eq":0.2153},"31":{"raw":0.0454,"skill":0,"coverage":0.1416,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.5106,"skill":0.2219,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0976,"skill":0.0856,"coverage":0.557,"random":0.0131,"tracks":[],"combined":0.0864,"private_eq":0.1077},"36":{"raw":0.0053,"skill":0,"coverage":0.0409,"random":0.116,"tracks":[],"combined":0.1003,"private_eq":0.2128},"37":{"raw":0.0431,"skill":0,"coverage":0.6818,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.031,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.6135,"skill":0.4315,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4614,"private_eq":0.6373},"40":{"raw":0.1163,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.1163,"headline":true},{"track":"C · Arabic pairs","score":0.37,"headline":false},{"track":"C · English pairs","score":0.385,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.3823,"skill":0.0734,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3095,"private_eq":0.5442},"42":{"raw":0.2839,"skill":0,"coverage":0.5896,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3544,"skill":0,"coverage":0.9982,"random":0.3697,"tracks":[],"combined":0.0227,"private_eq":0.3904},"44":{"raw":0.5108,"skill":0.0216,"coverage":1,"random":0.5,"tracks":[],"combined":0.155,"private_eq":0.5818},"45":{"raw":0.1417,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.0054,"skill":0,"coverage":0.0078,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.501,"skill":0.002,"coverage":0.9985,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1758,"skill":0.073,"coverage":0.9828,"random":0.1109,"tracks":[],"combined":0.0167,"private_eq":0.1348},"58":{"raw":0.3637,"skill":0.0777,"coverage":0.9985,"random":0.3101,"tracks":[],"combined":0,"private_eq":0.3101},"59":{"raw":0.0041,"skill":0,"coverage":0.177,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.383,"skill":0,"coverage":0.7495,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"62":{"raw":0.0482,"skill":0,"coverage":0.236,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"64":{"raw":0.3125,"skill":0.1406,"coverage":1,"random":0.2,"tracks":[],"combined":0.0408,"private_eq":0.2406}},"coverage":0.6028,"pending":0.3972,"scores":{"balanced_raw":26.31,"balanced_skill":8.78,"breadth_skill":8.66,"public_skill":10.28},"frozen_scores":{"balanced_raw":26.31,"balanced_skill":10.28,"breadth_skill":8.66},"categories":[{"id":"knowledge","raw":0.2904,"skill":0.0536,"combined":0.0473,"coverage":0.8895,"pending":0,"n":11},{"id":"language","raw":0.2452,"skill":0.0904,"combined":0.138,"coverage":0.7641,"pending":0,"n":9},{"id":"retrieval","raw":0.4283,"skill":0.2866,"combined":0.2035,"coverage":0.7392,"pending":0,"n":6},{"id":"tools","raw":0.0934,"skill":0.031,"combined":0,"coverage":0.1506,"pending":0,"n":5},{"id":"arts","raw":0.2175,"skill":0.0255,"combined":0.0576,"coverage":0.5013,"pending":0,"n":6}],"v03":{"engine":"jevk5-lite","pub":10.28,"same_skills":10.37,"new_domains":4.18,"v":8.78,"lab":false,"jev":false,"rank_v":232,"rank_pub":231,"name":"JevK5 Lite","eq":{"S":9.44,"O":7.81},"rank_tie":55,"tied":true,"parts":{"public":0,"same_skills":5.66,"new_domains":3.12}},"notes":["Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."]},{"engine":"gliner-decide","name":"GLiNER2.5-Decide","long_name":"GLiNER2.5-Decide","variant":null,"meta":{"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":486444053,"params_estimated":false,"params_from_name":false,"trained_bytes":1945828140,"model_url":"https://huggingface.co/fastino/GLiNER2.5-Decide","code_url":"https://github.com/fastino-ai/GLiNER2","base_model":"fastino/gliner2-large-v1","base_url":"https://huggingface.co/fastino/gliner2-large-v1","weights_repo":"fastino/GLiNER2.5-Decide","served_checkpoint":null,"org":{"id":"fastino","name":"Fastino","logo":"data/orgs/fastino.png"}},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.434,"conf":0.506,"ece":0.0884,"brier":0.7049,"over95":0.0132,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0728,"conf":0.0668,"acc":0.0955},{"lo":0.1,"w":0.0602,"conf":0.1395,"acc":0.1625},{"lo":0.2,"w":0.0435,"conf":0.269,"acc":0.2737},{"lo":0.3,"w":0.1254,"conf":0.3513,"acc":0.3479},{"lo":0.4,"w":0.1185,"conf":0.443,"acc":0.458},{"lo":0.5,"w":0.2637,"conf":0.5461,"acc":0.5565},{"lo":0.6,"w":0.1213,"conf":0.6415,"acc":0.6031},{"lo":0.7,"w":0.0781,"conf":0.746,"acc":0.4909},{"lo":0.8,"w":0.0623,"conf":0.8508,"acc":0.4376},{"lo":0.9,"w":0.0541,"conf":0.9493,"acc":0.4055}],"areas":{"knowledge":{"ece":0.1068,"acc":0.3494,"conf":0.4562},"language":{"ece":0.1455,"acc":0.5272,"conf":0.6136},"retrieval":{"ece":0.3318,"acc":0.4856,"conf":0.5177},"tools":{"ece":0.0778,"acc":0.4236,"conf":0.4034},"arts":{"ece":0.0567,"acc":0.4176,"conf":0.4743}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":23.3,"p95":228.6,"mean":156.4},"results":{"1":{"score":0.4622,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":29.1},"2":{"score":0.382,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":4962.3},"3":{"score":0.0768,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6563,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":42.5},"5":{"score":0.6042,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":96.5},"6":{"score":0.5935,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":142.3},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0079,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":82.6},"11":{"score":0.2177,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3091,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.2},"20":{"score":0.5816,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.4},"21":{"score":0.5065,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":21.8},"22":{"score":0.017,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1342.9},"23":{"score":0.5336,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.6},"24":{"score":0.3349,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":22.4},"25":{"score":0.2143,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5412,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":22.2},"27":{"score":0.3677,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":22.5},"28":{"score":0.5067,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":22},"29":{"score":0.3224,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":22.8},"30":{"score":0.1732,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0808,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.1},"32":{"score":0.4521,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0073,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":30.9},"36":{"score":0.201,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2328,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.3},"38":{"score":0,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1549.9},"39":{"score":0.6616,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3235,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":22.5,"tracks":[{"track":"A · Arabic","score":0.0847,"headline":false},{"track":"A · English","score":0.3235,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.535,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3527,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":22.3},"42":{"score":0.4801,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":23.2},"43":{"score":0.4018,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5012,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.3},"45":{"score":0.1677,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4547,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":27.6},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":25.2,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.161,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":22.9,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.3114,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":22.5,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.2533,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":25.3,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5068,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":23.8,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.3341,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":27.5,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.322,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":33.08,"balanced_skill":10.14,"breadth_skill":10.13,"public_skill":11.48},"frozen_scores":{"balanced_raw":33.08,"balanced_skill":11.48,"breadth_skill":10.13},"categories":[{"id":"knowledge","raw":0.2708,"skill":0.0229,"combined":0.0196,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3241,"skill":0.0826,"combined":0.1548,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4535,"skill":0.2532,"combined":0.1829,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.2524,"skill":0.1478,"combined":0.1228,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4004,"skill":0.0979,"combined":0.0508,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.4622,"skill":0.274,"coverage":1,"random":0.2592,"tracks":[],"combined":0.2177,"private_eq":0.4263},"2":{"raw":0.382,"skill":0.2863,"coverage":1,"random":0.1341,"tracks":[],"combined":0.2507,"private_eq":0.3577},"3":{"raw":0.0768,"skill":0.059,"coverage":1,"random":0.0189,"tracks":[],"combined":0.0878,"private_eq":0.1143},"4":{"raw":0.6563,"skill":0.6519,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6436,"private_eq":0.6511},"5":{"raw":0.6042,"skill":0.6018,"coverage":1,"random":0.006,"tracks":[],"combined":0.3166,"private_eq":0.3274},"6":{"raw":0.5935,"skill":0.0621,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5176,"headline":false},{"track":"RouterBench-5shot","score":0.6695,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2177,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3091,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5701,"skill":0.1401,"coverage":1,"random":0.5,"tracks":[],"combined":0.0973,"private_eq":0.5533},"21":{"raw":0.5065,"skill":0.0129,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0168,"skill":0.0091,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0093},"23":{"raw":0.5321,"skill":0.0642,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3365,"skill":0.1154,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2143,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"28":{"raw":0.5067,"skill":0.0134,"coverage":1,"random":0.5,"tracks":[],"combined":0.2292,"private_eq":0.6185},"29":{"raw":0.3224,"skill":0.0965,"coverage":1,"random":0.25,"tracks":[],"combined":0.2208,"private_eq":0.4215},"30":{"raw":0.1732,"skill":0.0021,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2426,"random":0.25},{"track":"10-choice","raw":0.1039,"random":0.1}],"combined":0.0008,"private_eq":0.1842},"31":{"raw":0.0808,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0009,"private_eq":0.0923},"32":{"raw":0.4521,"skill":0.1289,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.0073,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.201,"skill":0.0961,"coverage":1,"random":0.116,"tracks":[],"combined":0.0514,"private_eq":0.1701},"37":{"raw":0.2328,"skill":0.0378,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.6616,"skill":0.5023,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5715,"private_eq":0.7113},"40":{"raw":0.3235,"skill":0.1297,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0847,"headline":false},{"track":"A · English","score":0.3235,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.535,"headline":false}],"combined":0.1605,"private_eq":0.3541},"41":{"raw":0.3527,"skill":0.029,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3476,"private_eq":0.5693},"42":{"raw":0.4801,"skill":0,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.5012,"skill":0.0024,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"45":{"raw":0.1677,"skill":0.0043,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4547,"skill":0.2085,"coverage":1,"random":0.311,"tracks":[],"combined":0.1162,"private_eq":0.3973},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0146,"private_eq":0.5124},"57":{"raw":0.161,"skill":0.0563,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.1109},"58":{"raw":0.3114,"skill":0.0019,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.3107},"59":{"raw":0.2533,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5068,"skill":0.0136,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5},"62":{"raw":0.3341,"skill":0.1121,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.0564,"private_eq":0.2996},"64":{"raw":0.322,"skill":0.1525,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.0603,"private_eq":0.256}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author AutoExtractor.classify_text with TF32 matmuls on (JEV_DECIDE_TF32=1; 23.3 / 228.6 ms vs 24.0 / 251.3 ms in fp32, kept as latency-v1-full.fp32), encoder attention eager as shipped, venv-q torch 2.14.0+cu130 (+peft)","v03":{"engine":"gliner-decide","pub":11.48,"same_skills":12.05,"new_domains":4.86,"v":10.14,"lab":false,"jev":false,"rank_v":230,"rank_pub":230,"name":"GLiNER2.5-Decide","eq":{"S":11.14,"O":8.64},"rank_tie":54,"tied":false,"parts":{"public":0,"same_skills":6.69,"new_domains":3.46}}},{"engine":"bosun-v3.1-0.6b","name":"Bosun v3.1 0.6B","long_name":"Bosun v3.1 0.6B LoRA","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":596049920,"params_estimated":false,"params_from_name":false,"trained_bytes":41470928,"model_url":"https://huggingface.co/Hanno-Labs/bosun-v3.1-0.6b","code_url":"https://github.com/Hanno-Labs/jev-compatible-server","base_model":"Qwen/Qwen3-0.6B-Base","base_url":"https://huggingface.co/Qwen/Qwen3-0.6B-Base","weights_repo":"Hanno-Labs/bosun-v3.1-0.6b","served_checkpoint":null,"org":{"id":"Hanno-Labs","name":"Hanno Labs","logo":"data/orgs/Hanno-Labs.png"}},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.4496,"conf":0.5883,"ece":0.1419,"brier":0.6671,"over95":0.0082,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0561,"conf":0.0453,"acc":0.0512},{"lo":0.1,"w":0.0272,"conf":0.1506,"acc":0.1913},{"lo":0.2,"w":0.0276,"conf":0.2547,"acc":0.2614},{"lo":0.3,"w":0.0891,"conf":0.3548,"acc":0.2501},{"lo":0.4,"w":0.1046,"conf":0.4491,"acc":0.3119},{"lo":0.5,"w":0.1894,"conf":0.5505,"acc":0.4418},{"lo":0.6,"w":0.1705,"conf":0.6471,"acc":0.5073},{"lo":0.7,"w":0.132,"conf":0.7465,"acc":0.5471},{"lo":0.8,"w":0.1027,"conf":0.8495,"acc":0.598},{"lo":0.9,"w":0.1006,"conf":0.9495,"acc":0.7503}],"areas":{"knowledge":{"ece":0.1633,"acc":0.3386,"conf":0.5018},"language":{"ece":0.1595,"acc":0.5205,"conf":0.68},"retrieval":{"ece":0.101,"acc":0.6176,"conf":0.6825},"tools":{"ece":0.21,"acc":0.5258,"conf":0.6823},"arts":{"ece":0.0962,"acc":0.386,"conf":0.4816}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":38.7,"p95":107.3,"mean":53.6},"results":{"1":{"score":0.4658,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":43.9},"2":{"score":0.5737,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":690.8},"3":{"score":0.5315,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7613,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":39.8},"5":{"score":0.6483,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":41.8},"6":{"score":0.6409,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":86.3},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5432,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":39.5},"11":{"score":0.2165,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2749,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":38.6},"20":{"score":0.489,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.5},"21":{"score":0.5091,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":37.8},"22":{"score":0.035,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":95},"23":{"score":0.5094,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":37.9},"24":{"score":0.3468,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":38.1},"25":{"score":0.2755,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5614,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":38.1},"27":{"score":0.4334,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":37.9},"28":{"score":0.4972,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":37.6},"29":{"score":0.3277,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":38.2},"30":{"score":0.1808,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0834,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.3},"32":{"score":0.4428,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0145,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":76.4},"36":{"score":0.3171,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.4299,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.7},"38":{"score":0.0556,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":249},"39":{"score":0.4799,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.0554,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":38,"tracks":[{"track":"A · Arabic","score":0.022,"headline":false},{"track":"A · English","score":0.0554,"headline":true},{"track":"C · Arabic pairs","score":0.455,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2628,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":38.1},"42":{"score":0.4968,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.6},"43":{"score":0.3579,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4948,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.4},"45":{"score":0.1497,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3663,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":39.5},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":48.3,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1774,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":38.3,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.3434,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":38.2,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.2917,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":39.7,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5055,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":38.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.3001,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":39.6,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.3864,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":35.71,"balanced_skill":11.04,"breadth_skill":11.27,"public_skill":14.5},"frozen_scores":{"balanced_raw":35.71,"balanced_skill":14.5,"breadth_skill":11.27},"categories":[{"id":"knowledge","raw":0.2755,"skill":0.0263,"combined":0.0039,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2747,"skill":0.0432,"combined":0.0506,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5314,"skill":0.3415,"combined":0.2815,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.3834,"skill":0.2845,"combined":0.2647,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3844,"skill":0.0668,"combined":0.0658,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.4658,"skill":0.2788,"coverage":1,"random":0.2592,"tracks":[],"combined":0.2381,"private_eq":0.4413},"2":{"raw":0.5737,"skill":0.5077,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6182,"private_eq":0.6723},"3":{"raw":0.5315,"skill":0.5225,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4552,"private_eq":0.4706},"4":{"raw":0.7613,"skill":0.7582,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6043,"private_eq":0.6128},"5":{"raw":0.6483,"skill":0.6462,"coverage":1,"random":0.006,"tracks":[],"combined":0.3631,"private_eq":0.3731},"6":{"raw":0.6409,"skill":0.1556,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.6085,"headline":false},{"track":"RouterBench-5shot","score":0.6733,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2165,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2749,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.4897,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.5091,"skill":0.0183,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0333,"skill":0.0257,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5217,"skill":0.0434,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3579,"skill":0.1439,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"28":{"raw":0.4972,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"29":{"raw":0.3277,"skill":0.1036,"coverage":1,"random":0.25,"tracks":[],"combined":0.2406,"private_eq":0.4362},"30":{"raw":0.1808,"skill":0.0102,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2434,"random":0.25},{"track":"10-choice","raw":0.1183,"random":0.1}],"combined":0.0238,"private_eq":0.203},"31":{"raw":0.0834,"skill":0.0017,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0009,"private_eq":0.0923},"32":{"raw":0.4428,"skill":0.1141,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0145,"skill":0.0014,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.3171,"skill":0.2274,"coverage":1,"random":0.116,"tracks":[],"combined":0.1693,"private_eq":0.2731},"37":{"raw":0.4299,"skill":0.2849,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3353,"private_eq":0.4753},"38":{"raw":0.0556,"skill":0.0254,"coverage":1,"random":0.031,"tracks":[],"combined":0.0237,"private_eq":0.0638},"39":{"raw":0.4799,"skill":0.235,"coverage":1,"random":0.3201,"tracks":[],"combined":0.0273,"private_eq":0.3455},"40":{"raw":0.0554,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.022,"headline":false},{"track":"A · English","score":0.0554,"headline":true},{"track":"C · Arabic pairs","score":0.455,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.2628,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4968,"skill":0.021,"coverage":1,"random":0.486,"tracks":[],"combined":0.1294,"private_eq":0.5571},"43":{"raw":0.3579,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.4948,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0318,"private_eq":0.5209},"45":{"raw":0.1497,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3663,"skill":0.0804,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0146,"private_eq":0.5124},"57":{"raw":0.1774,"skill":0.0748,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.0344,"private_eq":0.1504},"58":{"raw":0.3434,"skill":0.0483,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0006,"private_eq":0.3177},"59":{"raw":0.2917,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5055,"skill":0.011,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1443,"private_eq":0.5765},"62":{"raw":0.3001,"skill":0.0668,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.3864,"skill":0.233,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.2154,"private_eq":0.3787}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author RegistryRuntime.decide with the row-batched backend (all questions of a row scored together, prompts rendered exactly as model.predict); board used exact per-question mode; bf16, torch 2.14.0+cu130","v03":{"engine":"bosun-v3.1-0.6b","pub":14.5,"same_skills":14.09,"new_domains":4.18,"v":11.04,"lab":false,"jev":false,"rank_v":227,"rank_pub":224,"name":"Bosun v3.1 0.6B LoRA","eq":{"S":13.21,"O":7.8},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":7.92,"new_domains":3.12}}},{"engine":"jev-style-0.8b-v3","name":"Jev-Style 0.8B v3","long_name":"Jev-Style 0.8B v3","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":870000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/chaoliangUNSW/Jev-Style-0.8B-Decision-v3","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110139,"unsupported":62,"error":0},"lifted":false,"calibration":{"acc":0.4707,"conf":0.5463,"ece":0.0841,"brier":0.6403,"over95":0.0021,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0495,"conf":0.0581,"acc":0.1242},{"lo":0.1,"w":0.0262,"conf":0.146,"acc":0.1818},{"lo":0.2,"w":0.0507,"conf":0.2622,"acc":0.2279},{"lo":0.3,"w":0.1201,"conf":0.3568,"acc":0.3069},{"lo":0.4,"w":0.1247,"conf":0.4506,"acc":0.433},{"lo":0.5,"w":0.2286,"conf":0.5477,"acc":0.5019},{"lo":0.6,"w":0.1539,"conf":0.6462,"acc":0.517},{"lo":0.7,"w":0.111,"conf":0.7487,"acc":0.5905},{"lo":0.8,"w":0.0831,"conf":0.8486,"acc":0.6685},{"lo":0.9,"w":0.0521,"conf":0.9394,"acc":0.8037}],"areas":{"knowledge":{"ece":0.1199,"acc":0.3922,"conf":0.5121},"language":{"ece":0.1248,"acc":0.5219,"conf":0.6468},"retrieval":{"ece":0.0844,"acc":0.5266,"conf":0.4487},"tools":{"ece":0.0355,"acc":0.6324,"conf":0.6351},"arts":{"ece":0.1021,"acc":0.4093,"conf":0.4657}}},"gaps":null,"latency":{"median":36,"p95":null,"mean":111.9},"results":{"1":{"score":0.6936,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":178.2},"2":{"score":0.5829,"jev":null,"metric":"nDCG@10","requests":685,"answered":631,"unsupported":54,"errors":0,"median_ms":1904.7},"3":{"score":0.4744,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":817.3},"4":{"score":0.5241,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":155.6},"5":{"score":0.5014,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":257.9},"6":{"score":0.7838,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":423.3},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1738.6},"10":{"score":0.5961,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":107.5},"11":{"score":0.4432,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2990},"12":{"score":0.2831,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":58.4},"20":{"score":0.4876,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.7},"21":{"score":0.4886,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":57.7},"22":{"score":0.1515,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1391.5},"23":{"score":0.5674,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.1},"24":{"score":0.3902,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":57.8},"25":{"score":0.2602,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":58.7},"26":{"score":0.7504,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":57.4},"27":{"score":0.5282,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":57.6},"28":{"score":0.5036,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":57},"29":{"score":0.3524,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":59},"30":{"score":0.1983,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.107,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":128.4},"32":{"score":0.4934,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":96.8},"33":{"score":0.0303,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":468.8},"36":{"score":0.2909,"jev":null,"metric":"nDCG@10","requests":220,"answered":197,"unsupported":23,"errors":0,"median_ms":2065},"37":{"score":0.3201,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":59.4},"38":{"score":0.0559,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3687.5},"39":{"score":0.6951,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":115.4},"40":{"score":0.1555,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":57.6},"41":{"score":0.3347,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":57.9},"42":{"score":0.5784,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":98.4},"43":{"score":0.4,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":58},"44":{"score":0.5262,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":57.5},"45":{"score":0.1316,"jev":null,"metric":"accuracy","requests":501,"answered":494,"unsupported":7,"errors":0,"median_ms":61.4},"50":{"score":0.3246,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":244.4},"56":{"score":0.512,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":526.2},"57":{"score":0.1759,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":61.6},"58":{"score":0.4029,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":59.3},"59":{"score":0.0531,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":149.2},"61":{"score":0.5715,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":85.2},"62":{"score":0.3875,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":177.5},"64":{"score":0.3712,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":60.6}},"benchmarks":{"1":{"raw":0.6936,"skill":0.5864,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.4812,"combined":0.2926},"2":{"raw":0.5369,"skill":0.4652,"coverage":0.9212,"random":0.1341,"tracks":[],"private_eq":0.6064,"combined":0.5412},"3":{"raw":0.4744,"skill":0.4643,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.2618,"combined":0.2399},"4":{"raw":0.5241,"skill":0.518,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.4812,"combined":0.4695},"5":{"raw":0.5014,"skill":0.4984,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.2309,"combined":0.2184},"6":{"raw":0.7838,"skill":0.4931,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0.01,"combined":0},"10":{"raw":0.5961,"skill":0.328,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.4432,"skill":0.1948,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2831,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.4118,"combined":0.1098},"20":{"raw":0.4894,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"21":{"raw":0.4886,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"22":{"raw":0.1459,"skill":0.1392,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.1431,"combined":0.1274},"23":{"raw":0.5626,"skill":0.1252,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3902,"skill":0.1869,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2612,"combined":0.0046},"26":{"raw":0.7504,"skill":0.6671,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5282,"skill":0.3708,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5036,"skill":0.0072,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5811,"combined":0.1536},"29":{"raw":0.3524,"skill":0.1365,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3885,"combined":0.1763},"30":{"raw":0.1983,"skill":0.0291,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.279,"random":0.25},{"track":"10-choice","raw":0.1175,"random":0.1}],"private_eq":0.2799,"combined":0.1181},"31":{"raw":0.107,"skill":0.0274,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0991,"combined":0.0084},"32":{"raw":0.4934,"skill":0.1946,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0.0303,"skill":0.0174,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0526,"combined":0.03},"36":{"raw":0.2605,"skill":0.1635,"coverage":0.8955,"random":0.116,"tracks":[],"private_eq":0.2653,"combined":0.1603},"37":{"raw":0.3201,"skill":0.1472,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.3362,"combined":0.1589},"38":{"raw":0.0559,"skill":0.0257,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.071,"combined":0.0312},"39":{"raw":0.6951,"skill":0.5516,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.5724,"combined":0.3649},"40":{"raw":0.1555,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2383,"headline":false},{"track":"A · English","score":0.1555,"headline":true},{"track":"C · Arabic pairs","score":0.395,"headline":false},{"track":"C · English pairs","score":0.215,"headline":false}],"private_eq":0.2227,"combined":0},"41":{"raw":0.3347,"skill":0.0021,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3923,"combined":0.079},"42":{"raw":0.5784,"skill":0.1798,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.5974,"combined":0.2087},"43":{"raw":0.4,"skill":0.048,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.3697,"combined":0},"44":{"raw":0.5262,"skill":0.0524,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"45":{"raw":0.1298,"skill":0,"coverage":0.986,"random":0.1641,"tracks":[]},"50":{"raw":0.3246,"skill":0.0197,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4022,"combined":0.1234},"56":{"raw":0.512,"skill":0.024,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"57":{"raw":0.1759,"skill":0.0731,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.4029,"skill":0.1345,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3286,"combined":0.0166},"59":{"raw":0.0531,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.5715,"skill":0.143,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6354,"combined":0.2634},"62":{"raw":0.3875,"skill":0.1833,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3351,"combined":0.1043},"64":{"raw":0.3712,"skill":0.214,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.3404,"combined":0.167}},"coverage":0.993,"pending":0.007,"scores":{"balanced_raw":36.84,"balanced_skill":12.5,"breadth_skill":14.88,"public_skill":17},"frozen_scores":{"balanced_raw":36.84,"balanced_skill":17,"breadth_skill":14.88},"categories":[{"id":"knowledge","raw":0.2935,"skill":0.0536,"combined":0.0298,"coverage":0.9986,"pending":0,"n":11},{"id":"language","raw":0.3275,"skill":0.1189,"combined":0.1198,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4465,"skill":0.2621,"combined":0.2166,"coverage":0.981,"pending":0,"n":6},{"id":"tools","raw":0.4307,"skill":0.3536,"combined":0.2378,"coverage":0.9854,"pending":0,"n":5},{"id":"arts","raw":0.397,"skill":0.083,"combined":0.0794,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-style-0.8b-v3","pub":17,"same_skills":14.65,"new_domains":6.44,"v":12.5,"lab":false,"jev":false,"rank_v":221,"rank_pub":221,"name":"Jev-Style 0.8B v3","eq":{"S":13.77,"O":10.6},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":8.26,"new_domains":4.24}}},{"engine":"lavoir","name":"Lavoir","long_name":"MoganAI Lavoir ModernBERT-large","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":395881664,"params_estimated":false,"params_from_name":false,"trained_bytes":1686272004,"model_url":"https://huggingface.co/moganai/lavoir","code_url":"https://github.com/moganai/lavoir","base_model":"answerdotai/ModernBERT-large","base_url":"https://huggingface.co/answerdotai/ModernBERT-large","weights_repo":"moganai/lavoir","served_checkpoint":"4c5eaeb99b2368f1416b9893c7aacc58235aa09f","org":{"id":"moganai","name":"Mogan AI","logo":"data/orgs/moganai.png"}},"completed":109877,"counts":{"ok":109877,"unsupported":324},"lifted":false,"calibration":{"acc":0.3983,"conf":0.531,"ece":0.1492,"brier":0.7278,"over95":0.0235,"n":35093,"benchmarks":32,"rel":[{"lo":0,"w":0.1235,"conf":0.0513,"acc":0.076},{"lo":0.1,"w":0.0391,"conf":0.1438,"acc":0.2767},{"lo":0.2,"w":0.0404,"conf":0.2665,"acc":0.2049},{"lo":0.3,"w":0.0964,"conf":0.3502,"acc":0.3113},{"lo":0.4,"w":0.0943,"conf":0.4522,"acc":0.367},{"lo":0.5,"w":0.191,"conf":0.548,"acc":0.4736},{"lo":0.6,"w":0.1314,"conf":0.6494,"acc":0.4856},{"lo":0.7,"w":0.0976,"conf":0.7483,"acc":0.4891},{"lo":0.8,"w":0.0862,"conf":0.849,"acc":0.5201},{"lo":0.9,"w":0.1001,"conf":0.9559,"acc":0.5835}],"areas":{"knowledge":{"ece":0.1472,"acc":0.3631,"conf":0.5064},"language":{"ece":0.2226,"acc":0.4819,"conf":0.7044},"retrieval":{"ece":0.0571,"acc":0.3325,"conf":0.3084},"tools":{"ece":0.1183,"acc":0.4097,"conf":0.4368},"arts":{"ece":0.1436,"acc":0.371,"conf":0.5077}}},"gaps":{"share":0.0027,"reasons":[{"reason":"instructions longer than its native budget (no truncation allowed)","share":0.0026},{"reason":"prompt longer than its context window (no truncation allowed)","share":0}]},"latency":{"median":20,"p95":49.5,"mean":25.6},"results":{"1":{"score":0.3949,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":22.2},"2":{"score":0.2765,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.2826,"jev":null,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":null},"4":{"score":0.2795,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":22.4},"5":{"score":0.0944,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":24.7},"6":{"score":0.4908,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":37.7},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.598,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":21.6},"11":{"score":0.301,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":null},"12":{"score":0.3686,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":19.6},"20":{"score":0.4994,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.4},"21":{"score":0.465,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":17.4},"22":{"score":0.0235,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":62.6},"23":{"score":0.4996,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.9},"24":{"score":0.347,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.3},"25":{"score":0.2551,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.577,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":17.7},"27":{"score":0.366,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.4},"28":{"score":0.5083,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":17.4},"29":{"score":0.4199,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":19.8},"30":{"score":0.1884,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0854,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.4},"32":{"score":0.4189,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0424,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":38},"36":{"score":0.1669,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2815,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20},"38":{"score":0.0391,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":78.4},"39":{"score":0.5894,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4629,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":17.5,"tracks":[{"track":"A · Arabic","score":0.129,"headline":false},{"track":"A · English","score":0.4629,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.45,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3513,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":19.6},"42":{"score":0.4869,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":20.3},"43":{"score":0.3947,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5036,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.6},"45":{"score":0.1597,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3484,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":21.2},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":26,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1553,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":19.7,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.3505,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":19.5,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.2471,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":20.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5965,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":20.2,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.2338,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":20.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.3295,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":30.82,"balanced_skill":10.85,"breadth_skill":8.33,"public_skill":8.85},"frozen_scores":{"balanced_raw":30.82,"balanced_skill":8.85,"breadth_skill":8.33},"categories":[{"id":"knowledge","raw":0.279,"skill":0.0277,"combined":0.0256,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.364,"skill":0.1148,"combined":0.0924,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.3071,"skill":0.12,"combined":0.1707,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.2444,"skill":0.1302,"combined":0.098,"coverage":0.9978,"pending":0,"n":5},{"id":"arts","raw":0.3585,"skill":0.0387,"combined":0.0479,"coverage":0.9999,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.3949,"skill":0.1832,"coverage":1,"random":0.2592,"tracks":[],"combined":0.0814,"private_eq":0.3265},"2":{"raw":0.2745,"skill":0.1622,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.2085,"private_eq":0.3216},"3":{"raw":0.2815,"skill":0.2676,"coverage":0.9961,"random":0.0189,"tracks":[],"combined":0.2032,"private_eq":0.2262},"4":{"raw":0.2795,"skill":0.2702,"coverage":1,"random":0.0127,"tracks":[],"combined":0.2939,"private_eq":0.3098},"5":{"raw":0.0944,"skill":0.0889,"coverage":1,"random":0.006,"tracks":[],"combined":0.3022,"private_eq":0.3133},"6":{"raw":0.4908,"skill":0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.43,"headline":false},{"track":"RouterBench-5shot","score":0.5517,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2983,"skill":0,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.3686,"skill":0.0542,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.4947,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.465,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0237,"skill":0.0161,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0079},"23":{"raw":0.4899,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3517,"skill":0.1356,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2551,"skill":0.0068,"coverage":1,"random":0.25,"tracks":[],"combined":0.0144,"private_eq":0.2685},"28":{"raw":0.5083,"skill":0.0166,"coverage":1,"random":0.5,"tracks":[],"combined":0.1283,"private_eq":0.5686},"29":{"raw":0.4199,"skill":0.2265,"coverage":1,"random":0.25,"tracks":[],"combined":0.2604,"private_eq":0.4509},"30":{"raw":0.1884,"skill":0.0179,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2767,"random":0.25},{"track":"10-choice","raw":0.1001,"random":0.1}],"combined":0.0665,"private_eq":0.2378},"31":{"raw":0.0854,"skill":0.0039,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.4189,"skill":0.0762,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.3746},"33":{"raw":0.0424,"skill":0.0297,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0146,"private_eq":0.0376},"36":{"raw":0.1669,"skill":0.0575,"coverage":1,"random":0.116,"tracks":[],"combined":0.0849,"private_eq":0.1994},"37":{"raw":0.2815,"skill":0.0988,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1415,"private_eq":0.3225},"38":{"raw":0.0391,"skill":0.0084,"coverage":1,"random":0.031,"tracks":[],"combined":0.0079,"private_eq":0.0486},"39":{"raw":0.5894,"skill":0.3961,"coverage":1,"random":0.3201,"tracks":[],"combined":0.2342,"private_eq":0.4846},"40":{"raw":0.4629,"skill":0.309,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.129,"headline":false},{"track":"A · English","score":0.4629,"headline":true},{"track":"C · Arabic pairs","score":0.45,"headline":false},{"track":"C · English pairs","score":0.45,"headline":false}],"combined":0.2654,"private_eq":0.4347},"41":{"raw":0.3513,"skill":0.027,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4869,"skill":0.0018,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3947,"skill":0.0397,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.5036,"skill":0.0072,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"45":{"raw":0.1597,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3484,"skill":0.0544,"coverage":1,"random":0.311,"tracks":[],"combined":0.1523,"private_eq":0.4219},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5},"57":{"raw":0.1553,"skill":0.0499,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.0256,"private_eq":0.1426},"58":{"raw":0.3505,"skill":0.0586,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0251,"private_eq":0.3344},"59":{"raw":0.2471,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5965,"skill":0.193,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1782,"private_eq":0.5933},"62":{"raw":0.2338,"skill":0,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.3295,"skill":0.1619,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.1185,"private_eq":0.302}},"coverage":0.7594,"pending":0.24,"latency_rows":749,"latency_path":"author-submitted LavoirEngine (di_lavoir_engine.py, native=true) in-process on one RTX PRO 6000 Space (rtx-13): ModernBERT-large encoder with SDPA attention (pinned by lavoir/model.py), bf16 autocast, a row's questions length-sorted and batched (64 questions / 65,536 padded tokens per forward); torch 2.14.0+cu130, transformers 5.17.0, laya 0.3.11, lavoir af79b0a","v03":{"engine":"lavoir","pub":8.85,"same_skills":9.9,"new_domains":8.94,"v":10.85,"lab":false,"jev":false,"rank_v":229,"rank_pub":232,"name":"MoganAI Lavoir ModernBERT-large","eq":{"S":8.96,"O":13.69},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":5.38,"new_domains":5.47}}},{"engine":"jeff-uncapped","name":"jeff","long_name":"jeff GLiFormer-large","variant":null,"meta":{"kind":"inference technique","technique":"GLi family","technique_name":"GLiFormer","served_params":575683963,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/logan-markewich/jeff","base_model":"knowledgator/gliformer-large-v1","base_url":"https://huggingface.co/knowledgator/gliformer-large-v1","weights_repo":null,"served_checkpoint":"knowledgator/gliformer-large-v1"},"completed":110201,"counts":{"ok":110201},"lifted":true,"calibration":{"acc":0.3824,"conf":0.4197,"ece":0.0969,"brier":0.6823,"over95":0,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.1485,"conf":0.037,"acc":0.2279},{"lo":0.1,"w":0.0306,"conf":0.146,"acc":0.1938},{"lo":0.2,"w":0.0665,"conf":0.2737,"acc":0.2396},{"lo":0.3,"w":0.2018,"conf":0.3492,"acc":0.3442},{"lo":0.4,"w":0.1022,"conf":0.4394,"acc":0.4299},{"lo":0.5,"w":0.2356,"conf":0.5454,"acc":0.4335},{"lo":0.6,"w":0.1335,"conf":0.646,"acc":0.5012},{"lo":0.7,"w":0.0683,"conf":0.7403,"acc":0.5444},{"lo":0.8,"w":0.0129,"conf":0.8286,"acc":0.549},{"lo":0.9,"w":0.0002,"conf":0.9121,"acc":0.0968}],"areas":{"knowledge":{"ece":0.0713,"acc":0.3278,"conf":0.3991},"language":{"ece":0.1031,"acc":0.4133,"conf":0.5119},"retrieval":{"ece":0.3082,"acc":0.553,"conf":0.2595},"tools":{"ece":0.1054,"acc":0.3014,"conf":0.3827},"arts":{"ece":0.096,"acc":0.3592,"conf":0.4491}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":21.8,"p95":67.6,"mean":33.1},"results":{"1":{"score":0.2149,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":35},"2":{"score":0.1763,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":243.5},"3":{"score":0.0315,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.6099,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":21.4},"5":{"score":0.443,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":23},"6":{"score":0.5384,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":46.6},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4479,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":49.7},"11":{"score":0.2126,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2934,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":19.2},"20":{"score":0.4814,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.2},"21":{"score":0.4665,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":18.8},"22":{"score":0.024,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":112.3},"23":{"score":0.5044,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.4},"24":{"score":0.316,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.2},"25":{"score":0.2449,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5059,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.9},"27":{"score":0.3166,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.9},"28":{"score":0.5036,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":18.6},"29":{"score":0.3455,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":19.6},"30":{"score":0.1865,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.086,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.6},"32":{"score":0.4787,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0158,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":29},"36":{"score":0.0808,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.2128,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.2},"38":{"score":0.0316,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":102.1},"39":{"score":0.7487,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2832,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":19.4,"tracks":[{"track":"A · Arabic","score":0.0679,"headline":false},{"track":"A · English","score":0.2832,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.37,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4047,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":19.2},"42":{"score":0.5504,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":28.2},"43":{"score":0.3895,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4956,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.3},"45":{"score":0.1657,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3252,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":29.3},"56":{"score":0.5025,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":25.8,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1423,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":20.1,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.335,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":21.1,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.171,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.1,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5677,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":27.3,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.2333,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":33.4,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.1875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":29.6,"balanced_skill":6.21,"breadth_skill":6.88,"public_skill":8.27},"frozen_scores":{"balanced_raw":29.6,"balanced_skill":8.27,"breadth_skill":6.88},"categories":[{"id":"knowledge","raw":0.2768,"skill":0.0267,"combined":0.009,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3371,"skill":0.1164,"combined":0.0776,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4005,"skill":0.2131,"combined":0.1718,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1306,"skill":0.0119,"combined":0,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3319,"skill":0.0092,"combined":0.0096,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.2149,"skill":0,"coverage":1,"random":0.2592,"tracks":[],"combined":0,"private_eq":0.2592},"2":{"raw":0.1763,"skill":0.0488,"coverage":1,"random":0.1341,"tracks":[],"combined":0,"private_eq":0.1341},"3":{"raw":0.0315,"skill":0.0128,"coverage":1,"random":0.0189,"tracks":[],"combined":0.0247,"private_eq":0.0531},"4":{"raw":0.6099,"skill":0.6049,"coverage":1,"random":0.0127,"tracks":[],"combined":0.4769,"private_eq":0.4884},"5":{"raw":0.443,"skill":0.4396,"coverage":1,"random":0.006,"tracks":[],"combined":0.2344,"private_eq":0.2467},"6":{"raw":0.5385,"skill":0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.4941,"headline":false},{"track":"RouterBench-5shot","score":0.5828,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2126,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2934,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.4807,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.4665,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0204,"skill":0.0127,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0123},"23":{"raw":0.511,"skill":0.0221,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.316,"skill":0.088,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0.0074,"private_eq":0.2633},"28":{"raw":0.5036,"skill":0.0072,"coverage":1,"random":0.5,"tracks":[],"combined":0.0627,"private_eq":0.5362},"29":{"raw":0.3455,"skill":0.1273,"coverage":1,"random":0.25,"tracks":[],"combined":0.1515,"private_eq":0.3701},"30":{"raw":0.1865,"skill":0.0137,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2585,"random":0.25},{"track":"10-choice","raw":0.1145,"random":0.1}],"combined":0.0546,"private_eq":0.2281},"31":{"raw":0.086,"skill":0.0045,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0844},"32":{"raw":0.4787,"skill":0.1712,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0158,"skill":0.0027,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0042,"private_eq":0.0275},"36":{"raw":0.0808,"skill":0,"coverage":1,"random":0.116,"tracks":[],"combined":0.0529,"private_eq":0.1714},"37":{"raw":0.2128,"skill":0.0127,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0316,"skill":0.0006,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.0326},"39":{"raw":0.7487,"skill":0.6304,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4642,"private_eq":0.6392},"40":{"raw":0.2832,"skill":0.0778,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0679,"headline":false},{"track":"A · English","score":0.2832,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.37,"headline":false}],"combined":0.0591,"private_eq":0.2762},"41":{"raw":0.4047,"skill":0.107,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5504,"skill":0.1253,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3895,"skill":0.0313,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.4956,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"45":{"raw":0.1657,"skill":0.0019,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3252,"skill":0.0206,"coverage":1,"random":0.311,"tracks":[],"combined":0.0726,"private_eq":0.3676},"56":{"raw":0.5025,"skill":0.005,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0318,"private_eq":0.5209},"57":{"raw":0.1423,"skill":0.0353,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.1109},"58":{"raw":0.335,"skill":0.0361,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.005,"private_eq":0.3207},"59":{"raw":0.171,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5677,"skill":0.1354,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1954,"private_eq":0.6018},"62":{"raw":0.2333,"skill":0,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.1875,"skill":0,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.0021,"private_eq":0.21}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author jeff encoder with flash attention, long-context lift as board (compile off in the package); venv-jeff","v03":{"engine":"jeff-uncapped","pub":8.27,"same_skills":6.86,"new_domains":3.29,"v":6.21,"lab":false,"jev":false,"rank_v":237,"rank_pub":233,"name":"jeff GLiFormer-large","eq":{"S":5.88,"O":6.7},"rank_tie":57,"tied":true,"parts":{"public":0,"same_skills":3.53,"new_domains":2.68}}},{"engine":"lfm2600","name":"LFM2.5-2.6B-RLCD","long_name":"LFM2.5-2.6B-RLCD","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2697198592,"params_estimated":false,"params_from_name":false,"trained_bytes":5394427456,"model_url":"https://huggingface.co/monotykamary/LFM2.5-2.6B-RLCD","code_url":null,"base_model":"LiquidAI/LFM2.5-2.6B-Base","base_url":"https://huggingface.co/LiquidAI/LFM2.5-2.6B-Base","weights_repo":"monotykamary/LFM2.5-2.6B-RLCD","served_checkpoint":null},"completed":109668,"counts":{"ok":109668,"unsupported":533},"lifted":true,"calibration":{"acc":0.3937,"conf":0.6482,"ece":0.2545,"brier":0.7921,"over95":0.0508,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.0001,"conf":0.0821,"acc":0},{"lo":0.1,"w":0.0102,"conf":0.1744,"acc":0.0494},{"lo":0.2,"w":0.0546,"conf":0.2555,"acc":0.0885},{"lo":0.3,"w":0.0881,"conf":0.3505,"acc":0.1509},{"lo":0.4,"w":0.1035,"conf":0.4478,"acc":0.2711},{"lo":0.5,"w":0.1837,"conf":0.5506,"acc":0.4199},{"lo":0.6,"w":0.136,"conf":0.6498,"acc":0.4539},{"lo":0.7,"w":0.1393,"conf":0.7478,"acc":0.4475},{"lo":0.8,"w":0.1117,"conf":0.8491,"acc":0.4569},{"lo":0.9,"w":0.1729,"conf":0.9643,"acc":0.5483}],"areas":{"knowledge":{"ece":0.181,"acc":0.4212,"conf":0.6022},"language":{"ece":0.2881,"acc":0.4095,"conf":0.6976},"retrieval":{"ece":0.2867,"acc":0.3718,"conf":0.6584},"tools":{"ece":0.5343,"acc":0.2426,"conf":0.7769},"arts":{"ece":0.1619,"acc":0.413,"conf":0.5749}}},"gaps":{"share":0.0044,"reasons":[{"reason":"request exceeds conservative GPU memory budget; reduce prompt or branch batch size","share":0.0044}]},"latency":{"median":39.3,"p95":95.4,"mean":54.2},"results":{"1":{"score":0.3872,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":46.5},"2":{"score":0.1217,"jev":null,"metric":"nDCG@10","requests":685,"answered":640,"unsupported":45,"errors":0,"median_ms":null},"3":{"score":0.0177,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.1778,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":67},"5":{"score":0.0732,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":95.3},"6":{"score":0.5927,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":70.7},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5294,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":51.5},"11":{"score":0.3112,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3486,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":38.5},"20":{"score":0.6016,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.5},"21":{"score":0.4981,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":38.4},"22":{"score":0,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":165.1},"23":{"score":0.512,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.7},"24":{"score":0.4916,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":38.6},"25":{"score":0.352,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.8131,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":38.4},"27":{"score":0.6596,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":38.6},"28":{"score":0.5083,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":38.2},"29":{"score":0.4556,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":39.2},"30":{"score":0.1804,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1048,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":56},"32":{"score":0.4707,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0006,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":54.7},"36":{"score":0.1825,"jev":null,"metric":"nDCG@10","requests":220,"answered":128,"unsupported":92,"errors":0,"median_ms":null},"37":{"score":0.2226,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39},"38":{"score":0.0561,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":422.6},"39":{"score":0.3685,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2629,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":38.2,"tracks":[{"track":"A · Arabic","score":0.2773,"headline":false},{"track":"A · English","score":0.2629,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3628,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":38.4},"42":{"score":0.3342,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":41.3},"43":{"score":0.3684,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5158,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.3},"45":{"score":0.1357,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4075,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":48.8},"56":{"score":0.6695,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":50.9,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.2461,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.5,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.4265,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":38.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5177,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":44.9,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5248,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":38.8,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.2738,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":44.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.2311,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":28.87,"balanced_skill":8.18,"breadth_skill":6.67,"public_skill":6.78},"frozen_scores":{"balanced_raw":28.87,"balanced_skill":6.78,"breadth_skill":6.67},"categories":[{"id":"knowledge","raw":0.3036,"skill":0.0658,"combined":0.0661,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3381,"skill":0.0591,"combined":0.0544,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2796,"skill":0.1053,"combined":0.0493,"coverage":0.924,"pending":0,"n":6},{"id":"tools","raw":0.1617,"skill":0.0443,"combined":0.081,"coverage":0.9878,"pending":0,"n":5},{"id":"arts","raw":0.3727,"skill":0.063,"combined":0.1144,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.3872,"skill":0.1728,"coverage":1,"random":0.2592,"tracks":[],"combined":0.1768,"private_eq":0.3964},"2":{"raw":0.1137,"skill":0,"coverage":0.9343,"random":0.1341,"tracks":[],"combined":0.064,"private_eq":0.1979},"3":{"raw":0.0177,"skill":0,"coverage":1,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0226},"4":{"raw":0.1778,"skill":0.1672,"coverage":1,"random":0.0127,"tracks":[],"combined":0.0379,"private_eq":0.06},"5":{"raw":0.0732,"skill":0.0676,"coverage":1,"random":0.006,"tracks":[],"combined":0.0189,"private_eq":0.0349},"6":{"raw":0.5927,"skill":0.0442,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5491,"headline":false},{"track":"RouterBench-5shot","score":0.6363,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.3112,"skill":0.0039,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3486,"skill":0.0243,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5994,"skill":0.1988,"coverage":1,"random":0.5,"tracks":[],"combined":0.242,"private_eq":0.6248},"21":{"raw":0.4981,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.1157,"private_eq":0.5624},"22":{"raw":0,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.4999,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5022,"skill":0.3363,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.352,"skill":0.1361,"coverage":1,"random":0.25,"tracks":[],"combined":0.0411,"private_eq":0.2883},"28":{"raw":0.5083,"skill":0.0166,"coverage":1,"random":0.5,"tracks":[],"combined":0.1637,"private_eq":0.5861},"29":{"raw":0.4556,"skill":0.2741,"coverage":1,"random":0.25,"tracks":[],"combined":0.3148,"private_eq":0.4912},"30":{"raw":0.1804,"skill":0.0066,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2547,"random":0.25},{"track":"10-choice","raw":0.1061,"random":0.1}],"combined":0.0485,"private_eq":0.2231},"31":{"raw":0.1048,"skill":0.025,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0171,"private_eq":0.107},"32":{"raw":0.4707,"skill":0.1585,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0006,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.1062,"skill":0,"coverage":0.5818,"random":0.116,"tracks":[],"combined":0.0194,"private_eq":0.1421},"37":{"raw":0.2226,"skill":0.0249,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0561,"skill":0.0259,"coverage":1,"random":0.031,"tracks":[],"combined":0.0498,"private_eq":0.0888},"39":{"raw":0.3685,"skill":0.0712,"coverage":1,"random":0.3201,"tracks":[],"combined":0,"private_eq":0.3201},"40":{"raw":0.2629,"skill":0.0518,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2773,"headline":false},{"track":"A · English","score":0.2629,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}],"combined":0.0842,"private_eq":0.2955},"41":{"raw":0.3628,"skill":0.0442,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.3342,"skill":0,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3684,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0561,"private_eq":0.4112},"44":{"raw":0.5158,"skill":0.0316,"coverage":1,"random":0.5,"tracks":[],"combined":0.1335,"private_eq":0.5712},"45":{"raw":0.1357,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4075,"skill":0.1401,"coverage":1,"random":0.311,"tracks":[],"combined":0.174,"private_eq":0.4367},"56":{"raw":0.6695,"skill":0.339,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.2308,"private_eq":0.6193},"57":{"raw":0.2461,"skill":0.1521,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.1633,"private_eq":0.2637},"58":{"raw":0.4265,"skill":0.1687,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0319,"private_eq":0.339},"59":{"raw":0.5177,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5248,"skill":0.0496,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0138,"private_eq":0.512},"62":{"raw":0.2738,"skill":0.0317,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.1797,"private_eq":0.391},"64":{"raw":0.2311,"skill":0.0389,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.0506,"private_eq":0.2483}},"coverage":0.7406,"pending":0.24,"latency_rows":748,"latency_path":"author pcd Engine.constrained, board guard lifts, fleet venv torch 2.14.0+cu130","v03":{"engine":"lfm2600","pub":6.78,"same_skills":7.65,"new_domains":6.31,"v":8.18,"lab":false,"jev":false,"rank_v":233,"rank_pub":236,"name":"LFM2.5-2.6B-RLCD","eq":{"S":6.68,"O":10.43},"rank_tie":56,"tied":true,"parts":{"public":0,"same_skills":4.01,"new_domains":4.17}}},{"engine":"sifr-0.8b-v3.1","name":"Sifr 0.8B v3.1","long_name":"Sifr 0.8B v3.1","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":870000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/mohamedlotfy50/sifr-0.8b-v3","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5695,"conf":0.5937,"ece":0.1203,"brier":0.5666,"over95":0.0185,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.1124,"conf":0.0459,"acc":0.2566},{"lo":0.1,"w":0.041,"conf":0.1374,"acc":0.6566},{"lo":0.2,"w":0.0193,"conf":0.2515,"acc":0.412},{"lo":0.3,"w":0.0605,"conf":0.3568,"acc":0.3005},{"lo":0.4,"w":0.0797,"conf":0.448,"acc":0.4077},{"lo":0.5,"w":0.1861,"conf":0.5517,"acc":0.4837},{"lo":0.6,"w":0.1115,"conf":0.6494,"acc":0.5231},{"lo":0.7,"w":0.0957,"conf":0.7505,"acc":0.6458},{"lo":0.8,"w":0.0894,"conf":0.8506,"acc":0.7531},{"lo":0.9,"w":0.2045,"conf":0.9678,"acc":0.8691}],"areas":{"knowledge":{"ece":0.0926,"acc":0.541,"conf":0.6255},"language":{"ece":0.1017,"acc":0.6721,"conf":0.7731},"retrieval":{"ece":0.3251,"acc":0.6538,"conf":0.3619},"tools":{"ece":0.1387,"acc":0.5573,"conf":0.4762},"arts":{"ece":0.0771,"acc":0.3875,"conf":0.457}}},"gaps":null,"latency":{"median":34.7,"p95":1042.3,"mean":286.3},"results":{"1":{"score":0.7957,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":68.9},"2":{"score":0.5058,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":690.8},"3":{"score":0.4075,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2146.6},"4":{"score":0.6801,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":337.5},"5":{"score":0.7281,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":980.9},"6":{"score":0.595,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":627.9},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":11723.3},"10":{"score":0.6038,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":149.3},"11":{"score":0.2929,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":10621.4},"12":{"score":0.5158,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":15.7},"20":{"score":0.5026,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.8},"21":{"score":0.4992,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":45.2},"22":{"score":0.0325,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3572.8},"23":{"score":0.5242,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.8},"24":{"score":0.4863,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.5},"25":{"score":0.2704,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":16.5},"26":{"score":0.8586,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":14.8},"27":{"score":0.6877,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":15.3},"28":{"score":0.5912,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":14.6},"29":{"score":0.7302,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":19.8},"30":{"score":0.1929,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0772,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":1400.8},"32":{"score":0.5625,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":42.3},"33":{"score":0.3327,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":176.9},"36":{"score":0.2509,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":868.3},"37":{"score":0.3139,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.4},"38":{"score":0.0928,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":942.7},"39":{"score":0.8566,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":29.5},"40":{"score":0.0769,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":14.9},"41":{"score":0.48,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":15.8},"42":{"score":0.3992,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":108.3},"43":{"score":0.3386,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":46.6},"44":{"score":0.6656,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.9},"45":{"score":0.1577,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":20},"50":{"score":0.3741,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":111},"56":{"score":0.999,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":232.8},"57":{"score":0.1982,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":16.3},"58":{"score":0.4453,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":44.1},"59":{"score":0.5177,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":35.1},"61":{"score":0.4973,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":59.6},"62":{"score":0.3757,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":31.8},"64":{"score":0.3409,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":14.9}},"benchmarks":{"1":{"raw":0.7957,"skill":0.7243,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.6558,"combined":0.531},"2":{"raw":0.5058,"skill":0.4292,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.6119,"combined":0.5476},"3":{"raw":0.4075,"skill":0.3961,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.2669,"combined":0.2452},"4":{"raw":0.6801,"skill":0.676,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.5854,"combined":0.5762},"5":{"raw":0.7281,"skill":0.7265,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.3467,"combined":0.3362},"6":{"raw":0.595,"skill":0.0504,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0,"combined":0},"10":{"raw":0.6038,"skill":0.3408,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.2929,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5158,"skill":0.2747,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.4715,"combined":0.2002},"20":{"raw":0.501,"skill":0.0019,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5283,"combined":0.0467},"21":{"raw":0.4992,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5226,"combined":0.0352},"22":{"raw":0.0332,"skill":0.0256,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0476,"combined":0.0301},"23":{"raw":0.5255,"skill":0.051,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4863,"skill":0.3151,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2704,"skill":0.0272,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2581,"combined":0.0004},"26":{"raw":0.8586,"skill":0.8114,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.6877,"skill":0.5835,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5912,"skill":0.1824,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5337,"combined":0.0577},"29":{"raw":0.7302,"skill":0.6403,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5096,"combined":0.3396},"30":{"raw":0.1929,"skill":0.0227,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2745,"random":0.25},{"track":"10-choice","raw":0.1114,"random":0.1}],"private_eq":0.2272,"combined":0.0535},"31":{"raw":0.0772,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0873,"combined":0},"32":{"raw":0.5625,"skill":0.3045,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0.3327,"skill":0.3238,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.2832,"combined":0.2663},"36":{"raw":0.2509,"skill":0.1526,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.2638,"combined":0.1586},"37":{"raw":0.3139,"skill":0.1395,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2284,"combined":0.0221},"38":{"raw":0.0928,"skill":0.0638,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.1421,"combined":0.1055},"39":{"raw":0.8566,"skill":0.7891,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.5082,"combined":0.2693},"40":{"raw":0.0769,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.0769,"headline":true},{"track":"C · Arabic pairs","score":0.48,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}],"private_eq":0.2227,"combined":0},"41":{"raw":0.48,"skill":0.2201,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.4816,"combined":0.2145},"42":{"raw":0.3992,"skill":0,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.486,"combined":0},"43":{"raw":0.3386,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.4477,"combined":0.1147},"44":{"raw":0.6656,"skill":0.3312,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.595,"combined":0.1817},"45":{"raw":0.1577,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3741,"skill":0.0916,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.3973,"combined":0.1162},"56":{"raw":0.999,"skill":0.998,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7049,"combined":0.404},"57":{"raw":0.1982,"skill":0.0982,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2325,"combined":0.1278},"58":{"raw":0.4453,"skill":0.196,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3422,"combined":0.0366},"59":{"raw":0.5177,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.4973,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5064,"combined":0.0024},"62":{"raw":0.3757,"skill":0.1676,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4215,"combined":0.2208},"64":{"raw":0.3409,"skill":0.1761,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.371,"combined":0.2057}},"coverage":1,"pending":0,"scores":{"balanced_raw":43.74,"balanced_skill":14.53,"breadth_skill":23.11,"public_skill":25.65},"frozen_scores":{"balanced_raw":43.74,"balanced_skill":25.65,"breadth_skill":23.11},"categories":[{"id":"knowledge","raw":0.3471,"skill":0.1322,"combined":0.0804,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.448,"skill":0.231,"combined":0.1435,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5759,"skill":0.4551,"combined":0.2597,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4306,"skill":0.3595,"combined":0.3128,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.379,"skill":0.0577,"combined":0.0868,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sifr-0.8b-v3.1","pub":25.65,"same_skills":18.93,"new_domains":5.31,"v":14.53,"lab":false,"jev":false,"rank_v":216,"rank_pub":204,"name":"Sifr 0.8B v3.1","eq":{"S":18.09,"O":9.2},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":10.86,"new_domains":3.68}}},{"engine":"laya-1e28ac20","name":"Laya","long_name":"Laya","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":421293830,"params_estimated":false,"params_from_name":false,"trained_bytes":2329053944,"model_url":"https://huggingface.co/convaiinnovations/laya","code_url":"https://github.com/NandhaKishorM/laya","base_model":null,"base_url":null,"weights_repo":"convaiinnovations/laya","served_checkpoint":null},"completed":109767,"counts":{"ok":109767,"unsupported":320},"lifted":false,"calibration":{"acc":0.377,"conf":0.5172,"ece":0.1402,"brier":0.7211,"over95":0.0116,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0485,"conf":0.0771,"acc":0.0455},{"lo":0.1,"w":0.0375,"conf":0.1433,"acc":0.1199},{"lo":0.2,"w":0.0673,"conf":0.2665,"acc":0.1752},{"lo":0.3,"w":0.2026,"conf":0.3479,"acc":0.2867},{"lo":0.4,"w":0.108,"conf":0.4446,"acc":0.3428},{"lo":0.5,"w":0.2027,"conf":0.5458,"acc":0.4187},{"lo":0.6,"w":0.1071,"conf":0.6457,"acc":0.4688},{"lo":0.7,"w":0.0765,"conf":0.7488,"acc":0.503},{"lo":0.8,"w":0.0755,"conf":0.8519,"acc":0.5633},{"lo":0.9,"w":0.0742,"conf":0.9455,"acc":0.6364}],"areas":{"knowledge":{"ece":0.1577,"acc":0.3142,"conf":0.4719},"language":{"ece":0.1218,"acc":0.545,"conf":0.6648},"retrieval":{"ece":0.097,"acc":0.2769,"conf":0.3529},"tools":{"ece":0.2475,"acc":0.2407,"conf":0.475},"arts":{"ece":0.1334,"acc":0.3744,"conf":0.5078}}},"gaps":{"share":0.0027,"reasons":[{"reason":"instructions longer than its native budget (no truncation allowed)","share":0.0026},{"reason":"prompt longer than its context window (no truncation allowed)","share":0},{"reason":"answer options longer than its per-option budget (no truncation allowed)","share":0}]},"latency":{"median":5.8,"p95":222.5,"mean":32.7},"results":{"1":{"score":0.3813,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":38.7},"2":{"score":0.1278,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.1146,"jev":null,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":null},"4":{"score":0.1429,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":12.3},"5":{"score":0.0319,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":17.2},"6":{"score":0.5714,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":26.5},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4241,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":11.4},"11":{"score":0.2901,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":null},"12":{"score":0.4866,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":4.6},"20":{"score":0.5158,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":4.6},"21":{"score":0.4722,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":4.2},"22":{"score":0.051,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":259.9},"23":{"score":0.5234,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.8},"24":{"score":0.3068,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":4.5},"25":{"score":0.2755,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.4697,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":4.3},"27":{"score":0.2858,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":4.4},"28":{"score":0.5051,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":4.1},"29":{"score":0.3312,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":5.2},"30":{"score":0.1694,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0768,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.2},"32":{"score":0.4322,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.003,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":276.8},"36":{"score":0.199,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.244,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.8},"38":{"score":0.0345,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":77.9},"39":{"score":0.6098,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.149,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":4.2,"tracks":[{"track":"A · Arabic","score":0.0184,"headline":false},{"track":"A · English","score":0.149,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.41,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.4053,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":4.8},"42":{"score":0.4772,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":6.5},"43":{"score":0.4018,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5288,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":4.5},"45":{"score":0.14,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"50":{"score":0.3341,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":9.6},"56":{"score":0.5015,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":26.4,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1361,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":5,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.341,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":4.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.4882,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":8.6,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5575,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":6.4,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.1194,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":8.9,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.2708,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":28.26,"balanced_skill":9.21,"breadth_skill":5.97,"public_skill":6.19},"frozen_scores":{"balanced_raw":28.26,"balanced_skill":6.19,"breadth_skill":5.97},"categories":[{"id":"knowledge","raw":0.2741,"skill":0.029,"combined":0.0138,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.3652,"skill":0.1005,"combined":0.1456,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.2654,"skill":0.0715,"combined":0.0922,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1557,"skill":0.0582,"combined":0.0794,"coverage":0.9978,"pending":0,"n":5},{"id":"arts","raw":0.3579,"skill":0.0347,"combined":0.0697,"coverage":0.9999,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.3813,"skill":0.1648,"coverage":1,"random":0.2592,"tracks":[],"combined":0.17,"private_eq":0.3914},"2":{"raw":0.1269,"skill":0,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.1152,"private_eq":0.2417},"3":{"raw":0.1141,"skill":0.0971,"coverage":0.9961,"random":0.0189,"tracks":[],"combined":0.1088,"private_eq":0.1346},"4":{"raw":0.1429,"skill":0.1319,"coverage":1,"random":0.0127,"tracks":[],"combined":0.1325,"private_eq":0.1523},"5":{"raw":0.0319,"skill":0.0261,"coverage":1,"random":0.006,"tracks":[],"combined":0.029,"private_eq":0.0448},"6":{"raw":0.5714,"skill":0.0397,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5623,"headline":false},{"track":"RouterBench-5shot","score":0.5805,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2875,"skill":0,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.4866,"skill":0.231,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2401,"private_eq":0.4978},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5136,"skill":0.0271,"coverage":1,"random":0.5,"tracks":[],"combined":0.1262,"private_eq":0.5676},"21":{"raw":0.4722,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0472,"skill":0.0398,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.0345,"private_eq":0.052},"23":{"raw":0.5095,"skill":0.019,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3091,"skill":0.0788,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"28":{"raw":0.5051,"skill":0.0102,"coverage":1,"random":0.5,"tracks":[],"combined":0.083,"private_eq":0.5462},"29":{"raw":0.3312,"skill":0.1083,"coverage":1,"random":0.25,"tracks":[],"combined":0.1911,"private_eq":0.3995},"30":{"raw":0.1694,"skill":0.0013,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2365,"random":0.25},{"track":"10-choice","raw":0.1024,"random":0.1}],"combined":0.0249,"private_eq":0.2039},"31":{"raw":0.0768,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0854},"32":{"raw":0.4322,"skill":0.0973,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.003,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.199,"skill":0.0938,"coverage":1,"random":0.116,"tracks":[],"combined":0.077,"private_eq":0.1925},"37":{"raw":0.244,"skill":0.0517,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1752,"private_eq":0.3491},"38":{"raw":0.0345,"skill":0.0036,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.0357},"39":{"raw":0.6098,"skill":0.4261,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6524,"private_eq":0.7657},"40":{"raw":0.149,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0184,"headline":false},{"track":"A · English","score":0.149,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.41,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4053,"skill":0.108,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0901,"private_eq":0.3996},"42":{"raw":0.4772,"skill":0,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.5288,"skill":0.0576,"coverage":1,"random":0.5,"tracks":[],"combined":0.0425,"private_eq":0.5262},"45":{"raw":0.1397,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.3341,"skill":0.0336,"coverage":1,"random":0.311,"tracks":[],"combined":0.1379,"private_eq":0.4121},"56":{"raw":0.5015,"skill":0.003,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0146,"private_eq":0.5124},"57":{"raw":0.1361,"skill":0.0283,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.0378,"private_eq":0.1534},"58":{"raw":0.341,"skill":0.0448,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.3101},"59":{"raw":0.4882,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5575,"skill":0.115,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1329,"private_eq":0.5709},"62":{"raw":0.1194,"skill":0,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.2708,"skill":0.0885,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.0603,"private_eq":0.256}},"coverage":0.7594,"pending":0.24,"latency_rows":749,"latency_path":"laya Agent.accelerate(use_graphs=True): TileLang fused kernels + CUDA graphs per shape bucket, built with the CUDA 13.0 nvcc from pip (nvidia-cuda-nvcc 13.0) because the card image ships nvcc 12.3 without sm_120a; measured with a warm TileLang JIT cache (a first pass with a cold cache had 72 rows of ~5.3 s kernel compilation, kept as latency-v1-full.fast-coldjit); p95 still includes first-use CUDA graph capture per bucket. Stock path (19.8 / 51.2 ms) kept as latency-v1-full.stock","v03":{"engine":"laya-1e28ac20","pub":6.19,"same_skills":9.08,"new_domains":6.63,"v":9.21,"lab":false,"jev":false,"rank_v":231,"rank_pub":237,"name":"Laya","eq":{"S":8.13,"O":10.84},"rank_tie":55,"tied":true,"parts":{"public":0,"same_skills":4.88,"new_domains":4.33}}},{"engine":"clm-v0.1-8b","name":"CLM-v0.1-8B","long_name":"CLM-v0.1-8B Qwen3-8B heads","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":8190735360,"params_estimated":false,"params_from_name":false,"trained_bytes":75557149,"model_url":"https://huggingface.co/Contrastive-LM/CLM-v0.1-8B","code_url":"https://github.com/Contrastive-LM/CLM","base_model":"Qwen/Qwen3-8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3-8B-Base","weights_repo":"Contrastive-LM/CLM-v0.1-8B","served_checkpoint":"Contrastive-LM/CLM-v0.1-8B"},"completed":109759,"counts":{"ok":109759},"lifted":false,"calibration":{"acc":0.3582,"conf":0.6809,"ece":0.3227,"brier":0.9042,"over95":0.1194,"n":35088,"benchmarks":32,"rel":[{"lo":0,"w":0.0011,"conf":0.0925,"acc":0.0332},{"lo":0.1,"w":0.0361,"conf":0.1671,"acc":0.05},{"lo":0.2,"w":0.0512,"conf":0.242,"acc":0.0698},{"lo":0.3,"w":0.0707,"conf":0.361,"acc":0.1182},{"lo":0.4,"w":0.068,"conf":0.4559,"acc":0.2311},{"lo":0.5,"w":0.1531,"conf":0.5496,"acc":0.39},{"lo":0.6,"w":0.1268,"conf":0.6502,"acc":0.4558},{"lo":0.7,"w":0.1105,"conf":0.7488,"acc":0.4026},{"lo":0.8,"w":0.1121,"conf":0.8508,"acc":0.4863},{"lo":0.9,"w":0.2704,"conf":0.966,"acc":0.4149}],"areas":{"knowledge":{"ece":0.2685,"acc":0.3857,"conf":0.6542},"language":{"ece":0.3593,"acc":0.4277,"conf":0.7865},"retrieval":{"ece":0.4925,"acc":0.1219,"conf":0.6144},"tools":{"ece":0.234,"acc":0.3779,"conf":0.5752},"arts":{"ece":0.3165,"acc":0.3497,"conf":0.6662}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":46.8,"p95":130.9,"mean":56.5},"results":{"1":{"score":0.389,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":62},"2":{"score":0.4031,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":289.9},"3":{"score":0.061,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0652,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":19.2},"5":{"score":0.0889,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":20.5},"6":{"score":0.3932,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":109.4},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1001,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":68.1},"11":{"score":0.2011,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.3156,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":18.8},"20":{"score":0.3294,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.3},"21":{"score":0.5556,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":29.6},"22":{"score":0,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":120.1},"23":{"score":0.508,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.6},"24":{"score":0.358,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":46.5},"25":{"score":0.2041,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.564,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":45.4},"27":{"score":0.4147,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":45.6},"28":{"score":0.5083,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":44.5},"29":{"score":0.4035,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":50.3},"30":{"score":0.1751,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0818,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":61.5},"32":{"score":0.3457,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0558,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":67.3},"36":{"score":0.2915,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.0497,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.5},"38":{"score":0.031,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":195.9},"39":{"score":0.3335,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.3122,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":16.9,"tracks":[{"track":"A · Arabic","score":0.3401,"headline":false},{"track":"A · English","score":0.3122,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.57,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2271,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":20.5},"42":{"score":0.5649,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":35},"43":{"score":0.4281,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5394,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.7},"45":{"score":0.1577,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3711,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":72.1},"56":{"score":0.6505,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":49.4,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1466,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":50.3,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.308,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":46.8,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5177,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":41,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.52,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":29.6,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.3869,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":67.5,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.2538,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":28.52,"balanced_skill":6.49,"breadth_skill":6.92,"public_skill":7.41},"frozen_scores":{"balanced_raw":28.52,"balanced_skill":7.41,"breadth_skill":6.92},"categories":[{"id":"knowledge","raw":0.268,"skill":0.0239,"combined":0.0009,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3245,"skill":0.0568,"combined":0.0571,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2659,"skill":0.1126,"combined":0.046,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.2463,"skill":0.1398,"combined":0.1227,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3375,"skill":0.0516,"combined":0.0151,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.389,"skill":0.1752,"coverage":1,"random":0.2592,"tracks":[],"combined":0.1768,"private_eq":0.3964},"2":{"raw":0.4031,"skill":0.3107,"coverage":1,"random":0.1341,"tracks":[],"combined":0.1571,"private_eq":0.2776},"3":{"raw":0.061,"skill":0.0429,"coverage":1,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0.0652,"skill":0.0532,"coverage":1,"random":0.0127,"tracks":[],"combined":0.0177,"private_eq":0.0402},"5":{"raw":0.0889,"skill":0.0834,"coverage":1,"random":0.006,"tracks":[],"combined":0.0314,"private_eq":0.0472},"6":{"raw":0.3932,"skill":0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.2747,"headline":false},{"track":"RouterBench-5shot","score":0.5118,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2011,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3156,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.3226,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.5556,"skill":0.1111,"coverage":1,"random":0.5,"tracks":[],"combined":0.0856,"private_eq":0.5475},"22":{"raw":0,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5219,"skill":0.0437,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3646,"skill":0.1529,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2041,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0.0298,"private_eq":0.2799},"28":{"raw":0.5083,"skill":0.0166,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"29":{"raw":0.4035,"skill":0.2047,"coverage":1,"random":0.25,"tracks":[],"combined":0.3841,"private_eq":0.5426},"30":{"raw":0.1751,"skill":0.0034,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2441,"random":0.25},{"track":"10-choice","raw":0.1061,"random":0.1}],"combined":0.0019,"private_eq":0.1851},"31":{"raw":0.0818,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0873},"32":{"raw":0.3457,"skill":0,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0558,"skill":0.0433,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.2915,"skill":0.1985,"coverage":1,"random":0.116,"tracks":[],"combined":0.0332,"private_eq":0.1542},"37":{"raw":0.0497,"skill":0,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.031,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.3335,"skill":0.0197,"coverage":1,"random":0.3201,"tracks":[],"combined":0,"private_eq":0.3201},"40":{"raw":0.3122,"skill":0.1152,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3401,"headline":false},{"track":"A · English","score":0.3122,"headline":true},{"track":"C · Arabic pairs","score":0.52,"headline":false},{"track":"C · English pairs","score":0.57,"headline":false}],"combined":0.0841,"private_eq":0.2954},"41":{"raw":0.2271,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5649,"skill":0.1535,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4281,"skill":0.0926,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.5394,"skill":0.0788,"coverage":1,"random":0.5,"tracks":[],"combined":0.0372,"private_eq":0.5236},"45":{"raw":0.1577,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3711,"skill":0.0873,"coverage":1,"random":0.311,"tracks":[],"combined":0,"private_eq":0.311},"56":{"raw":0.6505,"skill":0.301,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1962,"private_eq":0.6022},"57":{"raw":0.1466,"skill":0.0402,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.1172},"58":{"raw":0.308,"skill":0,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.3101},"59":{"raw":0.5177,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.52,"skill":0.04,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0194,"private_eq":0.5148},"62":{"raw":0.3869,"skill":0.1825,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.3166,"private_eq":0.4925},"64":{"raw":0.2538,"skill":0.0673,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.0216,"private_eq":0.2254}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"clm.Engine.answer against the owned vLLM pooling server with CUDA graphs (JEV_CLM_GRAPHS=1; the author script uses --enforce-eager: eager measured 58.5 / 133.2 ms vs graphs 46.8 / 130.9), prefix caching on, gpu util 0.85; eager run kept as latency-v1-full.eager","v03":{"engine":"clm-v0.1-8b","pub":7.41,"same_skills":5.96,"new_domains":4.94,"v":6.49,"lab":false,"jev":false,"rank_v":236,"rank_pub":235,"name":"CLM-v0.1-8B Qwen3-8B heads","eq":{"S":4.98,"O":8.75},"rank_tie":57,"tied":true,"parts":{"public":0,"same_skills":2.99,"new_domains":3.5}}},{"engine":"harsha","name":"Qwen-2.5-1B-RLCD","long_name":"Qwen-2.5-1B-RLCD","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":1543714304,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://huggingface.co/harshatheg/Qwen-2.5-1B-RLCD","base_model":"Qwen/Qwen2.5-1.5B","base_url":"https://huggingface.co/Qwen/Qwen2.5-1.5B","weights_repo":null,"served_checkpoint":null},"completed":109821,"counts":{"ok":109821,"unsupported":380},"lifted":false,"calibration":{"acc":0.3648,"conf":0.5671,"ece":0.2052,"brier":0.7934,"over95":0.0352,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.1428,"conf":0.0392,"acc":0.0474},{"lo":0.1,"w":0.01,"conf":0.1432,"acc":0.1705},{"lo":0.2,"w":0.019,"conf":0.2576,"acc":0.121},{"lo":0.3,"w":0.0613,"conf":0.3546,"acc":0.2552},{"lo":0.4,"w":0.1033,"conf":0.4547,"acc":0.3506},{"lo":0.5,"w":0.1708,"conf":0.5484,"acc":0.4176},{"lo":0.6,"w":0.1227,"conf":0.6483,"acc":0.4143},{"lo":0.7,"w":0.1501,"conf":0.7495,"acc":0.475},{"lo":0.8,"w":0.094,"conf":0.8516,"acc":0.4813},{"lo":0.9,"w":0.1261,"conf":0.9578,"acc":0.5039}],"areas":{"knowledge":{"ece":0.199,"acc":0.3585,"conf":0.5571},"language":{"ece":0.2699,"acc":0.4209,"conf":0.6906},"retrieval":{"ece":0.2027,"acc":0.1929,"conf":0.3787},"tools":{"ece":0.2339,"acc":0.347,"conf":0.5761},"arts":{"ece":0.1175,"acc":0.4261,"conf":0.5121}}},"gaps":{"share":0.0032,"reasons":[{"reason":"prompt longer than its context window (no truncation allowed)","share":0.0032}]},"latency":{"median":43.8,"p95":75.2,"mean":53.6},"results":{"1":{"score":0.474,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":45.8},"2":{"score":0.0852,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":257.4},"3":{"score":0.0433,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0259,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.9},"5":{"score":0.0116,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":56.4},"6":{"score":0.5886,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":65.9},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4644,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":47.6},"11":{"score":0.2668,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2819,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":42.9},"20":{"score":0.6808,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.2},"21":{"score":0.5068,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":42},"22":{"score":0.029,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":111.6},"23":{"score":0.5024,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44},"24":{"score":0.3688,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":42.7},"25":{"score":0.2806,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5648,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":42.5},"27":{"score":0.4369,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":42.2},"28":{"score":0.5043,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":42},"29":{"score":0.4255,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":43.5},"30":{"score":0.1831,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0882,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.6},"32":{"score":0.3777,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0236,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":47.6},"36":{"score":0.1433,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.0892,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.9},"38":{"score":0.0458,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":295.6},"39":{"score":0.4332,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2431,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":42.2,"tracks":[{"track":"A · Arabic","score":0.2768,"headline":false},{"track":"A · English","score":0.2431,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.475,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3142,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.3},"42":{"score":0.4373,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":44.4},"43":{"score":0.3579,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.503,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.1},"45":{"score":0.1717,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3848,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":46.7},"56":{"score":0.4905,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":45.6,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1154,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":43.2,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.3374,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":42.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.0063,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":45.8,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5012,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":44.1,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.2204,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":45.1,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.1932,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":24.99,"balanced_skill":7.85,"breadth_skill":3.77,"public_skill":4.02},"frozen_scores":{"balanced_raw":24.99,"balanced_skill":4.02,"breadth_skill":3.77},"categories":[{"id":"knowledge","raw":0.2687,"skill":0.014,"combined":0.0157,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2761,"skill":0.052,"combined":0.0774,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.1966,"skill":0.0094,"combined":0.01,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1715,"skill":0.07,"combined":0.0774,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.383,"skill":0.085,"combined":0.1317,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.474,"skill":0.29,"coverage":1,"random":0.2592,"tracks":[],"combined":0.3266,"private_eq":0.5061},"2":{"raw":0.0852,"skill":0,"coverage":1,"random":0.1341,"tracks":[],"combined":0.0486,"private_eq":0.1847},"3":{"raw":0.0433,"skill":0.0249,"coverage":1,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0277},"4":{"raw":0.0259,"skill":0.0134,"coverage":1,"random":0.0127,"tracks":[],"combined":0.0077,"private_eq":0.0305},"5":{"raw":0.0116,"skill":0.0056,"coverage":1,"random":0.006,"tracks":[],"combined":0,"private_eq":0.0162},"6":{"raw":0.5886,"skill":0.0372,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.5345,"headline":false},{"track":"RouterBench-5shot","score":0.6427,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2668,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2819,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.6735,"skill":0.347,"coverage":1,"random":0.5,"tracks":[],"combined":0.5094,"private_eq":0.757},"21":{"raw":0.5068,"skill":0.0137,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0298,"skill":0.0222,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0147,"private_eq":0.0325},"23":{"raw":0.51,"skill":0.02,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3832,"skill":0.1776,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2806,"skill":0.0408,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.2571},"28":{"raw":0.5043,"skill":0.0086,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"29":{"raw":0.4255,"skill":0.234,"coverage":1,"random":0.25,"tracks":[],"combined":0.3248,"private_eq":0.4986},"30":{"raw":0.1831,"skill":0.0099,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2578,"random":0.25},{"track":"10-choice","raw":0.1084,"random":0.1}],"combined":0,"private_eq":0.1796},"31":{"raw":0.0882,"skill":0.0069,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0883},"32":{"raw":0.3777,"skill":0.0107,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0236,"skill":0.0106,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0175},"36":{"raw":0.1433,"skill":0.0308,"coverage":1,"random":0.116,"tracks":[],"combined":0,"private_eq":0.1199},"37":{"raw":0.0892,"skill":0,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0458,"skill":0.0153,"coverage":1,"random":0.031,"tracks":[],"combined":0.021,"private_eq":0.0612},"39":{"raw":0.4332,"skill":0.1663,"coverage":1,"random":0.3201,"tracks":[],"combined":0.2497,"private_eq":0.495},"40":{"raw":0.2431,"skill":0.0262,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2768,"headline":false},{"track":"A · English","score":0.2431,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.475,"headline":false}],"combined":0.0335,"private_eq":0.2565},"41":{"raw":0.3142,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4373,"skill":0,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3579,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0018,"private_eq":0.3774},"44":{"raw":0.503,"skill":0.006,"coverage":1,"random":0.5,"tracks":[],"combined":0.1497,"private_eq":0.5792},"45":{"raw":0.1717,"skill":0.0091,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3848,"skill":0.1072,"coverage":1,"random":0.311,"tracks":[],"combined":0.1523,"private_eq":0.4219},"56":{"raw":0.4905,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0838,"private_eq":0.5466},"57":{"raw":0.1154,"skill":0.0051,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.0145,"private_eq":0.1329},"58":{"raw":0.3374,"skill":0.0396,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0324,"private_eq":0.3394},"59":{"raw":0.0063,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5012,"skill":0.0024,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5008},"62":{"raw":0.2204,"skill":0,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.1932,"skill":0,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.2023}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author engine_torch (bf16, SDPA, shared prefix + batched candidate suffixes), fleet venv torch 2.14.0+cu130; same as board","v03":{"engine":"harsha","pub":4.02,"same_skills":6.21,"new_domains":7.4,"v":7.85,"lab":false,"jev":false,"rank_v":234,"rank_pub":240,"name":"Qwen-2.5-1B-RLCD","eq":{"S":5.23,"O":11.78},"rank_tie":56,"tied":true,"parts":{"public":0,"same_skills":3.14,"new_domains":4.71}}},{"engine":"akash-gemma","name":"system-one-gemma","long_name":"system-one-gemma","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":268098176,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/akash-kamat/system-one-gemma","base_model":"google/gemma-3-270m","base_url":"https://huggingface.co/google/gemma-3-270m","weights_repo":null,"served_checkpoint":null},"completed":110098,"counts":{"ok":110098},"lifted":true,"calibration":{"acc":0.3215,"conf":0.5602,"ece":0.2387,"brier":0.7993,"over95":0.0161,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.0429,"conf":0.0656,"acc":0.0395},{"lo":0.1,"w":0.0155,"conf":0.1554,"acc":0.0865},{"lo":0.2,"w":0.0652,"conf":0.2645,"acc":0.0665},{"lo":0.3,"w":0.1053,"conf":0.3525,"acc":0.2187},{"lo":0.4,"w":0.1291,"conf":0.4511,"acc":0.2903},{"lo":0.5,"w":0.2205,"conf":0.5478,"acc":0.3956},{"lo":0.6,"w":0.1573,"conf":0.6462,"acc":0.3549},{"lo":0.7,"w":0.1126,"conf":0.7462,"acc":0.3527},{"lo":0.8,"w":0.0829,"conf":0.8484,"acc":0.4576},{"lo":0.9,"w":0.0688,"conf":0.9534,"acc":0.479}],"areas":{"knowledge":{"ece":0.2116,"acc":0.2962,"conf":0.507},"language":{"ece":0.325,"acc":0.3207,"conf":0.6457},"retrieval":{"ece":0.2167,"acc":0.405,"conf":0.62},"tools":{"ece":0.2613,"acc":0.2167,"conf":0.478},"arts":{"ece":0.15,"acc":0.3745,"conf":0.5245}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":31.9,"p95":748,"mean":168.7},"results":{"1":{"score":0.3483,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":64.9},"2":{"score":0.1639,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1004.2},"3":{"score":0.0118,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.5347,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.6},"5":{"score":0.2486,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":86.5},"6":{"score":0.3575,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":549.7},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4519,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":44.5},"11":{"score":0.2072,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2389,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31.2},"20":{"score":0.5776,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.5},"21":{"score":0.4726,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":28.1},"22":{"score":0.0025,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2918.6},"23":{"score":0.524,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31},"24":{"score":0.3081,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":30.8},"25":{"score":0.2449,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.4465,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":30.4},"27":{"score":0.3097,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":30.4},"28":{"score":0.4988,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":28.1},"29":{"score":0.2841,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":31.5},"30":{"score":0.2055,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0912,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.1},"32":{"score":0.4162,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0012,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":233.8},"36":{"score":0.1033,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.1949,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.5},"38":{"score":0.0309,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1816.4},"39":{"score":0.13,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2506,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":28.1,"tracks":[{"track":"A · Arabic","score":0.2466,"headline":false},{"track":"A · English","score":0.2506,"headline":true},{"track":"C · Arabic pairs","score":0.335,"headline":false},{"track":"C · English pairs","score":0.345,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3489,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":29.5},"42":{"score":0.4253,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":31},"43":{"score":0.3404,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.502,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.8},"45":{"score":0.1257,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.2959,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":35.3},"56":{"score":0.5995,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":262.6,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1317,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":31.7,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.2853,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":30.7,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.1293,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":33.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5052,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":32.1,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.1569,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":34.5,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.2917,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":25.9,"balanced_skill":5.22,"breadth_skill":4.29,"public_skill":5.19},"frozen_scores":{"balanced_raw":25.9,"balanced_skill":5.19,"breadth_skill":4.29},"categories":[{"id":"knowledge","raw":0.2583,"skill":0.0127,"combined":0.0227,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2287,"skill":0.0121,"combined":0.0168,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3581,"skill":0.1722,"combined":0.0879,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1394,"skill":0.0331,"combined":0.0253,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3588,"skill":0.049,"combined":0.0732,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.3483,"skill":0.1202,"coverage":1,"random":0.2592,"tracks":[],"combined":0.1155,"private_eq":0.3515},"2":{"raw":0.1639,"skill":0.0344,"coverage":1,"random":0.1341,"tracks":[],"combined":0,"private_eq":0.1341},"3":{"raw":0.0118,"skill":0,"coverage":1,"random":0.0189,"tracks":[],"combined":0.0247,"private_eq":0.0531},"4":{"raw":0.5347,"skill":0.5287,"coverage":1,"random":0.0127,"tracks":[],"combined":0.3361,"private_eq":0.351},"5":{"raw":0.2486,"skill":0.2441,"coverage":1,"random":0.006,"tracks":[],"combined":0.0252,"private_eq":0.0411},"6":{"raw":0.3576,"skill":0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.2032,"headline":false},{"track":"RouterBench-5shot","score":0.512,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2072,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2389,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5654,"skill":0.1309,"coverage":1,"random":0.5,"tracks":[],"combined":0.1625,"private_eq":0.5855},"21":{"raw":0.4726,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0027,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0211,"private_eq":0.0388},"23":{"raw":0.5242,"skill":0.0484,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3064,"skill":0.0752,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"28":{"raw":0.4988,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0981,"private_eq":0.5537},"29":{"raw":0.2841,"skill":0.0455,"coverage":1,"random":0.25,"tracks":[],"combined":0.1663,"private_eq":0.3811},"30":{"raw":0.2055,"skill":0.0386,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2926,"random":0.25},{"track":"10-choice","raw":0.1183,"random":0.1}],"combined":0.0463,"private_eq":0.2213},"31":{"raw":0.0912,"skill":0.0102,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0258,"private_eq":0.1149},"32":{"raw":0.4162,"skill":0.0719,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0012,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.1033,"skill":0,"coverage":1,"random":0.116,"tracks":[],"combined":0,"private_eq":0.116},"37":{"raw":0.1949,"skill":0,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0309,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.13,"skill":0,"coverage":1,"random":0.3201,"tracks":[],"combined":0,"private_eq":0.3201},"40":{"raw":0.2506,"skill":0.036,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2466,"headline":false},{"track":"A · English","score":0.2506,"headline":true},{"track":"C · Arabic pairs","score":0.335,"headline":false},{"track":"C · English pairs","score":0.345,"headline":false}],"combined":0.0069,"private_eq":0.2361},"41":{"raw":0.3489,"skill":0.0234,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4253,"skill":0,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3404,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0394,"private_eq":0.4008},"44":{"raw":0.502,"skill":0.004,"coverage":1,"random":0.5,"tracks":[],"combined":0.0747,"private_eq":0.5421},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.2959,"skill":0,"coverage":1,"random":0.311,"tracks":[],"combined":0.1234,"private_eq":0.4022},"56":{"raw":0.5995,"skill":0.199,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.1617,"private_eq":0.5851},"57":{"raw":0.1317,"skill":0.0234,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.1109},"58":{"raw":0.2853,"skill":0,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.3101},"59":{"raw":0.1293,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5052,"skill":0.0104,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0.0081,"private_eq":0.5092},"62":{"raw":0.1569,"skill":0,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.2917,"skill":0.1146,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.07,"private_eq":0.2637}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author infer.score, fp32 (the only precision infer.load_trained_model offers), one padded batch of all options per question, SDPA default; max_len 32768 lift as board; OOM retry sub-batches of 8","v03":{"engine":"akash-gemma","pub":5.19,"same_skills":5.03,"new_domains":3.52,"v":5.22,"lab":false,"jev":false,"rank_v":239,"rank_pub":238,"name":"system-one-gemma","eq":{"S":4.03,"O":7},"rank_tie":58,"tied":true,"parts":{"public":0,"same_skills":2.42,"new_domains":2.8}}},{"engine":"julia-1","name":"Julia 1","long_name":"Supersonic Labs Julia 1","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":140908331,"params_estimated":true,"params_from_name":false,"trained_bytes":577189056,"model_url":"https://huggingface.co/SupersonicLabs/Julia-1","code_url":null,"base_model":"jhu-clsp/mmBERT-small","base_url":"https://huggingface.co/jhu-clsp/mmBERT-small","weights_repo":"SupersonicLabs/Julia-1","served_checkpoint":null,"org":{"id":"SupersonicLabs","name":"Supersonic Labs","logo":"data/orgs/SupersonicLabs.png"}},"completed":109739,"counts":{"ok":109739,"unsupported":10},"lifted":false,"calibration":{"acc":0.3741,"conf":0.7938,"ece":0.4197,"brier":0.9866,"over95":0.1866,"n":35087,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0047,"conf":0.1592,"acc":0.1205},{"lo":0.2,"w":0.0139,"conf":0.2533,"acc":0.1083},{"lo":0.3,"w":0.0319,"conf":0.3527,"acc":0.1378},{"lo":0.4,"w":0.0522,"conf":0.4533,"acc":0.2241},{"lo":0.5,"w":0.0987,"conf":0.549,"acc":0.3663},{"lo":0.6,"w":0.1025,"conf":0.6506,"acc":0.3412},{"lo":0.7,"w":0.1133,"conf":0.7511,"acc":0.3826},{"lo":0.8,"w":0.1439,"conf":0.8525,"acc":0.386},{"lo":0.9,"w":0.4389,"conf":0.9705,"acc":0.4236}],"areas":{"knowledge":{"ece":0.434,"acc":0.3558,"conf":0.7898},"language":{"ece":0.3759,"acc":0.4373,"conf":0.8132},"retrieval":{"ece":0.3899,"acc":0.3611,"conf":0.751},"tools":{"ece":0.5319,"acc":0.2752,"conf":0.8038},"arts":{"ece":0.4256,"acc":0.3703,"conf":0.7958}}},"gaps":{"share":0.0001,"reasons":[{"reason":"prompt longer than its 8,192-token context window (no truncation allowed)","share":0.0001}]},"latency":{"median":5.8,"p95":15.3,"mean":72.7},"results":{"1":{"score":0.3749,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":6.4},"2":{"score":0.1358,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":0.1831,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.426,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":5.7},"5":{"score":0.1234,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":6},"6":{"score":0.5756,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":12.5},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3505,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":6.9},"11":{"score":0.3178,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.2864,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":5.5},"20":{"score":0.4878,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.4},"21":{"score":0.4825,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":5.3},"22":{"score":0.002,"jev":null,"metric":"accuracy","requests":2000,"answered":1997,"unsupported":3,"errors":0,"median_ms":18.9},"23":{"score":0.5042,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.7},"24":{"score":0.3015,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":5.5},"25":{"score":0.2551,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.391,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":5.4},"27":{"score":0.3259,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":5.5},"28":{"score":0.5067,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":5.4},"29":{"score":0.2988,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":5.8},"30":{"score":0.1691,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0752,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.4},"32":{"score":0.3896,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0097,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":10.9},"36":{"score":0.0914,"jev":null,"metric":"nDCG@10","requests":220,"answered":219,"unsupported":1,"errors":0,"median_ms":null},"37":{"score":0.206,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.6},"38":{"score":0.0312,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":28.6},"39":{"score":0.3279,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.0653,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":5.3,"tracks":[{"track":"A · Arabic","score":0.2039,"headline":false},{"track":"A · English","score":0.0653,"headline":true},{"track":"C · Arabic pairs","score":0.465,"headline":false},{"track":"C · English pairs","score":0.43,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.3293,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":5.5},"42":{"score":0.4973,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":5.9},"43":{"score":0.3807,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.5018,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.5},"45":{"score":0.144,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":null},"50":{"score":0.3538,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":6.9},"56":{"score":0.4985,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":5.9,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1467,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":5.8,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.3379,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":5.6,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5204,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":6.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.503,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":5.9,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.236,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":6.6,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.2348,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":27.52,"balanced_skill":4.75,"breadth_skill":3.86,"public_skill":4.24},"frozen_scores":{"balanced_raw":27.52,"balanced_skill":4.24,"breadth_skill":3.86},"categories":[{"id":"knowledge","raw":0.2662,"skill":0.0143,"combined":0.0406,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.297,"skill":0.0139,"combined":0.0146,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2994,"skill":0.0991,"combined":0.0349,"coverage":0.9992,"pending":0,"n":6},{"id":"tools","raw":0.1927,"skill":0.0721,"combined":0.0606,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.345,"skill":0.0209,"combined":0.024,"coverage":0.9997,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.3749,"skill":0.1561,"coverage":1,"random":0.2592,"tracks":[],"combined":0.136,"private_eq":0.3665},"2":{"raw":0.1349,"skill":0.0009,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.0422,"private_eq":0.1792},"3":{"raw":0.1831,"skill":0.1674,"coverage":1,"random":0.0189,"tracks":[],"combined":0.1193,"private_eq":0.1448},"4":{"raw":0.426,"skill":0.4186,"coverage":1,"random":0.0127,"tracks":[],"combined":0.2066,"private_eq":0.2246},"5":{"raw":0.1234,"skill":0.1181,"coverage":1,"random":0.006,"tracks":[],"combined":0.0158,"private_eq":0.0319},"6":{"raw":0.5757,"skill":0.0328,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.504,"headline":false},{"track":"RouterBench-5shot","score":0.6473,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.3178,"skill":0.0134,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2864,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.487,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0467,"private_eq":0.5283},"21":{"raw":0.4825,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0152,"private_eq":0.5127},"22":{"raw":0.0023,"skill":0,"coverage":0.9985,"random":0.0078,"tracks":[],"combined":0.0078,"private_eq":0.0258},"23":{"raw":0.5097,"skill":0.0194,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3092,"skill":0.0789,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2551,"skill":0.0068,"coverage":1,"random":0.25,"tracks":[],"combined":0.0453,"private_eq":0.2914},"28":{"raw":0.5067,"skill":0.0134,"coverage":1,"random":0.5,"tracks":[],"combined":0.088,"private_eq":0.5487},"29":{"raw":0.2988,"skill":0.0651,"coverage":1,"random":0.25,"tracks":[],"combined":0.1613,"private_eq":0.3774},"30":{"raw":0.1691,"skill":0,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2411,"random":0.25},{"track":"10-choice","raw":0.097,"random":0.1}],"combined":0.0238,"private_eq":0.203},"31":{"raw":0.0752,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0824},"32":{"raw":0.3896,"skill":0.0296,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.0097,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.091,"skill":0,"coverage":0.9955,"random":0.116,"tracks":[],"combined":0,"private_eq":0.116},"37":{"raw":0.206,"skill":0.0041,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0312,"skill":0.0002,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.0356},"39":{"raw":0.3279,"skill":0.0115,"coverage":1,"random":0.3201,"tracks":[],"combined":0,"private_eq":0.3201},"40":{"raw":0.0653,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2039,"headline":false},{"track":"A · English","score":0.0653,"headline":true},{"track":"C · Arabic pairs","score":0.465,"headline":false},{"track":"C · English pairs","score":0.43,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.3293,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4973,"skill":0.022,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3807,"skill":0.0174,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0185,"private_eq":0.3878},"44":{"raw":0.5018,"skill":0.0036,"coverage":1,"random":0.5,"tracks":[],"combined":0.0372,"private_eq":0.5236},"45":{"raw":0.1437,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.3538,"skill":0.0622,"coverage":1,"random":0.311,"tracks":[],"combined":0.029,"private_eq":0.3379},"56":{"raw":0.4985,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5038},"57":{"raw":0.1467,"skill":0.0403,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.1522,"private_eq":0.2539},"58":{"raw":0.3379,"skill":0.0403,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0.0285,"private_eq":0.3367},"59":{"raw":0.5204,"skill":0.0056,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.503,"skill":0.006,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5},"62":{"raw":0.236,"skill":0,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.2348,"skill":0.0435,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.0216,"private_eq":0.2254}},"coverage":0.7595,"pending":0.24,"latency_rows":749,"latency_path":"author julia runtime (supersonic-julia 0.1.0, FastEngine) on CUDA with its optional compile_model=True (torch.compile dynamic=True; recompiles on new shapes are inside the timed rows), BF16 autocast, board guard lifts, one process; torch 2.14.0+cu130, transformers 5.0.0. Other variants kept as latency-v1-full.: eager (median 12.4 / p95 35.0 ms), marker_only_head (12.3 / 39.5), compile+marker_only_head (5.9 / 14.6)","v03":{"engine":"julia-1","pub":4.24,"same_skills":4.54,"new_domains":3.18,"v":4.75,"lab":false,"jev":false,"rank_v":241,"rank_pub":239,"name":"Supersonic Labs Julia 1","eq":{"S":3.54,"O":6.58},"rank_tie":58,"tied":true,"parts":{"public":0,"same_skills":2.12,"new_domains":2.63}}},{"engine":"verdict-8af2496e","name":"Verdict","long_name":"Verdict","variant":null,"meta":{"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiClass","served_params":151378176,"params_estimated":false,"params_from_name":false,"trained_bytes":605529340,"model_url":"https://huggingface.co/heman10x/rlcd-modernbert-151m","code_url":"https://github.com/Heman10x-NGU/Verdict-open-jev","base_model":"knowledgator/gliclass-modern-base-v2.0","base_url":"https://huggingface.co/knowledgator/gliclass-modern-base-v2.0","weights_repo":"heman10x/rlcd-modernbert-151m","served_checkpoint":null},"completed":44328,"counts":{"ok":44328,"abstained":56045,"unsupported":15056},"lifted":false,"calibration":{"acc":0.3689,"conf":0.5228,"ece":0.154,"brier":0.6841,"over95":0,"n":58189,"benchmarks":28,"rel":[{"lo":0,"w":0.0151,"conf":0.0763,"acc":0.0637},{"lo":0.1,"w":0.0148,"conf":0.1474,"acc":0.1513},{"lo":0.2,"w":0.0319,"conf":0.2639,"acc":0.1987},{"lo":0.3,"w":0.1264,"conf":0.3563,"acc":0.2337},{"lo":0.4,"w":0.1942,"conf":0.4526,"acc":0.3511},{"lo":0.5,"w":0.2984,"conf":0.5445,"acc":0.3913},{"lo":0.6,"w":0.2431,"conf":0.6472,"acc":0.4543},{"lo":0.7,"w":0.0557,"conf":0.7378,"acc":0.453},{"lo":0.8,"w":0.0191,"conf":0.8387,"acc":0.4591},{"lo":0.9,"w":0.0013,"conf":0.9127,"acc":0.332}],"areas":{"knowledge":{"ece":0.1816,"acc":0.3151,"conf":0.4964},"language":{"ece":0.0844,"acc":0.4755,"conf":0.5469},"retrieval":{"ece":0.3057,"acc":0.1527,"conf":0.4584},"tools":{"ece":0.3816,"acc":0.241,"conf":0.6226},"arts":{"ece":0.0817,"acc":0.4493,"conf":0.5238}}},"gaps":{"share":0.5908,"reasons":[{"reason":"chose its own abstain option instead of an answer","share":0.4657},{"reason":"supports only 2–24 answer options per question","share":0.1224},{"reason":"prompt longer than its 8,192-token context window","share":0.0027}]},"latency":{"median":11.4,"p95":60.1,"mean":21},"results":{"1":{"score":0.3878,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":17.6},"2":{"score":0.1904,"jev":null,"metric":"nDCG@10","requests":685,"answered":496,"unsupported":5,"errors":0,"median_ms":null},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":null,"jev":null,"metric":null,"requests":3080,"answered":0,"unsupported":3080,"errors":0,"median_ms":null},"5":{"score":null,"jev":null,"metric":null,"requests":5500,"answered":0,"unsupported":5500,"errors":0,"median_ms":null},"6":{"score":0.5084,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":7536,"unsupported":0,"errors":0,"median_ms":43.6},"9":{"score":null,"jev":null,"metric":null,"requests":88,"answered":0,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0792,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":18.4},"11":{"score":0.2116,"jev":null,"metric":"macro-F1","requests":123,"answered":31,"unsupported":1,"errors":0,"median_ms":null},"12":{"score":0.1492,"jev":null,"metric":"macro-F1","requests":3200,"answered":328,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.4761,"jev":null,"metric":"accuracy","requests":5000,"answered":2283,"unsupported":0,"errors":0,"median_ms":10.7},"21":{"score":0.386,"jev":null,"metric":"accuracy","requests":2628,"answered":228,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":null,"jev":null,"metric":null,"requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"23":{"score":0.6562,"jev":null,"metric":"accuracy","requests":5000,"answered":96,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.3088,"jev":null,"metric":"accuracy","requests":14033,"answered":4812,"unsupported":0,"errors":0,"median_ms":10.6},"25":{"score":0.2874,"jev":null,"metric":"accuracy","requests":196,"answered":87,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.5206,"jev":null,"metric":"accuracy","requests":2376,"answered":632,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.2738,"jev":null,"metric":"accuracy","requests":1172,"answered":336,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.494,"jev":null,"metric":"accuracy","requests":1267,"answered":1241,"unsupported":0,"errors":0,"median_ms":9.5},"29":{"score":0.3506,"jev":null,"metric":"accuracy","requests":10042,"answered":6896,"unsupported":0,"errors":0,"median_ms":11.2},"30":{"score":0.0307,"jev":null,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":630,"unsupported":0,"errors":2008,"median_ms":null},"31":{"score":0.1735,"jev":null,"metric":"accuracy","requests":5000,"answered":1349,"unsupported":3641,"errors":0,"median_ms":11.8},"32":{"score":0.4101,"jev":null,"metric":"accuracy","requests":752,"answered":656,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0119,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":84,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.0903,"jev":null,"metric":"nDCG@10","requests":220,"answered":61,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.1277,"jev":null,"metric":"macro-F1","requests":5000,"answered":2146,"unsupported":0,"errors":0,"median_ms":11.5},"38":{"score":0.0365,"jev":null,"metric":"per-review F1","requests":1565,"answered":1559,"unsupported":0,"errors":0,"median_ms":80.6},"39":{"score":0.4211,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.1012,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":3614,"unsupported":0,"errors":0,"median_ms":9.5,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.1012,"headline":true},{"track":"C · Arabic pairs","score":0.195,"headline":false},{"track":"C · English pairs","score":0.1,"headline":false}]},"41":{"score":0.3337,"jev":null,"metric":"macro-F1","requests":3006,"answered":1388,"unsupported":0,"errors":0,"median_ms":10.9},"42":{"score":0.5155,"jev":null,"metric":"macro-F1","requests":5500,"answered":891,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.3926,"jev":null,"metric":"accuracy","requests":570,"answered":461,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.3788,"jev":null,"metric":"accuracy","requests":5000,"answered":66,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1496,"jev":null,"metric":"accuracy","requests":501,"answered":274,"unsupported":4,"errors":0,"median_ms":null},"50":{"score":0.3769,"jev":null,"metric":"accuracy","requests":1676,"answered":1661,"unsupported":0,"errors":0,"median_ms":16.2},"56":{"score":0.5532,"metric":"accuracy","requests":2000,"answered":47,"unsupported":0,"errors":1953,"median_ms":null,"edition_added":"v0.2","jev":null},"57":{"score":0.1345,"metric":"accuracy","requests":12032,"answered":7116,"unsupported":0,"errors":4916,"median_ms":10.9,"edition_added":"v0.2","jev":null},"58":{"score":0.3361,"metric":"accuracy","requests":5507,"answered":5444,"unsupported":0,"errors":63,"median_ms":10.7,"edition_added":"v0.2","jev":null},"59":{"score":0.119,"metric":"F1 on hallucinated class","requests":2700,"answered":2666,"unsupported":0,"errors":34,"median_ms":14.9,"edition_added":"v0.2","jev":null},"61":{"score":0.5173,"metric":"accuracy","requests":4000,"answered":1562,"unsupported":0,"errors":2438,"median_ms":11.6,"edition_added":"v0.2","jev":null},"62":{"score":0.3681,"metric":"accuracy","requests":3652,"answered":3540,"unsupported":0,"errors":112,"median_ms":13.6,"edition_added":"v0.2","jev":null},"64":{"score":0.8,"metric":"accuracy","requests":528,"answered":5,"unsupported":0,"errors":523,"median_ms":null,"edition_added":"v0.2","jev":null}},"scores":{"balanced_raw":12.91,"balanced_skill":2.04,"breadth_skill":1.68,"public_skill":1.86},"frozen_scores":{"balanced_raw":12.91,"balanced_skill":1.86,"breadth_skill":1.68},"categories":[{"id":"knowledge","raw":0.1737,"skill":0.0032,"combined":0.0212,"coverage":0.5428,"pending":0,"n":11},{"id":"language","raw":0.1341,"skill":0.0164,"combined":0,"coverage":0.5949,"pending":0,"n":9},{"id":"retrieval","raw":0.0431,"skill":0,"combined":0,"coverage":0.1782,"pending":0,"n":6},{"id":"tools","raw":0.1778,"skill":0.0658,"combined":0.0557,"coverage":0.5358,"pending":0,"n":5},{"id":"arts","raw":0.0846,"skill":0.0151,"combined":0,"coverage":0.2366,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.3878,"skill":0.1736,"coverage":1,"random":0.2592,"tracks":[],"combined":0.17,"private_eq":0.3914},"2":{"raw":0.1379,"skill":0.0044,"coverage":0.7241,"random":0.1341,"tracks":[],"combined":0,"private_eq":0.1387},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0189},"4":{"raw":0,"skill":0,"coverage":0,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0,"skill":0,"coverage":0,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.3006,"skill":0,"coverage":0.591,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.2851,"headline":false},{"track":"RouterBench-5shot","score":0.316,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":0,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.0631,"skill":0,"coverage":0.252,"random":0.3085,"tracks":[]},"12":{"raw":0.0153,"skill":0,"coverage":0.1025,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.0858,"skill":0,"coverage":0.3132,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.0335,"skill":0,"coverage":0.0868,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0,"skill":0,"coverage":0,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.0069,"skill":0,"coverage":0.0192,"random":0.5,"tracks":[]},"24":{"raw":0.1001,"skill":0,"coverage":0.3429,"random":0.25,"tracks":[]},"25":{"raw":0.1276,"skill":0,"coverage":0.4439,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"28":{"raw":0.4839,"skill":0,"coverage":0.9795,"random":0.5,"tracks":[],"combined":0.2745,"private_eq":0.6409},"29":{"raw":0.2408,"skill":0,"coverage":0.6867,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"30":{"raw":0.0307,"skill":0,"coverage":0.2388,"random":0.175,"tracks":[{"track":"4-choice","raw":0.025,"random":0.25},{"track":"10-choice","raw":0.0364,"random":0.1}],"combined":0,"private_eq":0.175},"31":{"raw":0.0468,"skill":0,"coverage":0.2698,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.3577,"skill":0,"coverage":0.8723,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0006,"skill":0,"coverage":0.0509,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.025,"skill":0,"coverage":0.2773,"random":0.116,"tracks":[],"combined":0.0191,"private_eq":0.1419},"37":{"raw":0.0393,"skill":0,"coverage":0.4292,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0364,"skill":0.0055,"coverage":0.9962,"random":0.031,"tracks":[],"combined":0.0028,"private_eq":0.0438},"39":{"raw":0.4211,"skill":0.1486,"coverage":1,"random":0.3201,"tracks":[],"combined":0.0518,"private_eq":0.362},"40":{"raw":0.1012,"skill":0,"coverage":0.7857,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.1012,"headline":true},{"track":"C · Arabic pairs","score":0.195,"headline":false},{"track":"C · English pairs","score":0.1,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.1301,"skill":0,"coverage":0.4617,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.0835,"skill":0,"coverage":0.162,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3175,"skill":0,"coverage":0.8088,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.005,"skill":0,"coverage":0.0132,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"45":{"raw":0.0818,"skill":0,"coverage":0.5469,"random":0.1641,"tracks":[]},"50":{"raw":0.3735,"skill":0.0908,"coverage":0.9911,"random":0.311,"tracks":[],"combined":0,"private_eq":0.311},"56":{"raw":0.013,"skill":0,"coverage":0.0235,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5},"57":{"raw":0.0795,"skill":0,"coverage":0.5914,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.1109},"58":{"raw":0.3323,"skill":0.0322,"coverage":0.9886,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.3101},"59":{"raw":0.1175,"skill":0,"coverage":0.9874,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.202,"skill":0,"coverage":0.3905,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5},"62":{"raw":0.3568,"skill":0.1424,"coverage":0.9693,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0.1249,"private_eq":0.3504},"64":{"raw":0.0076,"skill":0,"coverage":0.0095,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.2}},"coverage":0.3069,"pending":0.24,"latency_rows":357,"latency_path":"author DecisionEngine (ModernBERT 151M) evaluate, max_length 8192 lift as board; venv-vl torch 2.13.0+cu130 from the rerun freeze. Native abstentions are not timed (357 of 750 rows answered)","v03":{"engine":"verdict-8af2496e","pub":1.86,"same_skills":2.41,"new_domains":0.32,"v":2.04,"lab":false,"jev":false,"rank_v":244,"rank_pub":241,"name":"Verdict","eq":{"S":1.38,"O":3.04},"rank_tie":60,"tied":false,"parts":{"public":0,"same_skills":0.83,"new_domains":1.22}}},{"engine":"lfm350-sdpa","name":"LFM2.5-350M-RLCD","long_name":"LFM2.5-350M-RLCD","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":354483968,"params_estimated":false,"params_from_name":false,"trained_bytes":708984464,"model_url":"https://huggingface.co/notnotsamuel/LFM2.5-350M-RLCD","code_url":null,"base_model":null,"base_url":null,"weights_repo":"notnotsamuel/LFM2.5-350M-RLCD","served_checkpoint":null},"completed":110201,"counts":{"ok":110201},"lifted":false,"calibration":{"acc":0.2785,"conf":0.8464,"ece":0.5681,"brier":1.2239,"over95":0.3378,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0003,"conf":0.192,"acc":0.5766},{"lo":0.2,"w":0.0075,"conf":0.262,"acc":0.0486},{"lo":0.3,"w":0.0186,"conf":0.3519,"acc":0.0377},{"lo":0.4,"w":0.0316,"conf":0.4506,"acc":0.0665},{"lo":0.5,"w":0.0821,"conf":0.5496,"acc":0.2987},{"lo":0.6,"w":0.0851,"conf":0.6515,"acc":0.3012},{"lo":0.7,"w":0.0945,"conf":0.7502,"acc":0.2712},{"lo":0.8,"w":0.119,"conf":0.8504,"acc":0.2794},{"lo":0.9,"w":0.5614,"conf":0.9815,"acc":0.2961}],"areas":{"knowledge":{"ece":0.5573,"acc":0.2783,"conf":0.835},"language":{"ece":0.5795,"acc":0.279,"conf":0.8586},"retrieval":{"ece":0.6173,"acc":0.1868,"conf":0.8041},"tools":{"ece":0.5431,"acc":0.2504,"conf":0.7935},"arts":{"ece":0.5467,"acc":0.3684,"conf":0.9152}}},"gaps":{"share":0,"reasons":[]},"latency":{"median":26.7,"p95":174.3,"mean":46.5},"results":{"1":{"score":0.3837,"jev":0.964,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":27.9},"2":{"score":0.1233,"jev":0.6505,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":328.8},"3":{"score":0.0236,"jev":0.8661,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.0018,"jev":0.7994,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":91.6},"5":{"score":0.0029,"jev":0.9028,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":174.3},"6":{"score":0.5035,"jev":0.7988,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":49.1},"9":{"score":0,"jev":0.4545,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.3421,"jev":0.0903,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":28.3},"11":{"score":0.2021,"jev":0.7076,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.1962,"jev":0.7427,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":25},"20":{"score":0.5848,"jev":0.9006,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":25.3},"21":{"score":0.4958,"jev":0.6176,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":24.7},"22":{"score":0.051,"jev":0.1645,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":267.6},"23":{"score":0.4814,"jev":0.6634,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26},"24":{"score":0.2389,"jev":0.9167,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":25.6},"25":{"score":0.2704,"jev":0.7551,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.2647,"jev":0.9933,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":25.2},"27":{"score":0.2304,"jev":0.9761,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":25.8},"28":{"score":0.5201,"jev":0.9108,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":25},"29":{"score":0.2515,"jev":0.9434,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":26.3},"30":{"score":0.1797,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0796,"jev":0.1688,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.1},"32":{"score":0.3617,"jev":0.7261,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.0133,"jev":0.2788,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":35.7},"36":{"score":0.1103,"jev":0.4705,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.1238,"jev":0.5718,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.4},"38":{"score":0.031,"jev":0.3314,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":388.8},"39":{"score":0.3315,"jev":0.8768,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.2495,"jev":0.5327,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":24.9,"tracks":[{"track":"A · Arabic","score":0.2505,"headline":false},{"track":"A · English","score":0.2495,"headline":true},{"track":"C · Arabic pairs","score":0.51,"headline":false},{"track":"C · English pairs","score":0.455,"headline":false}],"jev_note":"Jev on all track cases"},"41":{"score":0.2532,"jev":0.7365,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":25.4},"42":{"score":0.4288,"jev":0.8512,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":26.9},"43":{"score":0.3228,"jev":0.7474,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.4974,"jev":0.8858,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.9},"45":{"score":0.1357,"jev":0.1936,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.3037,"jev":0.4457,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":30.1},"56":{"score":0.5,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":46,"edition_added":"v0.2","jev":0.8375},"57":{"score":0.1169,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":26.6,"edition_added":"v0.2","jev":0.8213},"58":{"score":0.3018,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":25.4,"edition_added":"v0.2","jev":0.9319},"59":{"score":0.5187,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":27.4,"edition_added":"v0.2","jev":0.7512},"61":{"score":0.5008,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":26.7,"edition_added":"v0.2","jev":0.7675},"62":{"score":0.1525,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":28,"edition_added":"v0.2","jev":0.8165},"64":{"score":0.1875,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null,"edition_added":"v0.2","jev":0.7254}},"scores":{"balanced_raw":23.53,"balanced_skill":5.86,"breadth_skill":1.34,"public_skill":1.43},"frozen_scores":{"balanced_raw":23.53,"balanced_skill":1.43,"breadth_skill":1.34},"categories":[{"id":"knowledge","raw":0.2557,"skill":0.0081,"combined":0,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2715,"skill":0.0059,"combined":0.02,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.1913,"skill":0.0002,"combined":0,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1416,"skill":0.0384,"combined":0.0346,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3483,"skill":0.0357,"combined":0.0433,"coverage":1,"pending":0,"n":6}],"benchmarks":{"1":{"raw":0.3837,"skill":0.168,"coverage":1,"random":0.2592,"tracks":[],"combined":0.1837,"private_eq":0.4014},"2":{"raw":0.1233,"skill":0,"coverage":1,"random":0.1341,"tracks":[],"combined":0,"private_eq":0.1341},"3":{"raw":0.0236,"skill":0.0048,"coverage":1,"random":0.0189,"tracks":[],"combined":0,"private_eq":0.0277},"4":{"raw":0.0018,"skill":0,"coverage":1,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0.0029,"skill":0,"coverage":1,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.5036,"skill":0,"coverage":1,"random":0.5245,"tracks":[{"track":"RouterBench-0shot","score":0.41,"headline":false},{"track":"RouterBench-5shot","score":0.5972,"headline":false}]},"7":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"11":{"raw":0.2021,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.1962,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"13":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"15":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"16":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"17":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"19":{"raw":0,"skill":0,"coverage":0,"random":null,"tracks":[]},"20":{"raw":0.5865,"skill":0.1731,"coverage":1,"random":0.5,"tracks":[],"combined":0.0684,"private_eq":0.539},"21":{"raw":0.4958,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0856,"private_eq":0.5475},"22":{"raw":0.0485,"skill":0.0411,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0711,"private_eq":0.0879},"23":{"raw":0.4679,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2408,"skill":0,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2704,"skill":0.0272,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.2519},"28":{"raw":0.5201,"skill":0.0402,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"29":{"raw":0.2515,"skill":0.002,"coverage":1,"random":0.25,"tracks":[],"combined":0.1663,"private_eq":0.3811},"30":{"raw":0.1797,"skill":0.0077,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2616,"random":0.25},{"track":"10-choice","raw":0.0978,"random":0.1}],"combined":0,"private_eq":0.175},"31":{"raw":0.0796,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0883},"32":{"raw":0.3617,"skill":0,"coverage":1,"random":0.371,"tracks":[],"combined":0.0107,"private_eq":0.3842},"33":{"raw":0.0133,"skill":0.0002,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.1103,"skill":0,"coverage":1,"random":0.116,"tracks":[],"combined":0,"private_eq":0.116},"37":{"raw":0.1238,"skill":0,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.031,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.3315,"skill":0.0168,"coverage":1,"random":0.3201,"tracks":[],"combined":0,"private_eq":0.3201},"40":{"raw":0.2495,"skill":0.0345,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2505,"headline":false},{"track":"A · English","score":0.2495,"headline":true},{"track":"C · Arabic pairs","score":0.51,"headline":false},{"track":"C · English pairs","score":0.455,"headline":false}],"combined":0.0335,"private_eq":0.2565},"41":{"raw":0.2532,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4288,"skill":0,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3228,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.4974,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"45":{"raw":0.1357,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3037,"skill":0,"coverage":1,"random":0.311,"tracks":[],"combined":0,"private_eq":0.311},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5},"57":{"raw":0.1169,"skill":0.0067,"coverage":1,"random":0.1109,"tracks":[],"edition_added":"v0.2","combined":0.0234,"private_eq":0.1407},"58":{"raw":0.3018,"skill":0,"coverage":1,"random":0.3101,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.3103},"59":{"raw":0.5187,"skill":0.0021,"coverage":1,"random":0.5177,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5177},"61":{"raw":0.5008,"skill":0.0016,"coverage":1,"random":0.5,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.5036},"62":{"raw":0.1525,"skill":0,"coverage":1,"random":0.25,"tracks":[],"edition_added":"v0.2","combined":0,"private_eq":0.25},"64":{"raw":0.1875,"skill":0,"coverage":1,"random":0.2,"tracks":[],"edition_added":"v0.2","combined":0.0021,"private_eq":0.21}},"coverage":0.76,"pending":0.24,"latency_rows":750,"latency_path":"author rlcd Engine with SDPA attention (board path), fleet venv torch 2.14.0+cu130","v03":{"engine":"lfm350-sdpa","pub":1.43,"same_skills":2.29,"new_domains":8.18,"v":5.86,"lab":false,"jev":false,"rank_v":238,"rank_pub":244,"name":"LFM2.5-350M-RLCD","eq":{"S":1.27,"O":12.75},"rank_tie":57,"tied":true,"parts":{"public":0,"same_skills":0.76,"new_domains":5.1}}},{"engine":"strands-decider-2b","name":"Strands Decider 2B","long_name":"Strands Decider 2B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":1900000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/StrandsAgents/strands-decider-2B-hobson-v21","code_url":null,"base_model":"Qwen/Qwen3.5-2B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B-Base","weights_repo":null,"served_checkpoint":null,"org":{"id":"StrandsAgents","name":"Strands Agents","logo":"data/orgs/StrandsAgents.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5406,"conf":0.5756,"ece":0.0441,"brier":0.552,"over95":0.0002,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0494,"conf":0.0483,"acc":0.0563},{"lo":0.1,"w":0.018,"conf":0.1432,"acc":0.1333},{"lo":0.2,"w":0.0327,"conf":0.2681,"acc":0.2413},{"lo":0.3,"w":0.093,"conf":0.3522,"acc":0.3652},{"lo":0.4,"w":0.1077,"conf":0.453,"acc":0.423},{"lo":0.5,"w":0.2117,"conf":0.5486,"acc":0.512},{"lo":0.6,"w":0.1853,"conf":0.6499,"acc":0.5543},{"lo":0.7,"w":0.1561,"conf":0.7463,"acc":0.6836},{"lo":0.8,"w":0.1024,"conf":0.8459,"acc":0.8629},{"lo":0.9,"w":0.0436,"conf":0.9316,"acc":0.9589}],"areas":{"knowledge":{"ece":0.0409,"acc":0.4866,"conf":0.5166},"language":{"ece":0.0175,"acc":0.6155,"conf":0.6201},"retrieval":{"ece":0.031,"acc":0.6852,"conf":0.6809},"tools":{"ece":0.1766,"acc":0.5525,"conf":0.6766},"arts":{"ece":0.0893,"acc":0.3912,"conf":0.4687}}},"gaps":null,"latency":{"median":52.3,"p95":126,"mean":74.7},"results":{"1":{"score":0.8135,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":284.3},"2":{"score":0.6047,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2295.4},"3":{"score":0.3957,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":430.4},"4":{"score":0.8036,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":142.6},"5":{"score":0.6469,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":216.3},"6":{"score":0.7886,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":271.5},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":402.9},"10":{"score":0.5961,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":153.9},"11":{"score":0.7302,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":337.8},"12":{"score":0.399,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":74.8},"20":{"score":0.56,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":72.5},"21":{"score":0.4973,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":74.1},"22":{"score":0.0685,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":255.2},"23":{"score":0.5066,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":170.6},"24":{"score":0.5096,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":111.1},"25":{"score":0.3316,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":121.1},"26":{"score":0.9049,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":111.1},"27":{"score":0.7108,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":114},"28":{"score":0.5738,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":112.7},"29":{"score":0.5741,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":114.2},"30":{"score":0.2468,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0916,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":193.6},"32":{"score":0.5106,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":187.2},"33":{"score":0.0976,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":259.6},"36":{"score":0.368,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3510.9},"37":{"score":0.2596,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.1},"38":{"score":0.0602,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":569.3},"39":{"score":0.7265,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":126.5},"40":{"score":0.2958,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":86.6},"41":{"score":0.4041,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":73.1},"42":{"score":0.6582,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":92.2},"43":{"score":0.3333,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":73.2},"44":{"score":0.5436,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.4},"45":{"score":0.1397,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":81.5},"50":{"score":0.3795,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":108.6},"56":{"score":0.5085,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":150.6},"57":{"score":0.3107,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":69.1},"58":{"score":0.4652,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":68},"59":{"score":0.3572,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":85.6},"61":{"score":0.5405,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":78.2},"62":{"score":0.5964,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":91.2},"64":{"score":0.4545,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":66.3}},"benchmarks":{"1":{"raw":0.8135,"skill":0.7482,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5447,"private_eq":0.6658},"2":{"raw":0.6047,"skill":0.5434,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6237,"private_eq":0.677},"3":{"raw":0.3957,"skill":0.3841,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2399,"private_eq":0.2618},"4":{"raw":0.8036,"skill":0.8011,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7069,"private_eq":0.7129},"5":{"raw":0.6469,"skill":0.6448,"coverage":1,"random":0.006,"tracks":[],"combined":0.3126,"private_eq":0.3235},"6":{"raw":0.7886,"skill":0.5043,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.5961,"skill":0.328,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7302,"skill":0.6099,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.399,"skill":0.0998,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2052,"private_eq":0.4748},"20":{"raw":0.5698,"skill":0.1397,"coverage":1,"random":0.5,"tracks":[],"combined":0.3288,"private_eq":0.6677},"21":{"raw":0.4973,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0642,"skill":0.0569,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1063,"private_eq":0.1224},"23":{"raw":0.5125,"skill":0.0251,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5096,"skill":0.3461,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3316,"skill":0.1088,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2884,"combined":0.0413},"26":{"raw":0.9049,"skill":0.8732,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7108,"skill":0.6143,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5738,"skill":0.1476,"coverage":1,"random":0.5,"tracks":[],"combined":0.0931,"private_eq":0.5512},"29":{"raw":0.5741,"skill":0.4321,"coverage":1,"random":0.25,"tracks":[],"combined":0.3791,"private_eq":0.5389},"30":{"raw":0.2468,"skill":0.0902,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3442,"random":0.25},{"track":"10-choice","raw":0.1494,"random":0.1}],"combined":0.2216,"private_eq":0.3643},"31":{"raw":0.0916,"skill":0.0106,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0844},"32":{"raw":0.5106,"skill":0.2219,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0976,"skill":0.0856,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0351,"private_eq":0.0576},"36":{"raw":0.368,"skill":0.285,"coverage":1,"random":0.116,"tracks":[],"combined":0.2507,"private_eq":0.3443},"37":{"raw":0.2596,"skill":0.0713,"coverage":1,"random":0.2027,"tracks":[],"combined":0.0796,"private_eq":0.2737},"38":{"raw":0.0602,"skill":0.0301,"coverage":1,"random":0.031,"tracks":[],"combined":0.048,"private_eq":0.0871},"39":{"raw":0.7265,"skill":0.5977,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4999,"private_eq":0.6632},"40":{"raw":0.2958,"skill":0.0941,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2167,"headline":false},{"track":"A · English","score":0.2958,"headline":true},{"track":"C · Arabic pairs","score":0.355,"headline":false},{"track":"C · English pairs","score":0.355,"headline":false}],"combined":0.0821,"private_eq":0.2939},"41":{"raw":0.4041,"skill":0.1062,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.6582,"skill":0.335,"coverage":1,"random":0.486,"tracks":[],"combined":0.3508,"private_eq":0.6696},"43":{"raw":0.3333,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1482,"private_eq":0.4686},"44":{"raw":0.5436,"skill":0.0872,"coverage":1,"random":0.5,"tracks":[],"combined":0.2353,"private_eq":0.6215},"45":{"raw":0.1397,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3795,"skill":0.0994,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.5085,"skill":0.017,"coverage":1,"random":0.5,"tracks":[],"combined":0.0492,"private_eq":0.5295},"57":{"raw":0.3107,"skill":0.2247,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2611,"combined":0.1604},"58":{"raw":0.4652,"skill":0.2248,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3531,"combined":0.0525},"59":{"raw":0.3572,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1441,"private_eq":0.5914},"61":{"raw":0.5405,"skill":0.081,"coverage":1,"random":0.5,"tracks":[],"combined":0.0308,"private_eq":0.5204},"62":{"raw":0.5964,"skill":0.4619,"coverage":1,"random":0.25,"tracks":[],"combined":0.2754,"private_eq":0.462},"64":{"raw":0.4545,"skill":0.3181,"coverage":1,"random":0.2,"tracks":[],"combined":0.3416,"private_eq":0.4785}},"coverage":1,"pending":0,"notes":["Lab run of the authors' strands-decider serve (v21, 4,096-token window; prompts over the window rerun with the window lifted). No Decision Index submission."],"scores":{"balanced_raw":44.43,"balanced_skill":20.72,"breadth_skill":23.26,"public_skill":25.41},"frozen_scores":{"balanced_raw":44.43,"balanced_skill":25.41,"breadth_skill":23.26},"categories":[{"id":"knowledge","raw":0.3341,"skill":0.1119,"combined":0.0962,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4681,"skill":0.2565,"combined":0.2162,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5289,"skill":0.3404,"combined":0.2551,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4911,"skill":0.4378,"combined":0.3389,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.413,"skill":0.1065,"combined":0.1909,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"strands-decider-2b","pub":25.41,"same_skills":22.43,"new_domains":13.52,"v":20.72,"lab":false,"jev":false,"rank_v":195,"rank_pub":205,"name":"Strands Decider 2B","eq":{"S":21.64,"O":19.34},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":12.98,"new_domains":7.74}}},{"engine":"d1-3b","name":"LiquidAI d1-3B","long_name":"LiquidAI d1-3B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":3120000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/LiquidAI/d1-3B","code_url":null,"base_model":"LiquidAI/LFM2.5-VL-3B","base_url":"https://huggingface.co/LiquidAI/LFM2.5-VL-3B","weights_repo":null,"served_checkpoint":null,"org":{"id":"LiquidAI","name":"Liquid AI","logo":"data/orgs/LiquidAI.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7024,"conf":0.7492,"ece":0.0468,"brier":0.3902,"over95":0.0082,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0065,"conf":0.0734,"acc":0.0788},{"lo":0.1,"w":0.018,"conf":0.1487,"acc":0.0985},{"lo":0.2,"w":0.0163,"conf":0.2513,"acc":0.2005},{"lo":0.3,"w":0.0426,"conf":0.3548,"acc":0.3084},{"lo":0.4,"w":0.0581,"conf":0.456,"acc":0.4026},{"lo":0.5,"w":0.1427,"conf":0.5506,"acc":0.5121},{"lo":0.6,"w":0.1056,"conf":0.6502,"acc":0.5785},{"lo":0.7,"w":0.1001,"conf":0.7491,"acc":0.6598},{"lo":0.8,"w":0.1198,"conf":0.8556,"acc":0.7493},{"lo":0.9,"w":0.3903,"conf":0.9623,"acc":0.9488}],"areas":{"knowledge":{"ece":0.0967,"acc":0.5605,"conf":0.657},"language":{"ece":0.0131,"acc":0.8251,"conf":0.832},"retrieval":{"ece":0.0335,"acc":0.7842,"conf":0.8118},"tools":{"ece":0.038,"acc":0.8905,"conf":0.9117},"arts":{"ece":0.0497,"acc":0.587,"conf":0.6367}}},"gaps":null,"latency":{"median":16.1,"p95":55.9,"mean":25.9},"results":{"1":{"score":0.8937,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":21.1},"2":{"score":0.6574,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":110.7},"3":{"score":0.7815,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":126.9},"4":{"score":0.8797,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":16.1},"5":{"score":0.9119,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":18.4},"6":{"score":0.7946,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":37},"9":{"score":0.6364,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":40.9},"10":{"score":0.692,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":26.8},"11":{"score":0.8057,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":52.8},"12":{"score":0.6238,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":11.2},"20":{"score":0.8236,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.2},"21":{"score":0.6362,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":10.8},"22":{"score":0.253,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":78.1},"23":{"score":0.6204,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.2},"24":{"score":0.6662,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":11},"25":{"score":0.2857,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":10.2},"26":{"score":0.9537,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":10.1},"27":{"score":0.8797,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":11.1},"28":{"score":0.7774,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":11.2},"29":{"score":0.8922,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":11.4},"30":{"score":0.3268,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1578,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.2},"32":{"score":0.5678,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":20.4},"33":{"score":0.3133,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":20.7},"36":{"score":0.4057,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":165.8},"37":{"score":0.5124,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.2},"38":{"score":0.377,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":92.2},"39":{"score":0.8804,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":14.1},"40":{"score":0.3597,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":12.1},"41":{"score":0.7696,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":11.1},"42":{"score":0.7412,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.1},"43":{"score":0.3105,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":11},"44":{"score":0.5658,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.1},"45":{"score":0.1417,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":12.1},"50":{"score":0.5877,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":19},"56":{"score":0.5385,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":18},"57":{"score":0.4485,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":12.4},"58":{"score":0.724,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":12.1},"59":{"score":0.7757,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":17.9},"61":{"score":0.7408,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":13.4},"62":{"score":0.8658,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":19.3},"64":{"score":0.6402,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":12.5}},"benchmarks":{"1":{"raw":0.8937,"skill":0.8566,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6605,"private_eq":0.7506},"2":{"raw":0.6574,"skill":0.6043,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6611,"private_eq":0.709},"3":{"raw":0.7815,"skill":0.7773,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7753,"private_eq":0.7811},"4":{"raw":0.8797,"skill":0.8782,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8025,"private_eq":0.8062},"5":{"raw":0.9119,"skill":0.9114,"coverage":1,"random":0.006,"tracks":[],"combined":0.6811,"private_eq":0.6856},"6":{"raw":0.7946,"skill":0.5184,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6364,"skill":0.6364,"coverage":1,"random":0,"tracks":[],"combined":0.2171,"private_eq":0.225},"10":{"raw":0.692,"skill":0.4875,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8057,"skill":0.719,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6238,"skill":0.4365,"coverage":1,"random":0.3324,"tracks":[],"combined":0.441,"private_eq":0.6304},"20":{"raw":0.8217,"skill":0.6435,"coverage":1,"random":0.5,"tracks":[],"combined":0.6035,"private_eq":0.8035},"21":{"raw":0.6362,"skill":0.2725,"coverage":1,"random":0.5,"tracks":[],"combined":0.3472,"private_eq":0.6768},"22":{"raw":0.2392,"skill":0.2333,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1032,"private_eq":0.1193},"23":{"raw":0.617,"skill":0.2341,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6662,"skill":0.5549,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2857,"skill":0.0476,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3339,"combined":0.1026},"26":{"raw":0.9537,"skill":0.9383,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8797,"skill":0.8396,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7774,"skill":0.5548,"coverage":1,"random":0.5,"tracks":[],"combined":0.2948,"private_eq":0.6509},"29":{"raw":0.8922,"skill":0.8563,"coverage":1,"random":0.25,"tracks":[],"combined":0.6614,"private_eq":0.7482},"30":{"raw":0.3268,"skill":0.1897,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4397,"random":0.25},{"track":"10-choice","raw":0.2138,"random":0.1}],"combined":0.3199,"private_eq":0.4445},"31":{"raw":0.1578,"skill":0.0827,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0907,"private_eq":0.1738},"32":{"raw":0.5678,"skill":0.3129,"coverage":1,"random":0.371,"tracks":[],"combined":0.1423,"private_eq":0.466},"33":{"raw":0.3133,"skill":0.3042,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2766,"private_eq":0.2932},"36":{"raw":0.4057,"skill":0.3277,"coverage":1,"random":0.116,"tracks":[],"combined":0.2796,"private_eq":0.3695},"37":{"raw":0.5124,"skill":0.3884,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4109,"private_eq":0.5349},"38":{"raw":0.377,"skill":0.3571,"coverage":1,"random":0.031,"tracks":[],"combined":0.4415,"private_eq":0.464},"39":{"raw":0.8804,"skill":0.8241,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5328,"private_eq":0.6853},"40":{"raw":0.3597,"skill":0.1763,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3584,"headline":false},{"track":"A · English","score":0.3597,"headline":true},{"track":"C · Arabic pairs","score":0.73,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}],"combined":0.1519,"private_eq":0.3475},"41":{"raw":0.7696,"skill":0.6545,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1374,"private_eq":0.4308},"42":{"raw":0.7412,"skill":0.4965,"coverage":1,"random":0.486,"tracks":[],"combined":0.6411,"private_eq":0.8171},"43":{"raw":0.3105,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2944,"private_eq":0.5597},"44":{"raw":0.5658,"skill":0.1316,"coverage":1,"random":0.5,"tracks":[],"combined":0.0587,"private_eq":0.5342},"45":{"raw":0.1417,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5877,"skill":0.4016,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.5385,"skill":0.077,"coverage":1,"random":0.5,"tracks":[],"combined":0.2395,"private_eq":0.6236},"57":{"raw":0.4485,"skill":0.3797,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.334,"combined":0.2433},"58":{"raw":0.724,"skill":0.5999,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3845,"combined":0.0986},"59":{"raw":0.7757,"skill":0.5349,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2894,"private_eq":0.6607},"61":{"raw":0.7408,"skill":0.4816,"coverage":1,"random":0.5,"tracks":[],"combined":0.4961,"private_eq":0.7504},"62":{"raw":0.8658,"skill":0.8211,"coverage":1,"random":0.25,"tracks":[],"combined":0.5493,"private_eq":0.6651},"64":{"raw":0.6402,"skill":0.5502,"coverage":1,"random":0.2,"tracks":[],"combined":0.6131,"private_eq":0.6932}},"coverage":1,"pending":0,"notes":["Submitted by LiquidAI (public suite run on AMD MI325X with the model card's model.system_one); private tests run by the lab the same way."],"scores":{"balanced_raw":62.38,"balanced_skill":31.12,"breadth_skill":46.22,"public_skill":48.99},"frozen_scores":{"balanced_raw":62.38,"balanced_skill":48.99,"breadth_skill":46.22},"categories":[{"id":"knowledge","raw":0.4246,"skill":0.2433,"combined":0.1916,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6945,"skill":0.5653,"combined":0.4187,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.671,"skill":0.5284,"combined":0.4943,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7722,"skill":0.7449,"combined":0.5834,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5903,"skill":0.3892,"combined":0.374,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"d1-3b","pub":48.99,"same_skills":41.23,"new_domains":11.47,"v":31.12,"lab":false,"jev":false,"rank_v":165,"rank_pub":123,"name":"LiquidAI d1-3B","eq":{"S":40.65,"O":16.81},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":24.39,"new_domains":6.72}}},{"engine":"d1-omni-600m","name":"LiquidAI d1-omni-600M","long_name":"LiquidAI d1-omni-600M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":587000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/LiquidAI/d1-omni-600M","code_url":null,"base_model":"LiquidAI/LFM2.5-Encoder-350M","base_url":"https://huggingface.co/LiquidAI/LFM2.5-Encoder-350M","weights_repo":null,"served_checkpoint":null,"org":{"id":"LiquidAI","name":"Liquid AI","logo":"data/orgs/LiquidAI.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5298,"conf":0.6571,"ece":0.1284,"brier":0.586,"over95":0.0163,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0211,"conf":0.0719,"acc":0.0968},{"lo":0.1,"w":0.0347,"conf":0.1623,"acc":0.0662},{"lo":0.2,"w":0.0273,"conf":0.2535,"acc":0.1784},{"lo":0.3,"w":0.0598,"conf":0.3513,"acc":0.2246},{"lo":0.4,"w":0.0755,"conf":0.4491,"acc":0.3376},{"lo":0.5,"w":0.1902,"conf":0.5479,"acc":0.4704},{"lo":0.6,"w":0.1514,"conf":0.6477,"acc":0.4947},{"lo":0.7,"w":0.12,"conf":0.7494,"acc":0.5872},{"lo":0.8,"w":0.1154,"conf":0.8495,"acc":0.6615},{"lo":0.9,"w":0.2045,"conf":0.9676,"acc":0.8335}],"areas":{"knowledge":{"ece":0.1714,"acc":0.4079,"conf":0.5729},"language":{"ece":0.1244,"acc":0.6203,"conf":0.7448},"retrieval":{"ece":0.1115,"acc":0.7266,"conf":0.8381},"tools":{"ece":0.0599,"acc":0.6451,"conf":0.6942},"arts":{"ece":0.1167,"acc":0.3838,"conf":0.5005}}},"gaps":null,"latency":{"median":10.9,"p95":34,"mean":14.5},"results":{"1":{"score":0.4764,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":11.4},"2":{"score":0.1566,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":384.7},"3":{"score":0.3996,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":56.1},"4":{"score":0.837,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":11},"5":{"score":0.8528,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":15.1},"6":{"score":0.5979,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":21.4},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":79},"10":{"score":0.6129,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":11.5},"11":{"score":0.6618,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":145.3},"12":{"score":0.3599,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":11.6},"20":{"score":0.4572,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.5},"21":{"score":0.5114,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":7.5},"22":{"score":0.008,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":50.8},"23":{"score":0.5638,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":8.2},"24":{"score":0.399,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":7},"25":{"score":0.2194,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":11.7},"26":{"score":0.702,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6.8},"27":{"score":0.5137,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":7.5},"28":{"score":0.6046,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":6.6},"29":{"score":0.5998,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":12.3},"30":{"score":0.1835,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1294,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.3},"32":{"score":0.3803,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":16.5},"33":{"score":0.0739,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":18.5},"36":{"score":0.1039,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":413.5},"37":{"score":0.2726,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":8},"38":{"score":0.005,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":48.4},"39":{"score":0.6293,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":8.5},"40":{"score":0.2745,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":7.6},"41":{"score":0.3878,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":11.8},"42":{"score":0.5325,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":8.1},"43":{"score":0.4088,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":7.7},"44":{"score":0.4966,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.5},"45":{"score":0.1477,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":8.4},"50":{"score":0.3628,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":14.9},"56":{"score":0.4475,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":13.9},"57":{"score":0.2003,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":7.8},"58":{"score":0.4324,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":7.8},"59":{"score":0.0228,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":9.2},"61":{"score":0.598,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":8.5},"62":{"score":0.4077,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":13.9},"64":{"score":0.3598,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":8.2}},"benchmarks":{"1":{"raw":0.4764,"skill":0.2931,"coverage":1,"random":0.2592,"tracks":[],"combined":0.17,"private_eq":0.3914},"2":{"raw":0.1566,"skill":0.026,"coverage":1,"random":0.1341,"tracks":[],"combined":0.0049,"private_eq":0.1473},"3":{"raw":0.3996,"skill":0.388,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3134,"private_eq":0.3331},"4":{"raw":0.837,"skill":0.8349,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6722,"private_eq":0.6791},"5":{"raw":0.8528,"skill":0.8519,"coverage":1,"random":0.006,"tracks":[],"combined":0.4413,"private_eq":0.45},"6":{"raw":0.5979,"skill":0.0572,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.6129,"skill":0.3559,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6618,"skill":0.5109,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3599,"skill":0.0412,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.4615,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.5114,"skill":0.0228,"coverage":1,"random":0.5,"tracks":[],"combined":0.0352,"private_eq":0.5226},"22":{"raw":0.0082,"skill":0.0005,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0106,"private_eq":0.0285},"23":{"raw":0.5464,"skill":0.0927,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.399,"skill":0.1987,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2194,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2685,"combined":0.0144},"26":{"raw":0.702,"skill":0.6026,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5137,"skill":0.3514,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6046,"skill":0.2092,"coverage":1,"random":0.5,"tracks":[],"combined":0.2545,"private_eq":0.631},"29":{"raw":0.5998,"skill":0.4664,"coverage":1,"random":0.25,"tracks":[],"combined":0.2256,"private_eq":0.4251},"30":{"raw":0.1835,"skill":0.0109,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2631,"random":0.25},{"track":"10-choice","raw":0.1039,"random":0.1}],"combined":0.0458,"private_eq":0.2209},"31":{"raw":0.1294,"skill":0.0518,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0127,"private_eq":0.103},"32":{"raw":0.3803,"skill":0.0148,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0739,"skill":0.0616,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0556,"private_eq":0.0776},"36":{"raw":0.1039,"skill":0,"coverage":1,"random":0.116,"tracks":[],"combined":0.0933,"private_eq":0.2067},"37":{"raw":0.2726,"skill":0.0876,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.005,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.6293,"skill":0.4548,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3527,"private_eq":0.5642},"40":{"raw":0.2745,"skill":0.0667,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2519,"headline":false},{"track":"A · English","score":0.2745,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.395,"headline":false}],"combined":0.0743,"private_eq":0.2879},"41":{"raw":0.3878,"skill":0.0816,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5325,"skill":0.0905,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4088,"skill":0.062,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0269,"private_eq":0.393},"44":{"raw":0.4966,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"45":{"raw":0.1477,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3628,"skill":0.0752,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.4475,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0492,"private_eq":0.5295},"57":{"raw":0.2003,"skill":0.1006,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2149,"combined":0.1078},"58":{"raw":0.4324,"skill":0.1773,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3169,"combined":0},"59":{"raw":0.0228,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.598,"skill":0.196,"coverage":1,"random":0.5,"tracks":[],"combined":0.0819,"private_eq":0.5457},"62":{"raw":0.4077,"skill":0.2103,"coverage":1,"random":0.25,"tracks":[],"combined":0.1797,"private_eq":0.391},"64":{"raw":0.3598,"skill":0.1997,"coverage":1,"random":0.2,"tracks":[],"combined":0.2543,"private_eq":0.4094}},"coverage":1,"pending":0,"notes":["Submitted by LiquidAI (public suite run on AMD MI325X with the model card's model.system_one); requests over the model's 16,384-token default budget were rerun by the lab with the budget lifted to 128,000, within its 128k context; private tests run by the lab the same way."],"scores":{"balanced_raw":37.58,"balanced_skill":11.75,"breadth_skill":16.11,"public_skill":17.9},"frozen_scores":{"balanced_raw":37.58,"balanced_skill":17.9,"breadth_skill":16.11},"categories":[{"id":"knowledge","raw":0.2998,"skill":0.0655,"combined":0.0529,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.3899,"skill":0.1929,"combined":0.0766,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5258,"skill":0.3497,"combined":0.2377,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.2992,"skill":0.1951,"combined":0.1397,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.375,"skill":0.0651,"combined":0.0971,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"d1-omni-600m","pub":17.9,"same_skills":12.86,"new_domains":7.13,"v":11.75,"lab":false,"jev":false,"rank_v":224,"rank_pub":220,"name":"LiquidAI d1-omni-600M","eq":{"S":11.96,"O":11.45},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":7.17,"new_domains":4.58}}},{"engine":"clef-omni","name":"Cloudflare clef-omni (30B-A3B)","long_name":"Cloudflare clef-omni (30B-A3B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":31830000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Cloudflare/clef-omni","code_url":null,"base_model":"Qwen/Qwen3-Omni-30B-A3B-Instruct","base_url":"https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct","weights_repo":null,"served_checkpoint":null,"org":{"id":"Cloudflare","name":"Cloudflare","logo":"data/orgs/Cloudflare.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7715,"conf":0.7844,"ece":0.0403,"brier":0.3234,"over95":0.0209,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0392,"conf":0.038,"acc":0.048},{"lo":0.1,"w":0.0168,"conf":0.1359,"acc":0.1683},{"lo":0.2,"w":0.0094,"conf":0.2552,"acc":0.2148},{"lo":0.3,"w":0.0241,"conf":0.3598,"acc":0.3763},{"lo":0.4,"w":0.0338,"conf":0.4477,"acc":0.429},{"lo":0.5,"w":0.0619,"conf":0.5505,"acc":0.5638},{"lo":0.6,"w":0.0888,"conf":0.6547,"acc":0.7397},{"lo":0.7,"w":0.0969,"conf":0.748,"acc":0.7891},{"lo":0.8,"w":0.1274,"conf":0.8568,"acc":0.773},{"lo":0.9,"w":0.5016,"conf":0.958,"acc":0.9284}],"areas":{"knowledge":{"ece":0.0419,"acc":0.7371,"conf":0.7533},"language":{"ece":0.0391,"acc":0.8468,"conf":0.886},"retrieval":{"ece":0.2465,"acc":0.7425,"conf":0.7866},"tools":{"ece":0.0826,"acc":0.9532,"conf":0.8733},"arts":{"ece":0.0209,"acc":0.619,"conf":0.6086}}},"gaps":null,"latency":{"median":191.3,"p95":281.5,"mean":213.7},"results":{"1":{"score":0.9864,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":196.8},"2":{"score":0.6524,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":660.2},"3":{"score":0.8701,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":462.3},"4":{"score":0.9423,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":205},"5":{"score":0.9751,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":281},"6":{"score":0.7996,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":237.4},"9":{"score":0.6818,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":276},"10":{"score":0.4306,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":197.7},"11":{"score":0.8237,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":323.4},"12":{"score":0.6363,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":182.1},"20":{"score":0.9728,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":184.9},"21":{"score":0.7108,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":177.5},"22":{"score":0.022,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":476.4},"23":{"score":0.6324,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":187.9},"24":{"score":0.9235,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":178.6},"25":{"score":0.4694,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":181.5},"26":{"score":0.9945,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":176.6},"27":{"score":0.9787,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":178},"28":{"score":0.9676,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":174.9},"29":{"score":0.9895,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":180.8},"30":{"score":0.6785,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2192,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":194.2},"32":{"score":0.8431,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":192.3},"33":{"score":0.3133,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":202.5},"36":{"score":0.4422,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":925.5},"37":{"score":0.5784,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":188.6},"38":{"score":0.242,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":675.7},"39":{"score":0.9665,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":188.6},"40":{"score":0.3725,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":178.1},"41":{"score":0.5145,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":181.7},"42":{"score":0.779,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":190},"43":{"score":0.8544,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":180.4},"44":{"score":0.972,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":183.9},"45":{"score":0.0918,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":186.7},"50":{"score":0.6999,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":194},"56":{"score":0.7245,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":202},"57":{"score":0.6927,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":182.3},"58":{"score":0.6895,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":179.1},"59":{"score":0.3451,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":195.3},"61":{"score":0.6245,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":187.8},"62":{"score":0.5665,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":195.4},"64":{"score":0.6648,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":183.8}},"benchmarks":{"1":{"raw":0.9864,"skill":0.9817,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6524,"skill":0.5986,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6869,"private_eq":0.7311},"3":{"raw":0.8701,"skill":0.8676,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7018,"private_eq":0.7098},"4":{"raw":0.9423,"skill":0.9416,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8696,"private_eq":0.8717},"5":{"raw":0.9751,"skill":0.9749,"coverage":1,"random":0.006,"tracks":[],"combined":0.7146,"private_eq":0.7185},"6":{"raw":0.7996,"skill":0.5301,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6818,"skill":0.6818,"coverage":1,"random":0,"tracks":[],"combined":0.2778,"private_eq":0.285},"10":{"raw":0.4306,"skill":0.0526,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8237,"skill":0.745,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6363,"skill":0.4552,"coverage":1,"random":0.3324,"tracks":[],"combined":0.463,"private_eq":0.6449},"20":{"raw":0.9711,"skill":0.9422,"coverage":1,"random":0.5,"tracks":[],"combined":0.8493,"private_eq":0.925},"21":{"raw":0.7108,"skill":0.4216,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.02,"skill":0.0123,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0095},"23":{"raw":0.6316,"skill":0.2632,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.9235,"skill":0.898,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4694,"skill":0.2925,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4212,"combined":0.2204},"26":{"raw":0.9945,"skill":0.9927,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9787,"skill":0.9716,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9676,"skill":0.9352,"coverage":1,"random":0.5,"tracks":[],"combined":0.4512,"private_eq":0.7282},"29":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6785,"skill":0.6162,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7604,"random":0.25},{"track":"10-choice","raw":0.5967,"random":0.1}],"combined":0.688,"private_eq":0.7447},"31":{"raw":0.2192,"skill":0.1496,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0615,"private_eq":0.1473},"32":{"raw":0.8431,"skill":0.7506,"coverage":1,"random":0.371,"tracks":[],"combined":0.1423,"private_eq":0.466},"33":{"raw":0.3133,"skill":0.3042,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2663,"private_eq":0.2832},"36":{"raw":0.4422,"skill":0.3689,"coverage":1,"random":0.116,"tracks":[],"combined":0.3471,"private_eq":0.4285},"37":{"raw":0.5784,"skill":0.4712,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5516,"private_eq":0.6458},"38":{"raw":0.242,"skill":0.2178,"coverage":1,"random":0.031,"tracks":[],"combined":0.3704,"private_eq":0.3959},"39":{"raw":0.9665,"skill":0.9507,"coverage":1,"random":0.3201,"tracks":[],"combined":0.57,"private_eq":0.7103},"40":{"raw":0.3725,"skill":0.1927,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.7974,"headline":false},{"track":"A · English","score":0.3725,"headline":true},{"track":"C · Arabic pairs","score":0.96,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.5145,"skill":0.2718,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.779,"skill":0.57,"coverage":1,"random":0.486,"tracks":[],"combined":0.7552,"private_eq":0.8751},"43":{"raw":0.8544,"skill":0.769,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3363,"private_eq":0.5858},"44":{"raw":0.972,"skill":0.944,"coverage":1,"random":0.5,"tracks":[],"combined":0.8137,"private_eq":0.9074},"45":{"raw":0.0918,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6999,"skill":0.5644,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.7245,"skill":0.449,"coverage":1,"random":0.5,"tracks":[],"combined":0.5944,"private_eq":0.799},"57":{"raw":0.6927,"skill":0.6544,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3808,"combined":0.2966},"58":{"raw":0.6895,"skill":0.5499,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4378,"combined":0.1767},"59":{"raw":0.3451,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6245,"skill":0.249,"coverage":1,"random":0.5,"tracks":[],"combined":0.2691,"private_eq":0.6382},"62":{"raw":0.5665,"skill":0.422,"coverage":1,"random":0.25,"tracks":[],"combined":0.3029,"private_eq":0.4824},"64":{"raw":0.6648,"skill":0.581,"coverage":1,"random":0.2,"tracks":[],"combined":0.5452,"private_eq":0.6395}},"coverage":1,"pending":0,"notes":["Lab run of the model card's joint_schema_model path (load_release_model, encode_record, one record per forward), bf16; context raised from the code default 64,000 to the backbone window 65,536 tokens (33 ToolRet requests whose options alone exceed it count as wrong). No Decision Index submission."],"scores":{"balanced_raw":66.94,"balanced_skill":45.04,"breadth_skill":56.04,"public_skill":56.54},"frozen_scores":{"balanced_raw":66.94,"balanced_skill":56.54,"breadth_skill":56.04},"categories":[{"id":"knowledge","raw":0.6145,"skill":0.5377,"combined":0.3409,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6377,"skill":0.4976,"combined":0.4073,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.721,"skill":0.5927,"combined":0.5655,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7645,"skill":0.7262,"combined":0.5995,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6164,"skill":0.4641,"combined":0.3783,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"clef-omni","pub":56.54,"same_skills":46.51,"new_domains":33.17,"v":45.04,"lab":false,"jev":false,"rank_v":111,"rank_pub":94,"name":"Cloudflare clef-omni (30B-A3B)","eq":{"S":45.99,"O":43.61},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":27.6,"new_domains":17.44}}},{"engine":"drex-v1.5","name":"Nace.AI Drex v1.5","long_name":"Nace.AI Drex v1.5","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":8950000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/nace-ai/drex-v1.5","code_url":null,"base_model":"XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B","base_url":"https://huggingface.co/XiaomiMiMo/MiMo-V2.6-Distill-Qwen-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"nace-ai","name":"Nace.AI","logo":"data/orgs/nace-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7354,"conf":0.8238,"ece":0.0897,"brier":0.3829,"over95":0.054,"n":34412,"benchmarks":32,"rel":[{"lo":0,"w":0.0098,"conf":0.0737,"acc":0.1401},{"lo":0.1,"w":0.0116,"conf":0.143,"acc":0.1475},{"lo":0.2,"w":0.0076,"conf":0.2522,"acc":0.2146},{"lo":0.3,"w":0.0265,"conf":0.3595,"acc":0.328},{"lo":0.4,"w":0.0357,"conf":0.4551,"acc":0.4464},{"lo":0.5,"w":0.0879,"conf":0.5483,"acc":0.4929},{"lo":0.6,"w":0.083,"conf":0.6535,"acc":0.5996},{"lo":0.7,"w":0.0851,"conf":0.7511,"acc":0.6487},{"lo":0.8,"w":0.0945,"conf":0.8522,"acc":0.7007},{"lo":0.9,"w":0.5582,"conf":0.9794,"acc":0.8805}],"areas":{"knowledge":{"ece":0.1622,"acc":0.6458,"conf":0.8045},"language":{"ece":0.0473,"acc":0.8454,"conf":0.8923},"retrieval":{"ece":0.069,"acc":0.7648,"conf":0.8338},"tools":{"ece":0.0615,"acc":0.9207,"conf":0.9734},"arts":{"ece":0.0304,"acc":0.6799,"conf":0.6801}}},"gaps":null,"latency":{"median":53.5,"p95":342.3,"mean":127.5},"results":{"1":{"score":0.9481,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.7058,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8563,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8544,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9179,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7994,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.3864,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6553,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8323,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7032,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.8216,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6305,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.763,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.5834,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.785,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4439,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.979,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9582,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8777,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9207,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.5462,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2192,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.5785,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2673,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4447,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5619,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.0504,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.8805,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.552,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7703,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8027,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.7596,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8632,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1118,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.5967,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.6645,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.5869,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.8562,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8288,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.861,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8962,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7292,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9481,"skill":0.9299,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.7058,"skill":0.6602,"coverage":1,"random":0.1341,"tracks":[],"combined":0.684,"private_eq":0.7286},"3":{"raw":0.8563,"skill":0.8535,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.8544,"skill":0.8525,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8044,"private_eq":0.8081},"5":{"raw":0.9179,"skill":0.9174,"coverage":1,"random":0.006,"tracks":[],"combined":0.6859,"private_eq":0.6903},"6":{"raw":0.7994,"skill":0.5297,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3864,"skill":0.3864,"coverage":1,"random":0,"tracks":[],"combined":0.3435,"private_eq":0.35},"10":{"raw":0.6553,"skill":0.4265,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8323,"skill":0.7574,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7032,"skill":0.5554,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4437,"private_eq":0.6322},"20":{"raw":0.8157,"skill":0.6314,"coverage":1,"random":0.5,"tracks":[],"combined":0.7479,"private_eq":0.8749},"21":{"raw":0.6305,"skill":0.261,"coverage":1,"random":0.5,"tracks":[],"combined":0.3672,"private_eq":0.6867},"22":{"raw":0.7609,"skill":0.7591,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5716,"private_eq":0.5788},"23":{"raw":0.5992,"skill":0.1984,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.785,"skill":0.7133,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.389,"combined":0.177},"26":{"raw":0.979,"skill":0.972,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9582,"skill":0.9443,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8777,"skill":0.7554,"coverage":1,"random":0.5,"tracks":[],"combined":0.4916,"private_eq":0.7482},"29":{"raw":0.9207,"skill":0.8943,"coverage":1,"random":0.25,"tracks":[],"combined":0.7801,"private_eq":0.8362},"30":{"raw":0.5462,"skill":0.457,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6505,"random":0.25},{"track":"10-choice","raw":0.442,"random":0.1}],"combined":0.5441,"private_eq":0.6273},"31":{"raw":0.2192,"skill":0.1496,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0713,"private_eq":0.1562},"32":{"raw":0.5785,"skill":0.3299,"coverage":1,"random":0.371,"tracks":[],"combined":0.2892,"private_eq":0.5574},"33":{"raw":0.2673,"skill":0.2576,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0247,"private_eq":0.0475},"36":{"raw":0.4447,"skill":0.3718,"coverage":1,"random":0.116,"tracks":[],"combined":0.3503,"private_eq":0.4313},"37":{"raw":0.5619,"skill":0.4505,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5031,"private_eq":0.6075},"38":{"raw":0.0504,"skill":0.02,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.8805,"skill":0.8242,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5184,"private_eq":0.6756},"40":{"raw":0.552,"skill":0.4236,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4397,"headline":false},{"track":"A · English","score":0.552,"headline":true},{"track":"C · Arabic pairs","score":0.955,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.4045,"private_eq":0.5416},"41":{"raw":0.7703,"skill":0.6554,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.8027,"skill":0.6161,"coverage":1,"random":0.486,"tracks":[],"combined":0.7944,"private_eq":0.895},"43":{"raw":0.7596,"skill":0.6187,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3279,"private_eq":0.5806},"44":{"raw":0.8632,"skill":0.7264,"coverage":1,"random":0.5,"tracks":[],"combined":0.6476,"private_eq":0.8253},"45":{"raw":0.1118,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5967,"skill":0.4146,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.6645,"skill":0.329,"coverage":1,"random":0.5,"tracks":[],"combined":0.5857,"private_eq":0.7947},"57":{"raw":0.5869,"skill":0.5354,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4785,"combined":0.4077},"58":{"raw":0.8562,"skill":0.7916,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4172,"combined":0.1465},"59":{"raw":0.8288,"skill":0.645,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2603,"private_eq":0.6468},"61":{"raw":0.861,"skill":0.722,"coverage":1,"random":0.5,"tracks":[],"combined":0.6096,"private_eq":0.8065},"62":{"raw":0.8962,"skill":0.8616,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.7292,"skill":0.6615,"coverage":1,"random":0.2,"tracks":[],"combined":0.6519,"private_eq":0.7239}},"coverage":1,"pending":0,"notes":["Submitted by Nace.AI (public suite run in FP8 on NVIDIA B200 through the authors' Kev /v1/systemone server and the kit's http engine); private tests run by the lab in bf16 on one RTX PRO 6000 with the model card's inference.py path, context lifted to 131,072.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":68.08,"balanced_skill":45.78,"breadth_skill":57.27,"public_skill":58.08},"frozen_scores":{"balanced_raw":68.08,"balanced_skill":58.08,"breadth_skill":57.27},"categories":[{"id":"knowledge","raw":0.5571,"skill":0.4457,"combined":0.3122,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7091,"skill":0.6044,"combined":0.4078,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7194,"skill":0.6169,"combined":0.592,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7692,"skill":0.7497,"combined":0.712,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6887,"skill":0.4877,"combined":0.5011,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"drex-v1.5","pub":58.08,"same_skills":49.65,"new_domains":30.82,"v":45.78,"lab":false,"jev":false,"rank_v":107,"rank_pub":77,"name":"Nace.AI Drex v1.5","eq":{"S":49.17,"O":40.7},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":29.5,"new_domains":16.28}}},{"engine":"decisiontune-1.0","name":"DecisionTune 1.0","long_name":"DecisionTune 1.0","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":394781696,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/decision-tune/decisiontune-1.0","code_url":null,"base_model":"answerdotai/ModernBERT-large","base_url":"https://huggingface.co/answerdotai/ModernBERT-large","weights_repo":null,"served_checkpoint":null,"org":{"id":"decision-tune","name":"DecisionTune","logo":"data/orgs/decision-tune.png"}},"completed":110201,"counts":{"ok":110195,"unsupported":6,"error":0},"lifted":false,"calibration":{"acc":0.5507,"conf":0.6983,"ece":0.1475,"brier":0.6027,"over95":0.0513,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0071,"conf":0.0811,"acc":0.0611},{"lo":0.1,"w":0.0471,"conf":0.16,"acc":0.0295},{"lo":0.2,"w":0.0165,"conf":0.2523,"acc":0.1296},{"lo":0.3,"w":0.0497,"conf":0.3542,"acc":0.2682},{"lo":0.4,"w":0.0647,"conf":0.4516,"acc":0.3441},{"lo":0.5,"w":0.1448,"conf":0.5472,"acc":0.4286},{"lo":0.6,"w":0.1178,"conf":0.6479,"acc":0.4828},{"lo":0.7,"w":0.1498,"conf":0.7564,"acc":0.6788},{"lo":0.8,"w":0.1572,"conf":0.8457,"acc":0.7321},{"lo":0.9,"w":0.2454,"conf":0.9674,"acc":0.7153}],"areas":{"knowledge":{"ece":0.1948,"acc":0.3991,"conf":0.5939},"language":{"ece":0.1255,"acc":0.6687,"conf":0.7942},"retrieval":{"ece":0.1389,"acc":0.78,"conf":0.7961},"tools":{"ece":0.2504,"acc":0.6573,"conf":0.9077},"arts":{"ece":0.1358,"acc":0.3944,"conf":0.5302}}},"gaps":null,"latency":{"median":13.8,"p95":111,"mean":37.3},"results":{"1":{"score":0.8046,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":115},"2":{"score":0.5689,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":1155.9},"3":{"score":0.4626,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":802.1},"4":{"score":0.9223,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":42.2},"5":{"score":0.919,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":39.3},"6":{"score":0.5464,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":286.3},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1248.3},"10":{"score":0.6058,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":97.9},"11":{"score":0.8212,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":3360.2},"12":{"score":0.2972,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":15.5},"20":{"score":0.5992,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.5},"21":{"score":0.4673,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":10.5},"22":{"score":0.0145,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":506.5},"23":{"score":0.4992,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.6},"24":{"score":0.3522,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":18.7},"25":{"score":0.2245,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":26.4},"26":{"score":0.601,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.9},"27":{"score":0.4462,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":19.7},"28":{"score":0.719,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":18.5},"29":{"score":0.8386,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":19.5},"30":{"score":0.1907,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.07,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":59.9},"32":{"score":0.5,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":78.8},"33":{"score":0.0303,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":226.9},"36":{"score":0.1845,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1635.4},"37":{"score":0.4999,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.1},"38":{"score":0.0633,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":495.5},"39":{"score":0.8486,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":18.4},"40":{"score":0.2075,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":11.6},"41":{"score":0.36,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":14.7},"42":{"score":0.4903,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":34.9},"43":{"score":0.407,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":13.9},"44":{"score":0.7238,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.8},"45":{"score":0.136,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":17.6},"50":{"score":0.3723,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":72.8},"56":{"score":0.501,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":148.1},"57":{"score":0.1256,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":14.8},"58":{"score":0.375,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":14.4},"59":{"score":0.6573,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":60.7},"61":{"score":0.6478,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":34.7},"62":{"score":0.6903,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":57.5},"64":{"score":0.25,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":16.5}},"benchmarks":{"1":{"raw":0.8046,"skill":0.7362,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6059,"private_eq":0.7106},"2":{"raw":0.5648,"skill":0.4974,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.437,"private_eq":0.5172},"3":{"raw":0.4626,"skill":0.4522,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2924,"private_eq":0.3127},"4":{"raw":0.9223,"skill":0.9213,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6913,"private_eq":0.6977},"5":{"raw":0.919,"skill":0.9185,"coverage":1,"random":0.006,"tracks":[],"combined":0.5105,"private_eq":0.518},"6":{"raw":0.5464,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.6058,"skill":0.3441,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.808,"skill":0.7223,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.2972,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3346},"20":{"raw":0.5922,"skill":0.1844,"coverage":1,"random":0.5,"tracks":[],"combined":0.1262,"private_eq":0.5676},"21":{"raw":0.4673,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.014,"skill":0.0063,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.4942,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3522,"skill":0.1363,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2245,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.254,"combined":0},"26":{"raw":0.601,"skill":0.4679,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.4462,"skill":0.2614,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.719,"skill":0.438,"coverage":1,"random":0.5,"tracks":[],"combined":0.436,"private_eq":0.7207},"29":{"raw":0.8386,"skill":0.7848,"coverage":1,"random":0.25,"tracks":[],"combined":0.5673,"private_eq":0.6784},"30":{"raw":0.1907,"skill":0.0197,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2707,"random":0.25},{"track":"10-choice","raw":0.1107,"random":0.1}],"combined":0.0243,"private_eq":0.2034},"31":{"raw":0.07,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.5,"skill":0.2051,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0303,"skill":0.0174,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.1845,"skill":0.0775,"coverage":1,"random":0.116,"tracks":[],"combined":0.1736,"private_eq":0.2769},"37":{"raw":0.4999,"skill":0.3727,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3331,"private_eq":0.4735},"38":{"raw":0.0633,"skill":0.0333,"coverage":1,"random":0.031,"tracks":[],"combined":0.0368,"private_eq":0.0763},"39":{"raw":0.8486,"skill":0.7773,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5734,"private_eq":0.7126},"40":{"raw":0.2075,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1722,"headline":false},{"track":"A · English","score":0.2075,"headline":true},{"track":"C · Arabic pairs","score":0.425,"headline":false},{"track":"C · English pairs","score":0.275,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.36,"skill":0.04,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4903,"skill":0.0084,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.407,"skill":0.0592,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0604,"private_eq":0.4139},"44":{"raw":0.7238,"skill":0.4476,"coverage":1,"random":0.5,"tracks":[],"combined":0.487,"private_eq":0.7459},"45":{"raw":0.1357,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.3723,"skill":0.089,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.501,"skill":0.002,"coverage":1,"random":0.5,"tracks":[],"combined":0.0405,"private_eq":0.5252},"57":{"raw":0.1256,"skill":0.0165,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1885,"combined":0.0778},"58":{"raw":0.375,"skill":0.0941,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3138,"combined":0},"59":{"raw":0.6573,"skill":0.2894,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6478,"skill":0.2956,"coverage":1,"random":0.5,"tracks":[],"combined":0.2068,"private_eq":0.6074},"62":{"raw":0.6903,"skill":0.5871,"coverage":1,"random":0.25,"tracks":[],"combined":0.4741,"private_eq":0.6093},"64":{"raw":0.25,"skill":0.0625,"coverage":1,"random":0.2,"tracks":[],"combined":0.0408,"private_eq":0.2406}},"coverage":0.9995,"pending":0.0005,"notes":["New in the 2026-10-10 PR batch (PR #69).","Submitted public 0.3 run adopted (kit rescore 28.85, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.66% on 891 rows (bar 98.5%).","Distilled from Cloudflare clef-flash soft labels; trains on train splits of benchmark source datasets; card discloses eight development runs of the full index (author-declared)."],"scores":{"balanced_raw":46.84,"balanced_skill":17.22,"breadth_skill":25.43,"public_skill":28.85},"frozen_scores":{"balanced_raw":46.84,"balanced_skill":28.85,"breadth_skill":25.43},"categories":[{"id":"knowledge","raw":0.3181,"skill":0.1173,"combined":0.106,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.5104,"skill":0.2992,"combined":0.1477,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.6181,"skill":0.4502,"combined":0.338,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.514,"skill":0.4649,"combined":0.3664,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.365,"skill":0.057,"combined":0.0583,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"decisiontune-1.0","pub":28.85,"same_skills":22.11,"new_domains":6.82,"v":17.22,"lab":false,"jev":false,"rank_v":207,"rank_pub":193,"name":"DecisionTune 1.0","eq":{"S":21.32,"O":11.07},"rank_tie":48,"tied":true,"parts":{"public":0,"same_skills":12.79,"new_domains":4.43}}},{"engine":"vera-spark","name":"vera-spark","long_name":"vera-spark","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":638468358,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/scar-ai/vera-spark","code_url":null,"base_model":"answerdotai/ModernBERT-large","base_url":"https://huggingface.co/answerdotai/ModernBERT-large","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5214,"conf":0.6768,"ece":0.1572,"brier":0.631,"over95":0.0227,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.04,"conf":0.0637,"acc":0.0183},{"lo":0.1,"w":0.0343,"conf":0.1441,"acc":0.1222},{"lo":0.2,"w":0.0249,"conf":0.251,"acc":0.288},{"lo":0.3,"w":0.0522,"conf":0.3526,"acc":0.2711},{"lo":0.4,"w":0.0691,"conf":0.4516,"acc":0.3291},{"lo":0.5,"w":0.1207,"conf":0.549,"acc":0.4383},{"lo":0.6,"w":0.1228,"conf":0.6489,"acc":0.5024},{"lo":0.7,"w":0.129,"conf":0.7508,"acc":0.5633},{"lo":0.8,"w":0.1706,"conf":0.8528,"acc":0.6158},{"lo":0.9,"w":0.2363,"conf":0.9526,"acc":0.762}],"areas":{"knowledge":{"ece":0.222,"acc":0.4234,"conf":0.6453},"language":{"ece":0.179,"acc":0.6482,"conf":0.8271},"retrieval":{"ece":0.0822,"acc":0.5543,"conf":0.5788},"tools":{"ece":0.0836,"acc":0.5779,"conf":0.6605},"arts":{"ece":0.132,"acc":0.429,"conf":0.5571}}},"gaps":null,"latency":{"median":79.9,"p95":129.8,"mean":89.2},"results":{"1":{"score":0.4917,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":74.9},"2":{"score":0.3934,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":140.3},"3":{"score":0.2264,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":162.6},"4":{"score":0.6854,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.4},"5":{"score":0.4069,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":39.5},"6":{"score":0.5802,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":101.8},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":171},"10":{"score":0.4942,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":74},"11":{"score":0.5594,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":306.1},"12":{"score":0.4194,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":70.9},"20":{"score":0.6622,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.9},"21":{"score":0.5426,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":70.9},"22":{"score":0.0165,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":93},"23":{"score":0.509,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":72.9},"24":{"score":0.4253,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":70.9},"25":{"score":0.2755,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":71.8},"26":{"score":0.7252,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":70.9},"27":{"score":0.5956,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":70.9},"28":{"score":0.7174,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":70.9},"29":{"score":0.7845,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":70.9},"30":{"score":0.1895,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0736,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":75.7},"32":{"score":0.5213,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":73.1},"33":{"score":0.0152,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":78.9},"36":{"score":0.254,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":295.6},"37":{"score":0.2753,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":71.9},"38":{"score":0.0793,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":74.7},"39":{"score":0.5631,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":76.7},"40":{"score":0.3839,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":71.9},"41":{"score":0.4269,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":71.8},"42":{"score":0.592,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":72.7},"43":{"score":0.4018,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":71.8},"44":{"score":0.5324,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.9},"45":{"score":0.1557,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":71.9},"50":{"score":0.4087,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":72.9},"56":{"score":0.428,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":76.9},"57":{"score":0.2268,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":71.9},"58":{"score":0.3777,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":70.9},"59":{"score":0.3812,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":72.9},"61":{"score":0.7027,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":71.9},"62":{"score":0.2842,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":72.8},"64":{"score":0.3542,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":70.9}},"benchmarks":{"1":{"raw":0.4917,"skill":0.3139,"coverage":1,"random":0.2592,"tracks":[],"combined":0.2313,"private_eq":0.4363},"2":{"raw":0.3934,"skill":0.2995,"coverage":1,"random":0.1341,"tracks":[],"combined":0.3618,"private_eq":0.4528},"3":{"raw":0.2264,"skill":0.2115,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2295,"private_eq":0.2517},"4":{"raw":0.6854,"skill":0.6814,"coverage":1,"random":0.0127,"tracks":[],"combined":0.3655,"private_eq":0.3797},"5":{"raw":0.4069,"skill":0.4033,"coverage":1,"random":0.006,"tracks":[],"combined":0.1475,"private_eq":0.1613},"6":{"raw":0.5802,"skill":0.0157,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.4942,"skill":0.1584,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5594,"skill":0.3629,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4194,"skill":0.1303,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.6572,"skill":0.3143,"coverage":1,"random":0.5,"tracks":[],"combined":0.3721,"private_eq":0.6891},"21":{"raw":0.5426,"skill":0.0852,"coverage":1,"random":0.5,"tracks":[],"combined":0.0957,"private_eq":0.5525},"22":{"raw":0.016,"skill":0.0083,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.4879,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4253,"skill":0.2337,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2716,"combined":0.0186},"26":{"raw":0.7252,"skill":0.6335,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5956,"skill":0.4607,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7174,"skill":0.4348,"coverage":1,"random":0.5,"tracks":[],"combined":0.4613,"private_eq":0.7332},"29":{"raw":0.7845,"skill":0.7127,"coverage":1,"random":0.25,"tracks":[],"combined":0.4583,"private_eq":0.5976},"30":{"raw":0.1895,"skill":0.0173,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.26,"random":0.25},{"track":"10-choice","raw":0.119,"random":0.1}],"combined":0.0776,"private_eq":0.2469},"31":{"raw":0.0736,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.5213,"skill":0.239,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0152,"skill":0.0021,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.254,"skill":0.1561,"coverage":1,"random":0.116,"tracks":[],"combined":0.1091,"private_eq":0.2205},"37":{"raw":0.2753,"skill":0.091,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0793,"skill":0.0498,"coverage":1,"random":0.031,"tracks":[],"combined":0.0513,"private_eq":0.0902},"39":{"raw":0.5631,"skill":0.3574,"coverage":1,"random":0.3201,"tracks":[],"combined":0.1839,"private_eq":0.4508},"40":{"raw":0.3839,"skill":0.2074,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0188,"headline":false},{"track":"A · English","score":0.3839,"headline":true},{"track":"C · Arabic pairs","score":0.455,"headline":false},{"track":"C · English pairs","score":0.55,"headline":false}],"combined":0.141,"private_eq":0.3391},"41":{"raw":0.4269,"skill":0.1404,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.592,"skill":0.2062,"coverage":1,"random":0.486,"tracks":[],"combined":0.0349,"private_eq":0.5091},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0771,"private_eq":0.4243},"44":{"raw":0.5324,"skill":0.0648,"coverage":1,"random":0.5,"tracks":[],"combined":0.2138,"private_eq":0.6109},"45":{"raw":0.1557,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4087,"skill":0.1418,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.428,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.1875,"private_eq":0.5979},"57":{"raw":0.2268,"skill":0.1304,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2939,"combined":0.1977},"58":{"raw":0.3777,"skill":0.098,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3332,"combined":0.0234},"59":{"raw":0.3812,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.7027,"skill":0.4054,"coverage":1,"random":0.5,"tracks":[],"combined":0.3089,"private_eq":0.6579},"62":{"raw":0.2842,"skill":0.0456,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.25},"64":{"raw":0.3542,"skill":0.1928,"coverage":1,"random":0.2,"tracks":[],"combined":0.167,"private_eq":0.3404}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #126).","Submitted public 0.3 run adopted (kit rescore 19.59, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.57% on 831 rows (bar 98.5%).","No content firewall against the 0.3 suite (author statement). Agreement 98.57%, just over the 98.5% bar."],"scores":{"balanced_raw":38.94,"balanced_skill":11,"breadth_skill":18.62,"public_skill":19.59},"frozen_scores":{"balanced_raw":38.94,"balanced_skill":19.59,"breadth_skill":18.62},"categories":[{"id":"knowledge","raw":0.3198,"skill":0.1009,"combined":0.1091,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4713,"skill":0.2485,"combined":0.1104,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4578,"skill":0.3008,"combined":0.1867,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.2851,"skill":0.1807,"combined":0.1655,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4111,"skill":0.1237,"combined":0.1684,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vera-spark","pub":19.59,"same_skills":15.13,"new_domains":2.81,"v":11,"lab":false,"jev":false,"rank_v":228,"rank_pub":215,"name":"vera-spark","eq":{"S":14.25,"O":6.12},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":8.55,"new_domains":2.45}}},{"engine":"nirnaya-0.4b","name":"Nirnaya 0.4B","long_name":"Nirnaya 0.4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":389956608,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/kshitijthakkar/nirnaya-0.4b-0.2a-decision","code_url":null,"base_model":"kshitijthakkar/loggenix-moe-0.4B-0.2A-sft-s4","base_url":"https://huggingface.co/kshitijthakkar/loggenix-moe-0.4B-0.2A-sft-s4","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.3075,"conf":0.3534,"ece":0.0459,"brier":0.6855,"over95":0,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.1585,"conf":0.0334,"acc":0.0224},{"lo":0.1,"w":0.0121,"conf":0.157,"acc":0.1358},{"lo":0.2,"w":0.1745,"conf":0.2744,"acc":0.2662},{"lo":0.3,"w":0.2461,"conf":0.3509,"acc":0.2755},{"lo":0.4,"w":0.0206,"conf":0.4189,"acc":0.2799},{"lo":0.5,"w":0.3823,"conf":0.5224,"acc":0.4719},{"lo":0.6,"w":0.0058,"conf":0.6172,"acc":0.3227},{"lo":0.7,"w":0,"conf":null,"acc":null},{"lo":0.8,"w":0,"conf":null,"acc":null},{"lo":0.9,"w":0,"conf":null,"acc":null}],"areas":{"knowledge":{"ece":0.0523,"acc":0.2878,"conf":0.3377},"language":{"ece":0.1192,"acc":0.3146,"conf":0.4338},"retrieval":{"ece":0.0816,"acc":0.21,"conf":0.1838},"tools":{"ece":0.0961,"acc":0.4088,"conf":0.3501},"arts":{"ece":0.0282,"acc":0.3516,"conf":0.3778}}},"gaps":null,"latency":{"median":14.4,"p95":35.6,"mean":15.1},"results":{"1":{"score":0.3684,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":61.7},"2":{"score":0.1232,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":77},"3":{"score":0.0157,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":46},"4":{"score":0.0003,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.6},"5":{"score":0.0001,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":37.4},"6":{"score":0.5691,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":67.9},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":96.7},"10":{"score":0.2796,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":33.8},"11":{"score":0.1352,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":163.3},"12":{"score":0.2381,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":4.8},"20":{"score":0.4936,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.1},"21":{"score":0.4962,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":4.6},"22":{"score":0.025,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":50.2},"23":{"score":0.4978,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33},"24":{"score":0.246,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":5},"25":{"score":0.2347,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":6.7},"26":{"score":0.2534,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":4.7},"27":{"score":0.25,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":4.7},"28":{"score":0.5083,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":4.4},"29":{"score":0.2496,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":32},"30":{"score":0.1729,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0792,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.8},"32":{"score":0.3497,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":35.9},"33":{"score":0,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":74.7},"36":{"score":0.1113,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":138.8},"37":{"score":0.2205,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.7},"38":{"score":0.0244,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":90.4},"39":{"score":0.1764,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":8.7},"40":{"score":0.2491,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":4.8},"41":{"score":0.2672,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":5.1},"42":{"score":0.4925,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32.4},"43":{"score":0.3667,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":5.9},"44":{"score":0.4962,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.1},"45":{"score":0.178,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":32.8},"50":{"score":0.3126,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":38.1},"56":{"score":0.535,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":44.3},"57":{"score":0.1168,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":30.2},"58":{"score":0.3183,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":5.8},"59":{"score":0.5166,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":8.3},"61":{"score":0.5015,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":34.1},"62":{"score":0.3382,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":34.5},"64":{"score":0.2083,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":5.5}},"benchmarks":{"1":{"raw":0.3684,"skill":0.1473,"coverage":1,"random":0.2592,"tracks":[],"combined":0.0542,"private_eq":0.3066},"2":{"raw":0.1231,"skill":0,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.0209,"private_eq":0.161},"3":{"raw":0.0157,"skill":0,"coverage":1,"random":0.0189,"tracks":[],"combined":0.0037,"private_eq":0.0327},"4":{"raw":0.0003,"skill":0,"coverage":1,"random":0.0127,"tracks":[],"combined":0,"private_eq":0.0127},"5":{"raw":0.0001,"skill":0,"coverage":1,"random":0.006,"tracks":[],"combined":0,"private_eq":0.006},"6":{"raw":0.5691,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.2796,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.1352,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2381,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.4936,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.054,"private_eq":0.5319},"21":{"raw":0.4962,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.1058,"private_eq":0.5575},"22":{"raw":0.0269,"skill":0.0193,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.4912,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.246,"skill":0,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2347,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2622,"combined":0.0059},"26":{"raw":0.2534,"skill":0.0043,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.25,"skill":0,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5083,"skill":0.0166,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"29":{"raw":0.2496,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0.2256,"private_eq":0.4251},"30":{"raw":0.1729,"skill":0,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2479,"random":0.25},{"track":"10-choice","raw":0.0978,"random":0.1}],"combined":0.0165,"private_eq":0.197},"31":{"raw":0.0792,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0854},"32":{"raw":0.3497,"skill":0,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.1113,"skill":0,"coverage":1,"random":0.116,"tracks":[],"combined":0.0047,"private_eq":0.1293},"37":{"raw":0.2205,"skill":0.0223,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0244,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.1764,"skill":0,"coverage":1,"random":0.3201,"tracks":[],"combined":0,"private_eq":0.3201},"40":{"raw":0.2491,"skill":0.0339,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.25,"headline":false},{"track":"A · English","score":0.2491,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.53,"headline":false}],"combined":0.0198,"private_eq":0.246},"41":{"raw":0.2672,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4925,"skill":0.0126,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.3667,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0018,"private_eq":0.3774},"44":{"raw":0.4962,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0532,"private_eq":0.5315},"45":{"raw":0.1776,"skill":0.0162,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.3126,"skill":0.0024,"coverage":1,"random":0.311,"tracks":[],"combined":0,"private_eq":0.311},"56":{"raw":0.535,"skill":0.07,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1168,"skill":0.0066,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1221,"combined":0.0022},"58":{"raw":0.3183,"skill":0.0119,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3103,"combined":0},"59":{"raw":0.5166,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5015,"skill":0.003,"coverage":1,"random":0.5,"tracks":[],"combined":0.0024,"private_eq":0.5064},"62":{"raw":0.3382,"skill":0.1176,"coverage":1,"random":0.25,"tracks":[],"combined":0.1387,"private_eq":0.3606},"64":{"raw":0.2083,"skill":0.0104,"coverage":1,"random":0.2,"tracks":[],"combined":0,"private_eq":0.2023}},"coverage":0.9999,"pending":0.0001,"notes":["New in the 2026-10-10 PR batch (PR #100).","Submitted public 0.3 run adopted (kit rescore 1.60, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.26% on 830 rows (bar 98.5%).","Served RoPE theta differs from the trained config (vLLM/transformers 4.x uses 10,000); the lab reproduced the submitted serving."],"scores":{"balanced_raw":24.11,"balanced_skill":3.51,"breadth_skill":1.48,"public_skill":1.6},"frozen_scores":{"balanced_raw":24.11,"balanced_skill":1.6,"breadth_skill":1.48},"categories":[{"id":"knowledge","raw":0.2576,"skill":0.0051,"combined":0.0027,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.2603,"skill":0.0049,"combined":0.0268,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2108,"skill":0.0144,"combined":0,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1708,"skill":0.0545,"combined":0.0405,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.3381,"skill":0.0053,"combined":0.0262,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"nirnaya-0.4b","pub":1.6,"same_skills":2.66,"new_domains":2.98,"v":3.51,"lab":false,"jev":false,"rank_v":242,"rank_pub":242,"name":"Nirnaya 0.4B","eq":{"S":1.64,"O":6.32},"rank_tie":59,"tied":true,"parts":{"public":0,"same_skills":0.98,"new_domains":2.53}}},{"engine":"saracura-ptbr-4b","name":"Saracura PT-BR 4B","long_name":"Saracura PT-BR 4B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/felhen-ai/saracura-ptbr-4b","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null,"org":{"id":"felhen-ai","name":"Felhen","logo":"data/orgs/felhen-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6249,"conf":0.8043,"ece":0.1794,"brier":0.5379,"over95":0.0798,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0208,"conf":0.0751,"acc":0.0682},{"lo":0.1,"w":0.0344,"conf":0.1464,"acc":0.0509},{"lo":0.2,"w":0.011,"conf":0.2461,"acc":0.0492},{"lo":0.3,"w":0.0227,"conf":0.3607,"acc":0.2105},{"lo":0.4,"w":0.0354,"conf":0.4556,"acc":0.3475},{"lo":0.5,"w":0.0717,"conf":0.5485,"acc":0.4837},{"lo":0.6,"w":0.0726,"conf":0.6475,"acc":0.4534},{"lo":0.7,"w":0.0831,"conf":0.7512,"acc":0.5174},{"lo":0.8,"w":0.1042,"conf":0.8524,"acc":0.5762},{"lo":0.9,"w":0.544,"conf":0.9798,"acc":0.7968}],"areas":{"knowledge":{"ece":0.1704,"acc":0.573,"conf":0.7433},"language":{"ece":0.1822,"acc":0.7149,"conf":0.8941},"retrieval":{"ece":0.1437,"acc":0.7027,"conf":0.8403},"tools":{"ece":0.2017,"acc":0.7672,"conf":0.9588},"arts":{"ece":0.2246,"acc":0.4194,"conf":0.6431}}},"gaps":null,"latency":{"median":61.9,"p95":242.1,"mean":98.9},"results":{"1":{"score":0.9085,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":86.3},"2":{"score":0.6369,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":701.6},"3":{"score":0.5256,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":652.1},"4":{"score":0.8158,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":82},"5":{"score":0.794,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":136.5},"6":{"score":0.7992,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":214.9},"9":{"score":0.1818,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":228.8},"10":{"score":0.549,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":110.2},"11":{"score":0.655,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":322.8},"12":{"score":0.4737,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":24.9},"20":{"score":0.5242,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.3},"21":{"score":0.5251,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":16.6},"22":{"score":0.0215,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":667.2},"23":{"score":0.588,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.8},"24":{"score":0.7261,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":17.9},"25":{"score":0.4031,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":25.9},"26":{"score":0.9731,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":16.3},"27":{"score":0.9198,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":17},"28":{"score":0.6906,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":15.7},"29":{"score":0.7888,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":26.6},"30":{"score":0.4401,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.102,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":95.3},"32":{"score":0.5652,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":88.1},"33":{"score":0.1836,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":99.3},"36":{"score":0.4103,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1616.8},"37":{"score":0.4221,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.9},"38":{"score":0.1271,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":544.1},"39":{"score":0.8659,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":39.4},"40":{"score":0.4181,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":28},"41":{"score":0.491,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":26},"42":{"score":0.7538,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":59.1},"43":{"score":0.486,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":31.6},"44":{"score":0.6226,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.3},"45":{"score":0.0958,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":27.5},"50":{"score":0.4266,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":89.4},"56":{"score":0.612,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":85.3},"57":{"score":0.516,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":36.4},"58":{"score":0.6101,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":31.4},"59":{"score":0.7157,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":86.7},"61":{"score":0.613,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":55.5},"62":{"score":0.609,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":86.8},"64":{"score":0.5663,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":33.1}},"benchmarks":{"1":{"raw":0.9085,"skill":0.8765,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8103,"private_eq":0.8603},"2":{"raw":0.6369,"skill":0.5806,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6652,"private_eq":0.7125},"3":{"raw":0.5256,"skill":0.5165,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2924,"private_eq":0.3127},"4":{"raw":0.8158,"skill":0.8134,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7065,"private_eq":0.7125},"5":{"raw":0.794,"skill":0.7928,"coverage":1,"random":0.006,"tracks":[],"combined":0.6616,"private_eq":0.6665},"6":{"raw":0.7992,"skill":0.5292,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1818,"skill":0.1818,"coverage":1,"random":0,"tracks":[],"combined":0.1867,"private_eq":0.195},"10":{"raw":0.549,"skill":0.2496,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.655,"skill":0.5011,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4737,"skill":0.2117,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4254,"private_eq":0.6201},"20":{"raw":0.5271,"skill":0.0543,"coverage":1,"random":0.5,"tracks":[],"combined":0.278,"private_eq":0.6426},"21":{"raw":0.5251,"skill":0.0502,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0211,"skill":0.0135,"coverage":1,"random":0.0078,"tracks":[],"combined":0.036,"private_eq":0.0534},"23":{"raw":0.5738,"skill":0.1476,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7261,"skill":0.6348,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4031,"skill":0.2041,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3381,"combined":0.1083},"26":{"raw":0.9731,"skill":0.9641,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9198,"skill":0.893,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6906,"skill":0.3812,"coverage":1,"random":0.5,"tracks":[],"combined":0.3504,"private_eq":0.6784},"29":{"raw":0.7888,"skill":0.7184,"coverage":1,"random":0.25,"tracks":[],"combined":0.7108,"private_eq":0.7848},"30":{"raw":0.4401,"skill":0.3266,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5383,"random":0.25},{"track":"10-choice","raw":0.3419,"random":0.1}],"combined":0.5351,"private_eq":0.62},"31":{"raw":0.102,"skill":0.0219,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0387,"private_eq":0.1266},"32":{"raw":0.5652,"skill":0.3087,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.1836,"skill":0.1728,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0351,"private_eq":0.0576},"36":{"raw":0.4103,"skill":0.3329,"coverage":1,"random":0.116,"tracks":[],"combined":0.3282,"private_eq":0.412},"37":{"raw":0.4221,"skill":0.2752,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3133,"private_eq":0.4579},"38":{"raw":0.1271,"skill":0.0992,"coverage":1,"random":0.031,"tracks":[],"combined":0.1888,"private_eq":0.2219},"39":{"raw":0.8659,"skill":0.8028,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4819,"private_eq":0.6511},"40":{"raw":0.4181,"skill":0.2514,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.378,"headline":false},{"track":"A · English","score":0.4181,"headline":true},{"track":"C · Arabic pairs","score":0.63,"headline":false},{"track":"C · English pairs","score":0.865,"headline":false}],"combined":0.1696,"private_eq":0.3611},"41":{"raw":0.491,"skill":0.2365,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3425,"private_eq":0.566},"42":{"raw":0.7538,"skill":0.521,"coverage":1,"random":0.486,"tracks":[],"combined":0.2845,"private_eq":0.6359},"43":{"raw":0.486,"skill":0.1844,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3405,"private_eq":0.5884},"44":{"raw":0.6226,"skill":0.2452,"coverage":1,"random":0.5,"tracks":[],"combined":0.3211,"private_eq":0.6639},"45":{"raw":0.0958,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4266,"skill":0.1678,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.612,"skill":0.224,"coverage":1,"random":0.5,"tracks":[],"combined":0.1617,"private_eq":0.5851},"57":{"raw":0.516,"skill":0.4556,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2617,"combined":0.1611},"58":{"raw":0.6101,"skill":0.4348,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3977,"combined":0.1179},"59":{"raw":0.7157,"skill":0.4105,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1722,"private_eq":0.6048},"61":{"raw":0.613,"skill":0.226,"coverage":1,"random":0.5,"tracks":[],"combined":0.3031,"private_eq":0.655},"62":{"raw":0.609,"skill":0.4787,"coverage":1,"random":0.25,"tracks":[],"combined":0.4741,"private_eq":0.6093},"64":{"raw":0.5663,"skill":0.4579,"coverage":1,"random":0.2,"tracks":[],"combined":0.4191,"private_eq":0.5398}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #75).","Submitted public 0.3 run adopted (kit rescore 37.96, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.71% on 895 rows (bar 98.5%).","Initialised from Kev 4B r10; public index equals the starting checkpoint's (37.95) at 88% decision agreement.","0.2.1 run resumed under 0.3."],"scores":{"balanced_raw":53.8,"balanced_skill":34.43,"breadth_skill":36.11,"public_skill":37.96},"frozen_scores":{"balanced_raw":53.8,"balanced_skill":37.96,"breadth_skill":36.11},"categories":[{"id":"knowledge","raw":0.4315,"skill":0.2525,"combined":0.1999,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5895,"skill":0.419,"combined":0.3575,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6169,"skill":0.4624,"combined":0.4263,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5831,"skill":0.5394,"combined":0.4906,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.44,"skill":0.1485,"combined":0.1808,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"saracura-ptbr-4b","pub":37.96,"same_skills":34.97,"new_domains":25.87,"v":34.43,"lab":false,"jev":false,"rank_v":152,"rank_pub":164,"name":"Saracura PT-BR 4B","eq":{"S":34.32,"O":34.6},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":20.59,"new_domains":13.84}}},{"engine":"aplomb-1","name":"Aplomb 1","long_name":"Aplomb 1","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":5295891968,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/empiriolabsai/aplomb-1","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"empiriolabsai","name":"EmpirioLabs AI","logo":"data/orgs/empiriolabsai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6646,"conf":0.7221,"ece":0.0754,"brier":0.4504,"over95":0.0182,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0626,"conf":0.0413,"acc":0.1015},{"lo":0.1,"w":0.0197,"conf":0.1422,"acc":0.3131},{"lo":0.2,"w":0.015,"conf":0.2516,"acc":0.3676},{"lo":0.3,"w":0.0387,"conf":0.3552,"acc":0.3571},{"lo":0.4,"w":0.0579,"conf":0.451,"acc":0.401},{"lo":0.5,"w":0.095,"conf":0.5502,"acc":0.5195},{"lo":0.6,"w":0.0917,"conf":0.6492,"acc":0.5703},{"lo":0.7,"w":0.106,"conf":0.7494,"acc":0.6248},{"lo":0.8,"w":0.1347,"conf":0.8523,"acc":0.7239},{"lo":0.9,"w":0.3787,"conf":0.9693,"acc":0.9087}],"areas":{"knowledge":{"ece":0.094,"acc":0.6064,"conf":0.7004},"language":{"ece":0.0498,"acc":0.7634,"conf":0.81},"retrieval":{"ece":0.1286,"acc":0.6348,"conf":0.6304},"tools":{"ece":0.0768,"acc":0.8247,"conf":0.8917},"arts":{"ece":0.0863,"acc":0.531,"conf":0.5791}}},"gaps":null,"latency":{"median":53.9,"p95":581.5,"mean":140.7},"results":{"1":{"score":0.9238,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":110},"2":{"score":0.6501,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":841.2},"3":{"score":0.6043,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":950.7},"4":{"score":0.7103,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":79.5},"5":{"score":0.5463,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":124.7},"6":{"score":0.7989,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":521.5},"9":{"score":0.2386,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2563.4},"10":{"score":0.6723,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":144.8},"11":{"score":0.7567,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3381.4},"12":{"score":0.5874,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":40.8},"20":{"score":0.8696,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.8},"21":{"score":0.621,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":40.2},"22":{"score":0.0825,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":812.5},"23":{"score":0.6562,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.7},"24":{"score":0.7603,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":41.1},"25":{"score":0.4796,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":41.5},"26":{"score":0.9773,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":40.3},"27":{"score":0.936,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":40.1},"28":{"score":0.7822,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":40.2},"29":{"score":0.8437,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":42.5},"30":{"score":0.3965,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.124,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.1},"32":{"score":0.5372,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":92.2},"33":{"score":0.3036,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":213.1},"36":{"score":0.4301,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1617.7},"37":{"score":0.4341,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.4},"38":{"score":0.1382,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":563.1},"39":{"score":0.8188,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":54.2},"40":{"score":0.4393,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":41.6},"41":{"score":0.4895,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":41.3},"42":{"score":0.7941,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":65.7},"43":{"score":0.4947,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":41.6},"44":{"score":0.6416,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41},"45":{"score":0.1118,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":46.5},"50":{"score":0.4212,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":84.5},"56":{"score":0.707,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":173.3},"57":{"score":0.5475,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":42.7},"58":{"score":0.6483,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":41.5},"59":{"score":0.7234,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":52.1},"61":{"score":0.653,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":57},"62":{"score":0.7735,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":100},"64":{"score":0.6117,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":41.4}},"benchmarks":{"1":{"raw":0.9238,"skill":0.8972,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7626,"private_eq":0.8254},"2":{"raw":0.6501,"skill":0.5959,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6771,"private_eq":0.7227},"3":{"raw":0.6043,"skill":0.5967,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4605,"private_eq":0.4757},"4":{"raw":0.7103,"skill":0.7066,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6944,"private_eq":0.7007},"5":{"raw":0.5463,"skill":0.5436,"coverage":1,"random":0.006,"tracks":[],"combined":0.5151,"private_eq":0.5225},"6":{"raw":0.7989,"skill":0.5285,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2386,"skill":0.2386,"coverage":1,"random":0,"tracks":[],"combined":0.2601,"private_eq":0.2675},"10":{"raw":0.6723,"skill":0.4547,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7567,"skill":0.6481,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5874,"skill":0.382,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4053,"private_eq":0.6068},"20":{"raw":0.8721,"skill":0.7442,"coverage":1,"random":0.5,"tracks":[],"combined":0.7552,"private_eq":0.8785},"21":{"raw":0.621,"skill":0.242,"coverage":1,"random":0.5,"tracks":[],"combined":0.317,"private_eq":0.6619},"22":{"raw":0.0875,"skill":0.0803,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0773,"private_eq":0.0939},"23":{"raw":0.6492,"skill":0.2984,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7603,"skill":0.6804,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3901,"combined":0.1784},"26":{"raw":0.9773,"skill":0.9697,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.936,"skill":0.9146,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7822,"skill":0.5644,"coverage":1,"random":0.5,"tracks":[],"combined":0.4261,"private_eq":0.7158},"29":{"raw":0.8437,"skill":0.7916,"coverage":1,"random":0.25,"tracks":[],"combined":0.7059,"private_eq":0.7812},"30":{"raw":0.3965,"skill":0.2738,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4989,"random":0.25},{"track":"10-choice","raw":0.2942,"random":0.1}],"combined":0.4846,"private_eq":0.5788},"31":{"raw":0.124,"skill":0.0459,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0778,"private_eq":0.1621},"32":{"raw":0.5372,"skill":0.2642,"coverage":1,"random":0.371,"tracks":[],"combined":0.0726,"private_eq":0.4227},"33":{"raw":0.3036,"skill":0.2944,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3948,"private_eq":0.4085},"36":{"raw":0.4301,"skill":0.3553,"coverage":1,"random":0.116,"tracks":[],"combined":0.2988,"private_eq":0.3863},"37":{"raw":0.4341,"skill":0.2902,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4179,"private_eq":0.5404},"38":{"raw":0.1382,"skill":0.1106,"coverage":1,"random":0.031,"tracks":[],"combined":0.1923,"private_eq":0.2253},"39":{"raw":0.8188,"skill":0.7335,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5709,"private_eq":0.7109},"40":{"raw":0.4393,"skill":0.2787,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0905,"headline":false},{"track":"A · English","score":0.4393,"headline":true},{"track":"C · Arabic pairs","score":0.665,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}],"combined":0.2072,"private_eq":0.39},"41":{"raw":0.4895,"skill":0.2342,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5022,"private_eq":0.6712},"42":{"raw":0.7941,"skill":0.5994,"coverage":1,"random":0.486,"tracks":[],"combined":0.7637,"private_eq":0.8794},"43":{"raw":0.4947,"skill":0.1984,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4241,"private_eq":0.6405},"44":{"raw":0.6416,"skill":0.2832,"coverage":1,"random":0.5,"tracks":[],"combined":0.3907,"private_eq":0.6983},"45":{"raw":0.1118,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4212,"skill":0.16,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.707,"skill":0.414,"coverage":1,"random":0.5,"tracks":[],"combined":0.1962,"private_eq":0.6022},"57":{"raw":0.5475,"skill":0.4911,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5703,"combined":0.5122},"58":{"raw":0.6483,"skill":0.4902,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4242,"combined":0.1568},"59":{"raw":0.7234,"skill":0.4265,"coverage":1,"random":0.5177,"tracks":[],"combined":0.0799,"private_eq":0.5608},"61":{"raw":0.653,"skill":0.306,"coverage":1,"random":0.5,"tracks":[],"combined":0.4109,"private_eq":0.7083},"62":{"raw":0.7735,"skill":0.698,"coverage":1,"random":0.25,"tracks":[],"combined":0.4672,"private_eq":0.6042},"64":{"raw":0.6117,"skill":0.5146,"coverage":1,"random":0.2,"tracks":[],"combined":0.5355,"private_eq":0.6318}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #72).","Submitted public 0.3 run adopted (kit rescore 43.49, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.60% on 831 rows (bar 98.5%).","Gated repo, EmpirioLabs Model License; training includes WinoGrande and ContractNLI train splits (author-declared)."],"scores":{"balanced_raw":57.04,"balanced_skill":40.75,"breadth_skill":42.39,"public_skill":43.49},"frozen_scores":{"balanced_raw":57.04,"balanced_skill":43.49,"breadth_skill":42.39},"categories":[{"id":"knowledge","raw":0.4651,"skill":0.2985,"combined":0.3122,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6252,"skill":0.4723,"combined":0.4344,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5785,"skill":0.445,"combined":0.4295,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6474,"skill":0.6158,"combined":0.5319,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5438,"skill":0.3399,"combined":0.382,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"aplomb-1","pub":43.49,"same_skills":42.29,"new_domains":29.67,"v":40.75,"lab":false,"jev":false,"rank_v":131,"rank_pub":139,"name":"Aplomb 1","eq":{"S":41.73,"O":39.29},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.04,"new_domains":15.72}}},{"engine":"rsi-jev-v6.1-vl-4b","name":"RSI-Jev v6.1 VL 4B","long_name":"RSI-Jev v6.1 VL 4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/shgao/rsi-jev-v6.1-vl-4b","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7248,"conf":0.7673,"ece":0.049,"brier":0.3776,"over95":0.0216,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0148,"conf":0.0795,"acc":0.1277},{"lo":0.1,"w":0.0213,"conf":0.1424,"acc":0.2284},{"lo":0.2,"w":0.0204,"conf":0.2661,"acc":0.2997},{"lo":0.3,"w":0.0396,"conf":0.355,"acc":0.3306},{"lo":0.4,"w":0.0545,"conf":0.4557,"acc":0.3849},{"lo":0.5,"w":0.0972,"conf":0.5505,"acc":0.5443},{"lo":0.6,"w":0.094,"conf":0.6502,"acc":0.6286},{"lo":0.7,"w":0.0939,"conf":0.7514,"acc":0.7123},{"lo":0.8,"w":0.1298,"conf":0.8544,"acc":0.7721},{"lo":0.9,"w":0.4345,"conf":0.9728,"acc":0.9177}],"areas":{"knowledge":{"ece":0.0652,"acc":0.6083,"conf":0.6627},"language":{"ece":0.0393,"acc":0.8321,"conf":0.8585},"retrieval":{"ece":0.0643,"acc":0.7485,"conf":0.8074},"tools":{"ece":0.0864,"acc":0.8603,"conf":0.9425},"arts":{"ece":0.06,"acc":0.6643,"conf":0.6754}}},"gaps":null,"latency":{"median":57.9,"p95":157.7,"mean":77.9},"results":{"1":{"score":0.925,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":43.1},"2":{"score":0.6358,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":426.6},"3":{"score":0.7106,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":176.7},"4":{"score":0.8029,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":43.3},"5":{"score":0.8657,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":65.5},"6":{"score":0.7902,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":105.8},"9":{"score":0.4432,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":138.7},"10":{"score":0.6686,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":43.7},"11":{"score":0.7652,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":141.7},"12":{"score":0.6783,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":27.4},"20":{"score":0.9064,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.4},"21":{"score":0.6221,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":26.9},"22":{"score":0.562,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":204},"23":{"score":0.6344,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.5},"24":{"score":0.713,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":27.3},"25":{"score":0.4082,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":38.5},"26":{"score":0.9663,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":27.1},"27":{"score":0.9138,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.4},"28":{"score":0.7885,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":26.9},"29":{"score":0.9096,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":27.4},"30":{"score":0.3923,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1972,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.9},"32":{"score":0.5904,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":42.6},"33":{"score":0.323,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":65.6},"36":{"score":0.4176,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":490.8},"37":{"score":0.5297,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28},"38":{"score":0.1763,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":268},"39":{"score":0.8748,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":30.6},"40":{"score":0.398,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":27},"41":{"score":0.7651,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":27.4},"42":{"score":0.7832,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":28.8},"43":{"score":0.5298,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":37.9},"44":{"score":0.7022,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.3},"45":{"score":0.1098,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":38.3},"50":{"score":0.5382,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":41.7},"56":{"score":0.7405,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":65.9},"57":{"score":0.4618,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":27.6},"58":{"score":0.6908,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":27.3},"59":{"score":0.7319,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":40.9},"61":{"score":0.734,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":28.1},"62":{"score":0.8108,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":38.6},"64":{"score":0.6477,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":38.2}},"benchmarks":{"1":{"raw":0.925,"skill":0.8988,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8852,"private_eq":0.9152},"2":{"raw":0.6358,"skill":0.5794,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6703,"private_eq":0.7169},"3":{"raw":0.7106,"skill":0.705,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7648,"private_eq":0.7709},"4":{"raw":0.8029,"skill":0.8004,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8011,"private_eq":0.8049},"5":{"raw":0.8657,"skill":0.8649,"coverage":1,"random":0.006,"tracks":[],"combined":0.6701,"private_eq":0.6748},"6":{"raw":0.7902,"skill":0.5081,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4432,"skill":0.4432,"coverage":1,"random":0,"tracks":[],"combined":0.3789,"private_eq":0.385},"10":{"raw":0.6686,"skill":0.4486,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7652,"skill":0.6605,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6783,"skill":0.5181,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4559,"private_eq":0.6402},"20":{"raw":0.9037,"skill":0.8074,"coverage":1,"random":0.5,"tracks":[],"combined":0.8566,"private_eq":0.9286},"21":{"raw":0.6221,"skill":0.2443,"coverage":1,"random":0.5,"tracks":[],"combined":0.3472,"private_eq":0.6768},"22":{"raw":0.5475,"skill":0.544,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5243,"private_eq":0.5324},"23":{"raw":0.6287,"skill":0.2574,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.713,"skill":0.6173,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4082,"skill":0.2109,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3371,"combined":0.107},"26":{"raw":0.9663,"skill":0.9551,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9138,"skill":0.885,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7885,"skill":0.577,"coverage":1,"random":0.5,"tracks":[],"combined":0.5017,"private_eq":0.7532},"29":{"raw":0.9096,"skill":0.8795,"coverage":1,"random":0.25,"tracks":[],"combined":0.8643,"private_eq":0.8986},"30":{"raw":0.3923,"skill":0.2685,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4928,"random":0.25},{"track":"10-choice","raw":0.2919,"random":0.1}],"combined":0.4407,"private_eq":0.543},"31":{"raw":0.1972,"skill":0.1256,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0939,"private_eq":0.1767},"32":{"raw":0.5904,"skill":0.3488,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.323,"skill":0.314,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2509,"private_eq":0.2681},"36":{"raw":0.4176,"skill":0.3411,"coverage":1,"random":0.116,"tracks":[],"combined":0.3451,"private_eq":0.4268},"37":{"raw":0.5297,"skill":0.4101,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5151,"private_eq":0.617},"38":{"raw":0.1763,"skill":0.1499,"coverage":1,"random":0.031,"tracks":[],"combined":0.1892,"private_eq":0.2223},"39":{"raw":0.8748,"skill":0.8159,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6368,"private_eq":0.7552},"40":{"raw":0.398,"skill":0.2256,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4118,"headline":false},{"track":"A · English","score":0.398,"headline":true},{"track":"C · Arabic pairs","score":0.745,"headline":false},{"track":"C · English pairs","score":0.925,"headline":false}],"combined":0.2844,"private_eq":0.4493},"41":{"raw":0.7651,"skill":0.6477,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2794,"private_eq":0.5244},"42":{"raw":0.7832,"skill":0.5782,"coverage":1,"random":0.486,"tracks":[],"combined":0.6039,"private_eq":0.7982},"43":{"raw":0.5298,"skill":0.254,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4074,"private_eq":0.6301},"44":{"raw":0.7022,"skill":0.4044,"coverage":1,"random":0.5,"tracks":[],"combined":0.4336,"private_eq":0.7195},"45":{"raw":0.1098,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5382,"skill":0.3298,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.7405,"skill":0.481,"coverage":1,"random":0.5,"tracks":[],"combined":0.4558,"private_eq":0.7305},"57":{"raw":0.4618,"skill":0.3947,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4796,"combined":0.409},"58":{"raw":0.6908,"skill":0.5518,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4199,"combined":0.1505},"59":{"raw":0.7319,"skill":0.4441,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1545,"private_eq":0.5964},"61":{"raw":0.734,"skill":0.468,"coverage":1,"random":0.5,"tracks":[],"combined":0.4621,"private_eq":0.7336},"62":{"raw":0.8108,"skill":0.7477,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.6477,"skill":0.5596,"coverage":1,"random":0.2,"tracks":[],"combined":0.5355,"private_eq":0.6318}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #87).","Submitted public 0.3 run adopted (kit rescore 50.98, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.71% on 895 rows (bar 98.5%).","Trained on Decision Index training pools (train splits of board benchmarks), test-row overlaps dropped per card.","Supersedes RSI-Jev v6.0 VL 4B (PR #84), which is not listed."],"scores":{"balanced_raw":63.04,"balanced_skill":44.27,"breadth_skill":49.51,"public_skill":50.98},"frozen_scores":{"balanced_raw":63.04,"balanced_skill":50.98,"breadth_skill":49.51},"categories":[{"id":"knowledge","raw":0.4738,"skill":0.3164,"combined":0.2806,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6808,"skill":0.5531,"combined":0.4443,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.683,"skill":0.5707,"combined":0.5473,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7134,"skill":0.6842,"combined":0.6716,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.648,"skill":0.4571,"combined":0.4962,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"rsi-jev-v6.1-vl-4b","pub":50.98,"same_skills":47.98,"new_domains":29.81,"v":44.27,"lab":false,"jev":false,"rank_v":115,"rank_pub":114,"name":"RSI-Jev v6.1 VL 4B","eq":{"S":47.49,"O":39.46},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":28.49,"new_domains":15.78}}},{"engine":"sieve-4b","name":"Sieve 4B","long_name":"Sieve 4B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/Sieve-4B","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.684,"conf":0.7152,"ece":0.0312,"brier":0.4092,"over95":0.0054,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0313,"conf":0.0784,"acc":0.0528},{"lo":0.1,"w":0.0277,"conf":0.1394,"acc":0.1226},{"lo":0.2,"w":0.0174,"conf":0.2553,"acc":0.1615},{"lo":0.3,"w":0.0486,"conf":0.3559,"acc":0.3414},{"lo":0.4,"w":0.0631,"conf":0.4549,"acc":0.4077},{"lo":0.5,"w":0.111,"conf":0.5496,"acc":0.541},{"lo":0.6,"w":0.1116,"conf":0.6504,"acc":0.608},{"lo":0.7,"w":0.1095,"conf":0.7495,"acc":0.6825},{"lo":0.8,"w":0.1549,"conf":0.8541,"acc":0.7935},{"lo":0.9,"w":0.3248,"conf":0.9557,"acc":0.949}],"areas":{"knowledge":{"ece":0.0673,"acc":0.6031,"conf":0.6655},"language":{"ece":0.0272,"acc":0.7979,"conf":0.7863},"retrieval":{"ece":0.0835,"acc":0.7044,"conf":0.7658},"tools":{"ece":0.0799,"acc":0.8899,"conf":0.8165},"arts":{"ece":0.0926,"acc":0.5013,"conf":0.5855}}},"gaps":null,"latency":{"median":93.4,"p95":156.8,"mean":130.5},"results":{"1":{"score":0.9439,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":347.6},"2":{"score":0.6731,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":5071.8},"3":{"score":0.7382,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1946.5},"4":{"score":0.8281,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":285.8},"5":{"score":0.902,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":425.7},"6":{"score":0.7901,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":707.6},"9":{"score":0.2614,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1000.5},"10":{"score":0.3824,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":389.1},"11":{"score":0.7984,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":999},"12":{"score":0.6346,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":164.9},"20":{"score":0.804,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":164.4},"21":{"score":0.5947,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":162.6},"22":{"score":0.0785,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1435.2},"23":{"score":0.6132,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":181},"24":{"score":0.7575,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":160.1},"25":{"score":0.4235,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":162.6},"26":{"score":0.9785,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":162.5},"27":{"score":0.9437,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":162.1},"28":{"score":0.8579,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":163},"29":{"score":0.9504,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":171.2},"30":{"score":0.6452,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1382,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":357.9},"32":{"score":0.5505,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":219.4},"33":{"score":0.2061,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":398.8},"36":{"score":0.4443,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":6619.2},"37":{"score":0.4912,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":345.3},"38":{"score":0.2064,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1604.8},"39":{"score":0.886,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":349.3},"40":{"score":0.4609,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":332.8},"41":{"score":0.5242,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":174.9},"42":{"score":0.7681,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":351.4},"43":{"score":0.5579,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":340.8},"44":{"score":0.6388,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":184.6},"45":{"score":0.0938,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":185.6},"50":{"score":0.3998,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":344.6},"56":{"score":0.8425,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":339.6},"57":{"score":0.5497,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":296.9},"58":{"score":0.6817,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":286},"59":{"score":0.7788,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":326.1},"61":{"score":0.75,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":285.5},"62":{"score":0.7987,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":326},"64":{"score":0.6155,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":274.6}},"benchmarks":{"1":{"raw":0.9439,"skill":0.9243,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8306,"private_eq":0.8752},"2":{"raw":0.6731,"skill":0.6225,"coverage":1,"random":0.1341,"tracks":[],"combined":0.686,"private_eq":0.7303},"3":{"raw":0.7382,"skill":0.7332,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4605,"private_eq":0.4757},"4":{"raw":0.8281,"skill":0.8259,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8225,"private_eq":0.8258},"5":{"raw":0.902,"skill":0.9014,"coverage":1,"random":0.006,"tracks":[],"combined":0.6497,"private_eq":0.6548},"6":{"raw":0.7901,"skill":0.5079,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2614,"skill":0.2614,"coverage":1,"random":0,"tracks":[],"combined":0.2525,"private_eq":0.26},"10":{"raw":0.3824,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7984,"skill":0.7084,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6346,"skill":0.4527,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5009,"private_eq":0.6699},"20":{"raw":0.8051,"skill":0.6103,"coverage":1,"random":0.5,"tracks":[],"combined":0.7046,"private_eq":0.8535},"21":{"raw":0.5947,"skill":0.1895,"coverage":1,"random":0.5,"tracks":[],"combined":0.1157,"private_eq":0.5624},"22":{"raw":0.0795,"skill":0.0723,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1687,"private_eq":0.1836},"23":{"raw":0.5889,"skill":0.1777,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7575,"skill":0.6767,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4235,"skill":0.2313,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3505,"combined":0.125},"26":{"raw":0.9785,"skill":0.9713,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9437,"skill":0.9249,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8579,"skill":0.7158,"coverage":1,"random":0.5,"tracks":[],"combined":0.4916,"private_eq":0.7482},"29":{"raw":0.9504,"skill":0.9339,"coverage":1,"random":0.25,"tracks":[],"combined":0.9879,"private_eq":0.9903},"30":{"raw":0.6452,"skill":0.5738,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7119,"random":0.25},{"track":"10-choice","raw":0.5785,"random":0.1}],"combined":0.6301,"private_eq":0.6975},"31":{"raw":0.1382,"skill":0.0614,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0983,"private_eq":0.1807},"32":{"raw":0.5505,"skill":0.2854,"coverage":1,"random":0.371,"tracks":[],"combined":0.0881,"private_eq":0.4323},"33":{"raw":0.2061,"skill":0.1956,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1173,"private_eq":0.1378},"36":{"raw":0.4443,"skill":0.3713,"coverage":1,"random":0.116,"tracks":[],"combined":0.343,"private_eq":0.4249},"37":{"raw":0.4912,"skill":0.3618,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5506,"private_eq":0.645},"38":{"raw":0.2064,"skill":0.181,"coverage":1,"random":0.031,"tracks":[],"combined":0.2868,"private_eq":0.3158},"39":{"raw":0.886,"skill":0.8323,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3992,"private_eq":0.5955},"40":{"raw":0.4609,"skill":0.3064,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4145,"headline":false},{"track":"A · English","score":0.4609,"headline":true},{"track":"C · Arabic pairs","score":0.925,"headline":false},{"track":"C · English pairs","score":0.915,"headline":false}],"combined":0.3338,"private_eq":0.4873},"41":{"raw":0.5242,"skill":0.2864,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0349,"private_eq":0.3632},"42":{"raw":0.7681,"skill":0.5488,"coverage":1,"random":0.486,"tracks":[],"combined":0.4028,"private_eq":0.696},"43":{"raw":0.5579,"skill":0.2986,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4324,"private_eq":0.6457},"44":{"raw":0.6388,"skill":0.2776,"coverage":1,"random":0.5,"tracks":[],"combined":0.3852,"private_eq":0.6956},"45":{"raw":0.0938,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3998,"skill":0.1289,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.8425,"skill":0.685,"coverage":1,"random":0.5,"tracks":[],"combined":0.7241,"private_eq":0.8631},"57":{"raw":0.5497,"skill":0.4935,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3555,"combined":0.2678},"58":{"raw":0.6817,"skill":0.5386,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3798,"combined":0.0917},"59":{"raw":0.7788,"skill":0.5414,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3129,"private_eq":0.6719},"61":{"raw":0.75,"skill":0.5,"coverage":1,"random":0.5,"tracks":[],"combined":0.5414,"private_eq":0.7728},"62":{"raw":0.7987,"skill":0.7316,"coverage":1,"random":0.25,"tracks":[],"combined":0.5903,"private_eq":0.6955},"64":{"raw":0.6155,"skill":0.5194,"coverage":1,"random":0.2,"tracks":[],"combined":0.5064,"private_eq":0.6088}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #89).","Submitted public 0.3 run adopted (kit rescore 50.06, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.28% on 895 rows (bar 98.5%).","Training data not published; no overlap disclosure."],"scores":{"balanced_raw":62.1,"balanced_skill":40.96,"breadth_skill":48.32,"public_skill":50.06},"frozen_scores":{"balanced_raw":62.1,"balanced_skill":50.06,"breadth_skill":48.32},"categories":[{"id":"knowledge","raw":0.4887,"skill":0.339,"combined":0.2706,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6729,"skill":0.5392,"combined":0.4234,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7111,"skill":0.6159,"combined":0.6052,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6948,"skill":0.6675,"combined":0.57,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5139,"skill":0.283,"combined":0.347,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-4b","pub":50.06,"same_skills":45.16,"new_domains":26.57,"v":40.96,"lab":false,"jev":false,"rank_v":128,"rank_pub":119,"name":"Sieve 4B","eq":{"S":44.63,"O":35.46},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":26.78,"new_domains":14.19}}},{"engine":"jiwo-4b-v2","name":"jiwo 4B v2","long_name":"jiwo 4B v2","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4205751296,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/eljiwo/jiwo-4b","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6675,"conf":0.735,"ece":0.0675,"brier":0.4284,"over95":0.0106,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0159,"conf":0.0732,"acc":0.0312},{"lo":0.1,"w":0.0318,"conf":0.1448,"acc":0.1111},{"lo":0.2,"w":0.0192,"conf":0.249,"acc":0.1632},{"lo":0.3,"w":0.033,"conf":0.3577,"acc":0.2536},{"lo":0.4,"w":0.0561,"conf":0.4551,"acc":0.3884},{"lo":0.5,"w":0.1245,"conf":0.5493,"acc":0.491},{"lo":0.6,"w":0.1185,"conf":0.65,"acc":0.529},{"lo":0.7,"w":0.1204,"conf":0.7516,"acc":0.6555},{"lo":0.8,"w":0.1263,"conf":0.8492,"acc":0.7659},{"lo":0.9,"w":0.3542,"conf":0.9707,"acc":0.9334}],"areas":{"knowledge":{"ece":0.0955,"acc":0.5941,"conf":0.6896},"language":{"ece":0.0377,"acc":0.7723,"conf":0.81},"retrieval":{"ece":0.1267,"acc":0.6468,"conf":0.7724},"tools":{"ece":0.0567,"acc":0.8748,"conf":0.9315},"arts":{"ece":0.0334,"acc":0.5176,"conf":0.5428}}},"gaps":null,"latency":{"median":49.8,"p95":257.6,"mean":128.2},"results":{"1":{"score":0.9262,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":117.5},"2":{"score":0.6555,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":710.2},"3":{"score":0.7343,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":268},"4":{"score":0.7131,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":50.8},"5":{"score":0.8944,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":60.3},"6":{"score":0.7971,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":159.1},"9":{"score":0.3068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":655.3},"10":{"score":0.2988,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":56.4},"11":{"score":0.829,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1095.6},"12":{"score":0.6136,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":48},"20":{"score":0.7764,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.5},"21":{"score":0.6039,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":47.1},"22":{"score":0.081,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":225.9},"23":{"score":0.6398,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.6},"24":{"score":0.7502,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":47.6},"25":{"score":0.4082,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":48.2},"26":{"score":0.9752,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":49.3},"27":{"score":0.9326,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":48.6},"28":{"score":0.7182,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":48.3},"29":{"score":0.9365,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":49.2},"30":{"score":0.4473,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1304,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.3},"32":{"score":0.5585,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":52.3},"33":{"score":0.3176,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":173.9},"36":{"score":0.4556,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1231.9},"37":{"score":0.5521,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":49.1},"38":{"score":0.1207,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":386},"39":{"score":0.844,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":48.7},"40":{"score":0.4025,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":46.8},"41":{"score":0.5943,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":46.8},"42":{"score":0.7835,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":47.8},"43":{"score":0.4842,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":45.1},"44":{"score":0.6618,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.5},"45":{"score":0.1198,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":46.5},"50":{"score":0.4469,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":48.7},"56":{"score":0.6895,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":96},"57":{"score":0.5441,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":45.8},"58":{"score":0.6721,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":45.3},"59":{"score":0.7817,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":48.7},"61":{"score":0.6065,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":46.9},"62":{"score":0.6588,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":47.9},"64":{"score":0.5909,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":45.9}},"benchmarks":{"1":{"raw":0.9262,"skill":0.9004,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8376,"private_eq":0.8803},"2":{"raw":0.6555,"skill":0.6022,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6918,"private_eq":0.7353},"3":{"raw":0.7343,"skill":0.7292,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7806,"private_eq":0.7862},"4":{"raw":0.7131,"skill":0.7094,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7659,"private_eq":0.7705},"5":{"raw":0.8944,"skill":0.8938,"coverage":1,"random":0.006,"tracks":[],"combined":0.6744,"private_eq":0.679},"6":{"raw":0.7971,"skill":0.5243,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3068,"skill":0.3068,"coverage":1,"random":0,"tracks":[],"combined":0.3435,"private_eq":0.35},"10":{"raw":0.2988,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.829,"skill":0.7526,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6136,"skill":0.4212,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4954,"private_eq":0.6663},"20":{"raw":0.779,"skill":0.5579,"coverage":1,"random":0.5,"tracks":[],"combined":0.6613,"private_eq":0.8321},"21":{"raw":0.6039,"skill":0.2078,"coverage":1,"random":0.5,"tracks":[],"combined":0.1661,"private_eq":0.5873},"22":{"raw":0.0848,"skill":0.0777,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0805,"private_eq":0.0971},"23":{"raw":0.629,"skill":0.258,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7502,"skill":0.6669,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4082,"skill":0.2109,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3651,"combined":0.1447},"26":{"raw":0.9752,"skill":0.9669,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9326,"skill":0.9101,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7182,"skill":0.4364,"coverage":1,"random":0.5,"tracks":[],"combined":0.3704,"private_eq":0.6883},"29":{"raw":0.9365,"skill":0.9153,"coverage":1,"random":0.25,"tracks":[],"combined":0.9534,"private_eq":0.9647},"30":{"raw":0.4473,"skill":0.3327,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5208,"random":0.25},{"track":"10-choice","raw":0.3738,"random":0.1}],"combined":0.5655,"private_eq":0.6448},"31":{"raw":0.1304,"skill":0.0529,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0842,"private_eq":0.1679},"32":{"raw":0.5585,"skill":0.2981,"coverage":1,"random":0.371,"tracks":[],"combined":0.0339,"private_eq":0.3986},"33":{"raw":0.3176,"skill":0.3085,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2457,"private_eq":0.2631},"36":{"raw":0.4556,"skill":0.3841,"coverage":1,"random":0.116,"tracks":[],"combined":0.3423,"private_eq":0.4243},"37":{"raw":0.5521,"skill":0.4382,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5174,"private_eq":0.6188},"38":{"raw":0.1207,"skill":0.0926,"coverage":1,"random":0.031,"tracks":[],"combined":0.204,"private_eq":0.2365},"39":{"raw":0.844,"skill":0.7706,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5592,"private_eq":0.703},"40":{"raw":0.4025,"skill":0.2313,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.295,"headline":false},{"track":"A · English","score":0.4025,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"combined":0.2046,"private_eq":0.388},"41":{"raw":0.5943,"skill":0.3914,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3774,"private_eq":0.589},"42":{"raw":0.7835,"skill":0.5788,"coverage":1,"random":0.486,"tracks":[],"combined":0.8469,"private_eq":0.9217},"43":{"raw":0.4842,"skill":0.1817,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4366,"private_eq":0.6483},"44":{"raw":0.6618,"skill":0.3236,"coverage":1,"random":0.5,"tracks":[],"combined":0.3907,"private_eq":0.6983},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4469,"skill":0.1973,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.6895,"skill":0.379,"coverage":1,"random":0.5,"tracks":[],"combined":0.3607,"private_eq":0.6835},"57":{"raw":0.5441,"skill":0.4872,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5469,"combined":0.4855},"58":{"raw":0.6721,"skill":0.5247,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4055,"combined":0.1294},"59":{"raw":0.7817,"skill":0.5474,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3777,"private_eq":0.7028},"61":{"raw":0.6065,"skill":0.213,"coverage":1,"random":0.5,"tracks":[],"combined":0.3542,"private_eq":0.6803},"62":{"raw":0.6588,"skill":0.5451,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"64":{"raw":0.5909,"skill":0.4886,"coverage":1,"random":0.2,"tracks":[],"combined":0.4968,"private_eq":0.6012}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #112).","Replaces the earlier board entry jiwo-4b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 46.06, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.42% on 895 rows (bar 98.5%).","Author check found shared source text with six index benchmarks.","0.2.1 run resumed under 0.3 (only the rebuilt GSM8K rows newly run)."],"scores":{"balanced_raw":59.63,"balanced_skill":44.1,"breadth_skill":44.43,"public_skill":46.06},"frozen_scores":{"balanced_raw":59.63,"balanced_skill":46.06,"breadth_skill":44.43},"categories":[{"id":"knowledge","raw":0.4629,"skill":0.2907,"combined":0.2882,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6613,"skill":0.5286,"combined":0.5129,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6551,"skill":0.5174,"combined":0.5108,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6692,"skill":0.6314,"combined":0.6518,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5224,"skill":0.2979,"combined":0.3288,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jiwo-4b-v2","pub":46.06,"same_skills":47.21,"new_domains":30.4,"v":44.1,"lab":false,"jev":false,"rank_v":117,"rank_pub":135,"name":"jiwo 4B v2","eq":{"S":46.7,"O":40.19},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":28.02,"new_domains":16.08}}},{"engine":"cam1","name":"Cam1","long_name":"Cam1","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/cbuk10001/Cam1","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":108544,"unsupported":1657,"error":0},"lifted":false,"calibration":{"acc":0.6,"conf":0.6609,"ece":0.0633,"brier":0.5179,"over95":0.0109,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0234,"conf":0.0795,"acc":0.0367},{"lo":0.1,"w":0.031,"conf":0.1458,"acc":0.1209},{"lo":0.2,"w":0.0274,"conf":0.257,"acc":0.2751},{"lo":0.3,"w":0.0706,"conf":0.3588,"acc":0.3058},{"lo":0.4,"w":0.0726,"conf":0.4488,"acc":0.4577},{"lo":0.5,"w":0.1395,"conf":0.5482,"acc":0.508},{"lo":0.6,"w":0.1425,"conf":0.6511,"acc":0.5937},{"lo":0.7,"w":0.1519,"conf":0.7506,"acc":0.6961},{"lo":0.8,"w":0.1764,"conf":0.8512,"acc":0.7626},{"lo":0.9,"w":0.1646,"conf":0.9487,"acc":0.8338}],"areas":{"knowledge":{"ece":0.0962,"acc":0.5468,"conf":0.643},"language":{"ece":0.1077,"acc":0.6531,"conf":0.7067},"retrieval":{"ece":0.0973,"acc":0.7252,"conf":0.6952},"tools":{"ece":0.0656,"acc":0.7872,"conf":0.7488},"arts":{"ece":0.139,"acc":0.3983,"conf":0.5344}}},"gaps":null,"latency":{"median":248.9,"p95":905.7,"mean":397.5},"results":{"1":{"score":0.6889,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":310.4},"2":{"score":0.5913,"jev":null,"metric":"nDCG@10","requests":685,"answered":660,"unsupported":25,"errors":0,"median_ms":3097.6},"3":{"score":0.6496,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2071.2},"4":{"score":0.7674,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":487.8},"5":{"score":0.7913,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":907.6},"6":{"score":0.7848,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":667.3},"9":{"score":0.0795,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":795.9},"10":{"score":0.6587,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":349.1},"11":{"score":0.6322,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1166},"12":{"score":0.5311,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":232.9},"20":{"score":0.5172,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":230.6},"21":{"score":0.5114,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":220.2},"22":{"score":0.088,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":2132.3},"23":{"score":0.5536,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":271.7},"24":{"score":0.6728,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":229.4},"25":{"score":0.3367,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":247},"26":{"score":0.9579,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":221.9},"27":{"score":0.8831,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":229.9},"28":{"score":0.633,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":220.3},"29":{"score":0.7391,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":246.7},"30":{"score":0.2635,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1038,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":373.8},"32":{"score":0.5625,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":352.5},"33":{"score":0.2279,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":369.5},"36":{"score":0.2919,"jev":null,"metric":"nDCG@10","requests":220,"answered":139,"unsupported":81,"errors":0,"median_ms":3242.4},"37":{"score":0.3946,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":262.5},"38":{"score":0.041,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2963.3},"39":{"score":0.8374,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":248.2},"40":{"score":0.3878,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":223.4},"41":{"score":0.5211,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":237},"42":{"score":0.6443,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":271},"43":{"score":0.4474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":227.9},"44":{"score":0.5688,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":228.8},"45":{"score":0.1198,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":248.8},"50":{"score":0.3532,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":333.3},"56":{"score":null,"jev":null,"metric":"accuracy","requests":2000,"answered":0,"unsupported":2000,"errors":0,"median_ms":null},"57":{"score":0.4261,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":249},"58":{"score":0.5996,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":237.3},"59":{"score":0.1344,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":344.7},"61":{"score":0.619,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":289.2},"62":{"score":0.5887,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":307.5},"64":{"score":0.4924,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":241.5}},"benchmarks":{"1":{"raw":0.6889,"skill":0.58,"coverage":1,"random":0.2592,"tracks":[],"combined":0.565,"private_eq":0.6807},"2":{"raw":0.5697,"skill":0.5031,"coverage":0.9635,"random":0.1341,"tracks":[],"combined":0.5384,"private_eq":0.604},"3":{"raw":0.6496,"skill":0.6428,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7386,"private_eq":0.7455},"4":{"raw":0.7674,"skill":0.7644,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7858,"private_eq":0.7899},"5":{"raw":0.7913,"skill":0.79,"coverage":1,"random":0.006,"tracks":[],"combined":0.6224,"private_eq":0.6279},"6":{"raw":0.7848,"skill":0.4954,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0795,"skill":0.0795,"coverage":1,"random":0,"tracks":[],"combined":0.0527,"private_eq":0.0625},"10":{"raw":0.6587,"skill":0.4321,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6322,"skill":0.4681,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5311,"skill":0.2976,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4671,"private_eq":0.6476},"20":{"raw":0.5165,"skill":0.0331,"coverage":1,"random":0.5,"tracks":[],"combined":0.1914,"private_eq":0.5998},"21":{"raw":0.5114,"skill":0.0228,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0844,"skill":0.0772,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.085,"private_eq":0.1015},"23":{"raw":0.5636,"skill":0.1271,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6728,"skill":0.5637,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3367,"skill":0.1156,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3298,"combined":0.0971},"26":{"raw":0.9579,"skill":0.9439,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8831,"skill":0.8441,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.633,"skill":0.266,"coverage":1,"random":0.5,"tracks":[],"combined":0.0475,"private_eq":0.5287},"29":{"raw":0.7391,"skill":0.6521,"coverage":1,"random":0.25,"tracks":[],"combined":0.5574,"private_eq":0.6711},"30":{"raw":0.2635,"skill":0.1117,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3707,"random":0.25},{"track":"10-choice","raw":0.1562,"random":0.1}],"combined":0.375,"private_eq":0.4894},"31":{"raw":0.1038,"skill":0.0239,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0127,"private_eq":0.103},"32":{"raw":0.5625,"skill":0.3045,"coverage":1,"random":0.371,"tracks":[],"combined":0.0493,"private_eq":0.4082},"33":{"raw":0.2279,"skill":0.2177,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1995,"private_eq":0.218},"36":{"raw":0.1844,"skill":0.0774,"coverage":0.6318,"random":0.116,"tracks":[],"combined":0.2126,"private_eq":0.311},"37":{"raw":0.3946,"skill":0.2407,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4168,"private_eq":0.5395},"38":{"raw":0.041,"skill":0.0103,"coverage":1,"random":0.031,"tracks":[],"combined":0.0097,"private_eq":0.0504},"39":{"raw":0.8374,"skill":0.7608,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4296,"private_eq":0.6159},"40":{"raw":0.3878,"skill":0.2124,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1818,"headline":false},{"track":"A · English","score":0.3878,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.62,"headline":false}],"combined":0.0657,"private_eq":0.2813},"41":{"raw":0.5211,"skill":0.2817,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2336,"private_eq":0.4942},"42":{"raw":0.6443,"skill":0.308,"coverage":1,"random":0.486,"tracks":[],"combined":0.7218,"private_eq":0.8581},"43":{"raw":0.4474,"skill":0.1232,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3196,"private_eq":0.5754},"44":{"raw":0.5688,"skill":0.1376,"coverage":1,"random":0.5,"tracks":[],"combined":0.2515,"private_eq":0.6295},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3532,"skill":0.0613,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0,"skill":0,"coverage":0,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.4261,"skill":0.3545,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3555,"combined":0.2678},"58":{"raw":0.5996,"skill":0.4196,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.39,"combined":0.1066},"59":{"raw":0.1344,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.619,"skill":0.238,"coverage":1,"random":0.5,"tracks":[],"combined":0.4564,"private_eq":0.7308},"62":{"raw":0.5887,"skill":0.4516,"coverage":1,"random":0.25,"tracks":[],"combined":0.2823,"private_eq":0.4671},"64":{"raw":0.4924,"skill":0.3655,"coverage":1,"random":0.2,"tracks":[],"combined":0.3319,"private_eq":0.4708}},"coverage":0.9549,"pending":0.0451,"notes":["New in the 2026-10-10 PR batch (PR #107).","Submitted public 0.3 run adopted (kit rescore 30.72, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 100.00% on 815 rows (bar 98.5%).","Training data not disclosed; context cap 16,384 kept (hard-coded), 107 overlength refusals count as wrong."],"scores":{"balanced_raw":46.39,"balanced_skill":34.36,"breadth_skill":29.05,"public_skill":30.72},"frozen_scores":{"balanced_raw":46.39,"balanced_skill":30.72,"breadth_skill":29.05},"categories":[{"id":"knowledge","raw":0.3927,"skill":0.1921,"combined":0.1695,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5024,"skill":0.3384,"combined":0.3189,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4705,"skill":0.3692,"combined":0.4254,"coverage":0.7815,"pending":0,"n":6},{"id":"tools","raw":0.5267,"skill":0.4633,"combined":0.4515,"coverage":0.9932,"pending":0,"n":5},{"id":"arts","raw":0.4202,"skill":0.1145,"combined":0.1573,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"cam1","pub":30.72,"same_skills":32.28,"new_domains":29.02,"v":34.36,"lab":false,"jev":false,"rank_v":153,"rank_pub":186,"name":"Cam1","eq":{"S":31.6,"O":38.49},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":18.96,"new_domains":15.4}}},{"engine":"purplemist-flash-1.0","name":"PurpleMIST-Flash-1.0","long_name":"PurpleMIST-Flash-1.0","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":7936684544,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/olaverse/PurpleMIST-Flash-1.0","code_url":null,"base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":null,"served_checkpoint":null,"org":{"id":"olaverse","name":"Olaverse Lab","logo":"data/orgs/olaverse.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6821,"conf":0.7128,"ece":0.032,"brier":0.4118,"over95":0.0085,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0292,"conf":0.0759,"acc":0.0839},{"lo":0.1,"w":0.0298,"conf":0.1401,"acc":0.1473},{"lo":0.2,"w":0.0184,"conf":0.2553,"acc":0.2646},{"lo":0.3,"w":0.0465,"conf":0.3537,"acc":0.3092},{"lo":0.4,"w":0.0657,"conf":0.4566,"acc":0.4212},{"lo":0.5,"w":0.1267,"conf":0.5484,"acc":0.5151},{"lo":0.6,"w":0.1152,"conf":0.6496,"acc":0.6093},{"lo":0.7,"w":0.1124,"conf":0.7507,"acc":0.7264},{"lo":0.8,"w":0.1326,"conf":0.8518,"acc":0.8062},{"lo":0.9,"w":0.3236,"conf":0.9693,"acc":0.9403}],"areas":{"knowledge":{"ece":0.0894,"acc":0.6045,"conf":0.6939},"language":{"ece":0.0273,"acc":0.8135,"conf":0.7968},"retrieval":{"ece":0.0524,"acc":0.6903,"conf":0.6814},"tools":{"ece":0.0406,"acc":0.8789,"conf":0.8569},"arts":{"ece":0.0695,"acc":0.4759,"conf":0.5383}}},"gaps":null,"latency":{"median":52.9,"p95":163.6,"mean":77},"results":{"1":{"score":0.9292,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":63.6},"2":{"score":0.651,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":490},"3":{"score":0.7539,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":428.2},"4":{"score":0.7377,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":68.9},"5":{"score":0.8062,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":102.7},"6":{"score":0.7988,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":135.4},"9":{"score":0.2955,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":137.6},"10":{"score":0.5731,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":75.2},"11":{"score":0.7637,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":182.7},"12":{"score":0.6723,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":60.2},"20":{"score":0.736,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":61.7},"21":{"score":0.5468,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":61.9},"22":{"score":0.144,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":379.2},"23":{"score":0.6082,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.9},"24":{"score":0.7932,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":61.3},"25":{"score":0.4133,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":63},"26":{"score":0.9819,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":61.7},"27":{"score":0.9548,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":62},"28":{"score":0.7987,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":61.1},"29":{"score":0.8079,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":61.8},"30":{"score":0.6141,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1214,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":66},"32":{"score":0.5798,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":66.6},"33":{"score":0.317,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":70.8},"36":{"score":0.3476,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":698.6},"37":{"score":0.4973,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":61.7},"38":{"score":0.3291,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":385.5},"39":{"score":0.9075,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":61.3},"40":{"score":0.4603,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":60.7},"41":{"score":0.6419,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":61.9},"42":{"score":0.7731,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":64.4},"43":{"score":0.586,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":68.8},"44":{"score":0.6734,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.4},"45":{"score":0.0798,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":76.7},"50":{"score":0.3652,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":75.1},"56":{"score":0.624,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":55.2},"57":{"score":0.594,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":71.8},"58":{"score":0.686,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":60.7},"59":{"score":0.679,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":61.5},"61":{"score":0.6787,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":52.8},"62":{"score":0.7306,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":62.6},"64":{"score":0.6553,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":55.8}},"benchmarks":{"1":{"raw":0.9292,"skill":0.9044,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8512,"private_eq":0.8903},"2":{"raw":0.651,"skill":0.597,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6839,"private_eq":0.7285},"3":{"raw":0.7539,"skill":0.7492,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8173,"private_eq":0.8218},"4":{"raw":0.7377,"skill":0.7343,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7624,"private_eq":0.7671},"5":{"raw":0.8062,"skill":0.805,"coverage":1,"random":0.006,"tracks":[],"combined":0.7147,"private_eq":0.7186},"6":{"raw":0.7988,"skill":0.5283,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2955,"skill":0.2955,"coverage":1,"random":0,"tracks":[],"combined":0.2019,"private_eq":0.21},"10":{"raw":0.5731,"skill":0.2897,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7637,"skill":0.6583,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6723,"skill":0.5091,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5115,"private_eq":0.6769},"20":{"raw":0.7368,"skill":0.4736,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.5468,"skill":0.0936,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.1369,"skill":0.1302,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1498,"private_eq":0.165},"23":{"raw":0.5914,"skill":0.1828,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7932,"skill":0.7243,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.361,"combined":0.1392},"26":{"raw":0.9819,"skill":0.9759,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9548,"skill":0.9397,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7987,"skill":0.5974,"coverage":1,"random":0.5,"tracks":[],"combined":0.3403,"private_eq":0.6734},"29":{"raw":0.8079,"skill":0.7439,"coverage":1,"random":0.25,"tracks":[],"combined":0.8444,"private_eq":0.8839},"30":{"raw":0.6141,"skill":0.537,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6922,"random":0.25},{"track":"10-choice","raw":0.536,"random":0.1}],"combined":0.6493,"private_eq":0.7131},"31":{"raw":0.1214,"skill":0.0431,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0463,"private_eq":0.1335},"32":{"raw":0.5798,"skill":0.332,"coverage":1,"random":0.371,"tracks":[],"combined":0.181,"private_eq":0.4901},"33":{"raw":0.317,"skill":0.3079,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1893,"private_eq":0.208},"36":{"raw":0.3476,"skill":0.262,"coverage":1,"random":0.116,"tracks":[],"combined":0.3743,"private_eq":0.4523},"37":{"raw":0.4973,"skill":0.3694,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4724,"private_eq":0.5833},"38":{"raw":0.3291,"skill":0.3076,"coverage":1,"random":0.031,"tracks":[],"combined":0.4213,"private_eq":0.4447},"39":{"raw":0.9075,"skill":0.864,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5443,"private_eq":0.693},"40":{"raw":0.4603,"skill":0.3056,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3846,"headline":false},{"track":"A · English","score":0.4603,"headline":true},{"track":"C · Arabic pairs","score":0.47,"headline":false},{"track":"C · English pairs","score":0.745,"headline":false}],"combined":0.2983,"private_eq":0.46},"41":{"raw":0.6419,"skill":0.4629,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4615,"private_eq":0.6444},"42":{"raw":0.7731,"skill":0.5586,"coverage":1,"random":0.486,"tracks":[],"combined":0.5862,"private_eq":0.7892},"43":{"raw":0.586,"skill":0.3431,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4241,"private_eq":0.6405},"44":{"raw":0.6734,"skill":0.3468,"coverage":1,"random":0.5,"tracks":[],"combined":0.3692,"private_eq":0.6877},"45":{"raw":0.0798,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3652,"skill":0.0786,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.624,"skill":0.248,"coverage":1,"random":0.5,"tracks":[],"combined":0.6377,"private_eq":0.8204},"57":{"raw":0.594,"skill":0.5434,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4365,"combined":0.3599},"58":{"raw":0.686,"skill":0.5449,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4316,"combined":0.1676},"59":{"raw":0.679,"skill":0.3344,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3326,"private_eq":0.6813},"61":{"raw":0.6787,"skill":0.3574,"coverage":1,"random":0.5,"tracks":[],"combined":0.5131,"private_eq":0.7588},"62":{"raw":0.7306,"skill":0.6408,"coverage":1,"random":0.25,"tracks":[],"combined":0.5287,"private_eq":0.6498},"64":{"raw":0.6553,"skill":0.5691,"coverage":1,"random":0.2,"tracks":[],"combined":0.5647,"private_eq":0.6549}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #123).","Submitted public 0.3 run adopted (kit rescore 46.74, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.36% on 831 rows (bar 98.5%).","Training mix includes tasksource and Open-Jev data (train splits of public benchmarks possibly included, per author)."],"scores":{"balanced_raw":60.2,"balanced_skill":47.54,"breadth_skill":45.34,"public_skill":46.74},"frozen_scores":{"balanced_raw":60.2,"balanced_skill":46.74,"breadth_skill":45.34},"categories":[{"id":"knowledge","raw":0.4982,"skill":0.3495,"combined":0.284,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6735,"skill":0.5314,"combined":0.5085,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6166,"skill":0.4752,"combined":0.5825,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6846,"skill":0.6514,"combined":0.6327,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5054,"skill":0.2546,"combined":0.3382,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"purplemist-flash-1.0","pub":46.74,"same_skills":48.24,"new_domains":36.1,"v":47.54,"lab":false,"jev":false,"rank_v":99,"rank_pub":134,"name":"PurpleMIST-Flash-1.0","eq":{"S":47.74,"O":47.23},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":28.65,"new_domains":18.89}}},{"engine":"sieve-9b-plus","name":"Sieve 9B v2","long_name":"Sieve 9B v2","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/Sieve-9B-v2","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7001,"conf":0.7729,"ece":0.0728,"brier":0.397,"over95":0.0277,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0478,"conf":0.0651,"acc":0.0615},{"lo":0.1,"w":0.0133,"conf":0.1288,"acc":0.1173},{"lo":0.2,"w":0.0057,"conf":0.2568,"acc":0.209},{"lo":0.3,"w":0.0326,"conf":0.3643,"acc":0.2354},{"lo":0.4,"w":0.0462,"conf":0.4558,"acc":0.3537},{"lo":0.5,"w":0.093,"conf":0.548,"acc":0.4802},{"lo":0.6,"w":0.0871,"conf":0.6503,"acc":0.5797},{"lo":0.7,"w":0.0915,"conf":0.7496,"acc":0.6339},{"lo":0.8,"w":0.1023,"conf":0.8529,"acc":0.7679},{"lo":0.9,"w":0.4806,"conf":0.9786,"acc":0.913}],"areas":{"knowledge":{"ece":0.1038,"acc":0.6152,"conf":0.719},"language":{"ece":0.0185,"acc":0.8214,"conf":0.8399},"retrieval":{"ece":0.1642,"acc":0.7107,"conf":0.8739},"tools":{"ece":0.0378,"acc":0.9114,"conf":0.9465},"arts":{"ece":0.0632,"acc":0.5167,"conf":0.5752}}},"gaps":null,"latency":{"median":50.5,"p95":190.6,"mean":118.5},"results":{"1":{"score":0.9581,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":223.6},"2":{"score":0.6733,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1456.3},"3":{"score":0.8583,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":535},"4":{"score":0.8596,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":104.4},"5":{"score":0.9118,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":129.5},"6":{"score":0.795,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":288.7},"9":{"score":0.25,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":447.3},"10":{"score":0.4378,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":111},"11":{"score":0.7343,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":417.5},"12":{"score":0.651,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":84},"20":{"score":0.882,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.1},"21":{"score":0.6138,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":89.3},"22":{"score":0.084,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":505.4},"23":{"score":0.6232,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.8},"24":{"score":0.7871,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":87.1},"25":{"score":0.4643,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":85.5},"26":{"score":0.9815,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":87.7},"27":{"score":0.9539,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":87.6},"28":{"score":0.9084,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":88.1},"29":{"score":0.9666,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":87.2},"30":{"score":0.6869,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1282,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":89.2},"32":{"score":0.5598,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":110},"33":{"score":0.2788,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":252.1},"36":{"score":0.4509,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2459.6},"37":{"score":0.4992,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":89.2},"38":{"score":0.1745,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":648},"39":{"score":0.8895,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":186},"40":{"score":0.5495,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":94.2},"41":{"score":0.5545,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":84.8},"42":{"score":0.7925,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":92.9},"43":{"score":0.5895,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":85},"44":{"score":0.6694,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.9},"45":{"score":0.0998,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":87.6},"50":{"score":0.4033,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":105.5},"56":{"score":0.8405,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":238.2},"57":{"score":0.5938,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":86.2},"58":{"score":0.7033,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":86.4},"59":{"score":0.802,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":103.3},"61":{"score":0.8025,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":88.9},"62":{"score":0.8015,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":101},"64":{"score":0.6591,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":84.1}},"benchmarks":{"1":{"raw":0.9581,"skill":0.9434,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8717,"private_eq":0.9053},"2":{"raw":0.6733,"skill":0.6227,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6815,"private_eq":0.7265},"3":{"raw":0.8583,"skill":0.8556,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8435,"private_eq":0.8472},"4":{"raw":0.8596,"skill":0.8578,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8367,"private_eq":0.8396},"5":{"raw":0.9118,"skill":0.9113,"coverage":1,"random":0.006,"tracks":[],"combined":0.6361,"private_eq":0.6414},"6":{"raw":0.795,"skill":0.5193,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.25,"skill":0.25,"coverage":1,"random":0,"tracks":[],"combined":0.298,"private_eq":0.305},"10":{"raw":0.4378,"skill":0.0646,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7343,"skill":0.6158,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.651,"skill":0.4772,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5343,"private_eq":0.692},"20":{"raw":0.8849,"skill":0.7698,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6138,"skill":0.2275,"coverage":1,"random":0.5,"tracks":[],"combined":0.1258,"private_eq":0.5674},"22":{"raw":0.078,"skill":0.0708,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0762,"private_eq":0.0929},"23":{"raw":0.6318,"skill":0.2636,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7871,"skill":0.7161,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4643,"skill":0.2857,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.389,"combined":0.177},"26":{"raw":0.9815,"skill":0.9753,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9539,"skill":0.9385,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9084,"skill":0.8168,"coverage":1,"random":0.5,"tracks":[],"combined":0.5066,"private_eq":0.7556},"29":{"raw":0.9666,"skill":0.9555,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6869,"skill":0.6238,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7453,"random":0.25},{"track":"10-choice","raw":0.6285,"random":0.1}],"combined":0.6745,"private_eq":0.7337},"31":{"raw":0.1282,"skill":0.0505,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0637,"private_eq":0.1493},"32":{"raw":0.5598,"skill":0.3002,"coverage":1,"random":0.371,"tracks":[],"combined":0.0649,"private_eq":0.4179},"33":{"raw":0.2788,"skill":0.2692,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1533,"private_eq":0.1729},"36":{"raw":0.4509,"skill":0.3788,"coverage":1,"random":0.116,"tracks":[],"combined":0.3745,"private_eq":0.4525},"37":{"raw":0.4992,"skill":0.3718,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5338,"private_eq":0.6317},"38":{"raw":0.1745,"skill":0.1481,"coverage":1,"random":0.031,"tracks":[],"combined":0.2122,"private_eq":0.2443},"39":{"raw":0.8895,"skill":0.8375,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4248,"private_eq":0.6127},"40":{"raw":0.5495,"skill":0.4205,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3912,"headline":false},{"track":"A · English","score":0.5495,"headline":true},{"track":"C · Arabic pairs","score":0.96,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.3347,"private_eq":0.488},"41":{"raw":0.5545,"skill":0.3317,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1383,"private_eq":0.4314},"42":{"raw":0.7925,"skill":0.5963,"coverage":1,"random":0.486,"tracks":[],"combined":0.7637,"private_eq":0.8794},"43":{"raw":0.5895,"skill":0.3487,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4199,"private_eq":0.6379},"44":{"raw":0.6694,"skill":0.3388,"coverage":1,"random":0.5,"tracks":[],"combined":0.4924,"private_eq":0.7486},"45":{"raw":0.0998,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4033,"skill":0.134,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.8405,"skill":0.681,"coverage":1,"random":0.5,"tracks":[],"combined":0.7589,"private_eq":0.8803},"57":{"raw":0.5938,"skill":0.5431,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4394,"combined":0.3632},"58":{"raw":0.7033,"skill":0.5699,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.421,"combined":0.1521},"59":{"raw":0.802,"skill":0.5895,"coverage":1,"random":0.5177,"tracks":[],"combined":0.272,"private_eq":0.6524},"61":{"raw":0.8025,"skill":0.605,"coverage":1,"random":0.5,"tracks":[],"combined":0.4847,"private_eq":0.7448},"62":{"raw":0.8015,"skill":0.7353,"coverage":1,"random":0.25,"tracks":[],"combined":0.5356,"private_eq":0.6549},"64":{"raw":0.6591,"skill":0.5739,"coverage":1,"random":0.2,"tracks":[],"combined":0.5259,"private_eq":0.6242}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #95).","Submitted public 0.3 run adopted (kit rescore 53.35, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.65% on 831 rows (bar 98.5%).","Training data not published; no overlap disclosure."],"scores":{"balanced_raw":64.39,"balanced_skill":47,"breadth_skill":51.95,"public_skill":53.35},"frozen_scores":{"balanced_raw":64.39,"balanced_skill":53.35,"breadth_skill":51.95},"categories":[{"id":"knowledge","raw":0.5197,"skill":0.3825,"combined":0.3062,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6849,"skill":0.5594,"combined":0.4748,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7286,"skill":0.6417,"combined":0.6051,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7231,"skill":0.6976,"combined":0.6617,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5452,"skill":0.3399,"combined":0.3705,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-9b-plus","pub":53.35,"same_skills":49.29,"new_domains":33.72,"v":47,"lab":false,"jev":false,"rank_v":102,"rank_pub":106,"name":"Sieve 9B v2","eq":{"S":48.81,"O":44.29},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":29.29,"new_domains":17.72}}},{"engine":"garas","name":"garas (Gemma 4 31B)","long_name":"garas (Gemma 4 31B)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/plantwaterer/garas","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"show_notes":true},"completed":110201,"counts":{"ok":109958,"unsupported":243,"error":0},"lifted":false,"calibration":{"acc":0.7232,"conf":0.7701,"ece":0.0488,"brier":0.3687,"over95":0.015,"n":34404,"benchmarks":31,"rel":[{"lo":0,"w":0.0052,"conf":0.0886,"acc":0.0458},{"lo":0.1,"w":0.0389,"conf":0.146,"acc":0.1259},{"lo":0.2,"w":0.0212,"conf":0.2467,"acc":0.2188},{"lo":0.3,"w":0.0334,"conf":0.3577,"acc":0.2265},{"lo":0.4,"w":0.052,"conf":0.4544,"acc":0.4607},{"lo":0.5,"w":0.093,"conf":0.5482,"acc":0.4977},{"lo":0.6,"w":0.0839,"conf":0.6515,"acc":0.659},{"lo":0.7,"w":0.1073,"conf":0.7477,"acc":0.6961},{"lo":0.8,"w":0.1191,"conf":0.8532,"acc":0.7489},{"lo":0.9,"w":0.446,"conf":0.9768,"acc":0.9339}],"areas":{"knowledge":{"ece":0.0841,"acc":0.6594,"conf":0.7435},"language":{"ece":0.0473,"acc":0.8327,"conf":0.7866},"retrieval":{"ece":0.2175,"acc":0.6581,"conf":0.872},"tools":{"ece":0.0409,"acc":0.9393,"conf":0.9676},"arts":{"ece":0.0318,"acc":0.5765,"conf":0.5936}}},"gaps":null,"latency":{"median":117.1,"p95":1858.4,"mean":488.5},"results":{"1":{"score":0.9669,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":369.9},"2":{"score":0.6581,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":4741.4},"3":{"score":0.8445,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":6079.1},"4":{"score":0.8747,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":341.6},"5":{"score":0.9341,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":327.1},"6":{"score":0.8004,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1121.9},"9":{"score":0.6477,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":3531.6},"10":{"score":0.0835,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":299.8},"11":{"score":0.7021,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":7489.3},"12":{"score":0.7151,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":140.1},"20":{"score":0.928,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":138.2},"21":{"score":0.6495,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":138.6},"22":{"score":0.236,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3470},"23":{"score":0.6578,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":145.2},"24":{"score":0.8471,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":140.5},"25":{"score":0.4388,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":142.6},"26":{"score":0.9912,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":140.1},"27":{"score":0.9701,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":141},"28":{"score":0.8729,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":139.9},"29":{"score":0.8938,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":140.1},"30":{"score":0.6289,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2004,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":197},"32":{"score":0.629,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":226.8},"33":{"score":0.2879,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":734.9},"36":{"score":0.5011,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":7846.3},"37":{"score":0.5233,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":145.1},"38":{"score":0.2249,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1678.8},"39":{"score":0.9321,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":144.7},"40":{"score":0.6581,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4595,"unsupported":5,"errors":0,"median_ms":141.5},"41":{"score":0.6793,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":140.9},"42":{"score":0.8103,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":146.2},"43":{"score":0.793,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":140.4},"44":{"score":0.7632,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":139.7},"45":{"score":0.1118,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":142.1},"50":{"score":0.4857,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":197.9},"56":{"score":0.8515,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":504.5},"57":{"score":0.6484,"jev":null,"metric":"accuracy","requests":12032,"answered":11728,"unsupported":304,"errors":0,"median_ms":140.1},"58":{"score":0.7265,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":140.1},"59":{"score":0.7206,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":179.4},"61":{"score":0.643,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":145},"62":{"score":0.6804,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":181.7},"64":{"score":0.7519,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":140.8}},"benchmarks":{"1":{"raw":0.9669,"skill":0.9554,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9192,"private_eq":0.9401},"2":{"raw":0.6581,"skill":0.6051,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6868,"private_eq":0.731},"3":{"raw":0.8445,"skill":0.8415,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9171,"private_eq":0.9186},"4":{"raw":0.8747,"skill":0.8731,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8453,"private_eq":0.848},"5":{"raw":0.9341,"skill":0.9337,"coverage":1,"random":0.006,"tracks":[],"combined":0.7458,"private_eq":0.7492},"6":{"raw":0.8004,"skill":0.532,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6477,"skill":0.6477,"coverage":1,"random":0,"tracks":[],"combined":0.6622,"private_eq":0.665},"10":{"raw":0.0835,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7021,"skill":0.5692,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7151,"skill":0.5732,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3768,"private_eq":0.588},"20":{"raw":0.9278,"skill":0.8557,"coverage":1,"random":0.5,"tracks":[],"combined":0.8999,"private_eq":0.95},"21":{"raw":0.6495,"skill":0.2991,"coverage":1,"random":0.5,"tracks":[],"combined":0.4075,"private_eq":0.7066},"22":{"raw":0.2294,"skill":0.2233,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3011,"private_eq":0.3135},"23":{"raw":0.6461,"skill":0.2923,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8471,"skill":0.7961,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4388,"skill":0.2517,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4753,"combined":0.2934},"26":{"raw":0.9912,"skill":0.9883,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8729,"skill":0.7458,"coverage":1,"random":0.5,"tracks":[],"combined":0.7085,"private_eq":0.8554},"29":{"raw":0.8938,"skill":0.8584,"coverage":1,"random":0.25,"tracks":[],"combined":0.9188,"private_eq":0.939},"30":{"raw":0.6289,"skill":0.5551,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7074,"random":0.25},{"track":"10-choice","raw":0.5504,"random":0.1}],"combined":0.6627,"private_eq":0.7241},"31":{"raw":0.2004,"skill":0.1291,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1026,"private_eq":0.1846},"32":{"raw":0.629,"skill":0.4102,"coverage":1,"random":0.371,"tracks":[],"combined":0.3743,"private_eq":0.6103},"33":{"raw":0.2879,"skill":0.2784,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2663,"private_eq":0.2832},"36":{"raw":0.5011,"skill":0.4356,"coverage":1,"random":0.116,"tracks":[],"combined":0.3951,"private_eq":0.4705},"37":{"raw":0.5233,"skill":0.4021,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3804,"private_eq":0.5108},"38":{"raw":0.2249,"skill":0.2001,"coverage":1,"random":0.031,"tracks":[],"combined":0.3708,"private_eq":0.3963},"39":{"raw":0.9321,"skill":0.9001,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4833,"private_eq":0.652},"40":{"raw":0.6581,"skill":0.5602,"coverage":0.9986,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1116,"headline":false},{"track":"A · English","score":0.6581,"headline":true},{"track":"C · Arabic pairs","score":0.89,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.5048,"private_eq":0.6187},"41":{"raw":0.6793,"skill":0.5189,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5533,"private_eq":0.7049},"42":{"raw":0.8103,"skill":0.6309,"coverage":1,"random":0.486,"tracks":[],"combined":1.001,"private_eq":1},"43":{"raw":0.793,"skill":0.6715,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6205,"private_eq":0.7629},"44":{"raw":0.7632,"skill":0.5264,"coverage":1,"random":0.5,"tracks":[],"combined":0.4924,"private_eq":0.7486},"45":{"raw":0.1118,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4857,"skill":0.2536,"coverage":1,"random":0.311,"tracks":[],"combined":0.2539,"private_eq":0.4911},"56":{"raw":0.8515,"skill":0.703,"coverage":1,"random":0.5,"tracks":[],"combined":0.5078,"private_eq":0.7562},"57":{"raw":0.632,"skill":0.5861,"coverage":0.9747,"random":0.1109,"tracks":[],"private_eq":0.6143,"combined":0.5622},"58":{"raw":0.7265,"skill":0.6036,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5043,"combined":0.2742},"59":{"raw":0.7206,"skill":0.4207,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2527,"private_eq":0.6432},"61":{"raw":0.643,"skill":0.286,"coverage":1,"random":0.5,"tracks":[],"combined":0.3146,"private_eq":0.6607},"62":{"raw":0.6804,"skill":0.5739,"coverage":1,"random":0.25,"tracks":[],"combined":0.5562,"private_eq":0.6702},"64":{"raw":0.7519,"skill":0.6899,"coverage":1,"random":0.2,"tracks":[],"combined":0.7489,"private_eq":0.8006}},"coverage":0.9995,"pending":0.0005,"notes":["New in the 2026-10-10 PR batch (PR #117).","Submitted public 0.3 run adopted (kit rescore 56.59, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.36% on 831 rows (bar 98.5%).","Training used train splits of 5 suite benchmarks; 19 self-reported train/test duplicates (56.57 public with them counted wrong).","Two private new-domain tests are not counted for this model: our checks found signs its training data overlapped them. Its new-domain score uses the remaining tests, put on the same scale with the same stock models."],"scores":{"balanced_raw":67.38,"balanced_skill":58.95,"breadth_skill":55.67,"public_skill":56.59},"frozen_scores":{"balanced_raw":67.38,"balanced_skill":56.59,"breadth_skill":55.67},"categories":[{"id":"knowledge","raw":0.5534,"skill":0.4329,"combined":0.4375,"coverage":0.9975,"pending":0,"n":11},{"id":"language","raw":0.7083,"skill":0.587,"combined":0.562,"coverage":0.9999,"pending":0,"n":9},{"id":"retrieval","raw":0.7257,"skill":0.6185,"combined":0.5433,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7703,"skill":0.7376,"combined":0.7609,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6151,"skill":0.4356,"combined":0.5222,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"garas","pub":56.59,"same_skills":56.63,"new_domains":48.89,"v":58.95,"lab":false,"jev":false,"rank_v":58,"rank_pub":93,"name":"garas (Gemma 4 31B)","eq":{"S":56.24,"O":63.03},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":33.74,"new_domains":25.21}}},{"engine":"darwin-27b-ztc","name":"Darwin-27B-ZTC","long_name":"Darwin-27B-ZTC","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25624600064,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/FINAL-Bench/Darwin-27B-ZTC","code_url":null,"base_model":"FINAL-Bench/Darwin-27B-RSI","base_url":"https://huggingface.co/FINAL-Bench/Darwin-27B-RSI","weights_repo":null,"served_checkpoint":null,"org":{"id":"FINAL-Bench","name":"FINAL_Bench","logo":"data/orgs/FINAL-Bench.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7526,"conf":0.7975,"ece":0.0454,"brier":0.3332,"over95":0.0226,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0106,"conf":0.0778,"acc":0.1012},{"lo":0.1,"w":0.0326,"conf":0.1479,"acc":0.1368},{"lo":0.2,"w":0.0176,"conf":0.2517,"acc":0.1841},{"lo":0.3,"w":0.0249,"conf":0.3603,"acc":0.3323},{"lo":0.4,"w":0.0395,"conf":0.4525,"acc":0.4097},{"lo":0.5,"w":0.091,"conf":0.5491,"acc":0.5063},{"lo":0.6,"w":0.0843,"conf":0.6507,"acc":0.6038},{"lo":0.7,"w":0.0781,"conf":0.7514,"acc":0.6897},{"lo":0.8,"w":0.0971,"conf":0.8505,"acc":0.8063},{"lo":0.9,"w":0.5242,"conf":0.9814,"acc":0.9352}],"areas":{"knowledge":{"ece":0.0716,"acc":0.6687,"conf":0.7402},"language":{"ece":0.0206,"acc":0.8649,"conf":0.8794},"retrieval":{"ece":0.1712,"acc":0.7125,"conf":0.8811},"tools":{"ece":0.0364,"acc":0.9457,"conf":0.9818},"arts":{"ece":0.0477,"acc":0.635,"conf":0.5873}}},"gaps":null,"latency":{"median":104,"p95":1156.9,"mean":317.9},"results":{"1":{"score":0.9669,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6748,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8484,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.918,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9457,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.6465,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.36,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8601,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7723,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9528,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.683,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.2475,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.7288,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8766,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5765,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9941,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.971,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.9305,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9763,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.5258,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1708,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.629,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2327,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4842,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5988,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.3734,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.7352,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6769,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.811,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8634,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.7193,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7496,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.0878,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.6289,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.506,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6796,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7848,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8482,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.888,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.9217,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8182,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9669,"skill":0.9554,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6748,"skill":0.6244,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7047,"private_eq":0.7463},"3":{"raw":0.8484,"skill":0.8455,"coverage":1,"random":0.0189,"tracks":[],"combined":0.854,"private_eq":0.8574},"4":{"raw":0.918,"skill":0.9169,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8663,"private_eq":0.8685},"5":{"raw":0.9457,"skill":0.9454,"coverage":1,"random":0.006,"tracks":[],"combined":0.833,"private_eq":0.8349},"6":{"raw":0.6465,"skill":0.1712,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8518,"private_eq":0.8525},"10":{"raw":0.36,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8601,"skill":0.7977,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7723,"skill":0.6589,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4289,"private_eq":0.6224},"20":{"raw":0.9549,"skill":0.9097,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.683,"skill":0.3661,"coverage":1,"random":0.5,"tracks":[],"combined":0.4679,"private_eq":0.7365},"22":{"raw":0.2468,"skill":0.2409,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2945,"private_eq":0.307},"23":{"raw":0.7248,"skill":0.4496,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8766,"skill":0.8355,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5765,"skill":0.4354,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5148,"combined":0.3466},"26":{"raw":0.9941,"skill":0.9921,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.971,"skill":0.9613,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9305,"skill":0.861,"coverage":1,"random":0.5,"tracks":[],"combined":0.6328,"private_eq":0.818},"29":{"raw":0.9763,"skill":0.9684,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.5258,"skill":0.4341,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.649,"random":0.25},{"track":"10-choice","raw":0.4026,"random":0.1}],"combined":0.5947,"private_eq":0.6686},"31":{"raw":0.1708,"skill":0.0969,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.629,"skill":0.4102,"coverage":1,"random":0.371,"tracks":[],"combined":0.3743,"private_eq":0.6103},"33":{"raw":0.2327,"skill":0.2225,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2714,"private_eq":0.2881},"36":{"raw":0.4842,"skill":0.4164,"coverage":1,"random":0.116,"tracks":[],"combined":0.3902,"private_eq":0.4662},"37":{"raw":0.5988,"skill":0.4968,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5508,"private_eq":0.6451},"38":{"raw":0.3734,"skill":0.3534,"coverage":1,"random":0.031,"tracks":[],"combined":0.264,"private_eq":0.294},"39":{"raw":0.7352,"skill":0.6105,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4403,"private_eq":0.6231},"40":{"raw":0.6769,"skill":0.5844,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5113,"headline":false},{"track":"A · English","score":0.6769,"headline":true},{"track":"C · Arabic pairs","score":0.97,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.6029,"private_eq":0.6941},"41":{"raw":0.811,"skill":0.7166,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0247,"private_eq":0.3565},"42":{"raw":0.8634,"skill":0.7342,"coverage":1,"random":0.486,"tracks":[],"combined":0.8924,"private_eq":0.9448},"43":{"raw":0.7193,"skill":0.5546,"coverage":1,"random":0.3697,"tracks":[],"combined":0.583,"private_eq":0.7395},"44":{"raw":0.7496,"skill":0.4992,"coverage":1,"random":0.5,"tracks":[],"combined":0.471,"private_eq":0.738},"45":{"raw":0.0878,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6289,"skill":0.4614,"coverage":1,"random":0.311,"tracks":[],"combined":0.2322,"private_eq":0.4763},"56":{"raw":0.506,"skill":0.012,"coverage":1,"random":0.5,"tracks":[],"combined":0.5857,"private_eq":0.7947},"57":{"raw":0.6796,"skill":0.6396,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6679,"combined":0.6232},"58":{"raw":0.7848,"skill":0.6881,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5156,"combined":0.2908},"59":{"raw":0.8482,"skill":0.6853,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3555,"private_eq":0.6922},"61":{"raw":0.888,"skill":0.776,"coverage":1,"random":0.5,"tracks":[],"combined":0.6605,"private_eq":0.8317},"62":{"raw":0.9217,"skill":0.8956,"coverage":1,"random":0.25,"tracks":[],"combined":0.6589,"private_eq":0.7463},"64":{"raw":0.8182,"skill":0.7728,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #98).","Submitted public 0.3 run adopted (kit rescore 62.63, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.65% on 829 rows (bar 98.5%).","Training on train/dev splits of public decision benchmarks, leakage-filtered per author; server batching makes near-tie answers vary run to run (0.7% per card).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":71.93,"balanced_skill":63.03,"breadth_skill":61.25,"public_skill":62.63},"frozen_scores":{"balanced_raw":71.93,"balanced_skill":62.63,"breadth_skill":61.25},"categories":[{"id":"knowledge","raw":0.5582,"skill":0.4472,"combined":0.4305,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.773,"skill":0.6846,"combined":0.5114,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7288,"skill":0.609,"combined":0.6522,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8842,"skill":0.8669,"combined":0.8119,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6761,"skill":0.5334,"combined":0.5432,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"darwin-27b-ztc","pub":62.63,"same_skills":58.71,"new_domains":54.59,"v":63.03,"lab":false,"jev":false,"rank_v":38,"rank_pub":47,"name":"Darwin-27B-ZTC","eq":{"S":58.34,"O":70.07},"rank_tie":8,"tied":true,"parts":{"public":0,"same_skills":35,"new_domains":28.03}}},{"engine":"linne-fern-1.0","name":"linne-fern 1.0 (27B)","long_name":"linne-fern 1.0 (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/MISA-core/linne-fern-1-0","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"MISA-core","name":"MISA-AICORE","logo":"data/orgs/MISA-core.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.741,"conf":0.8189,"ece":0.0786,"brier":0.3605,"over95":0.0258,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0048,"conf":0.0755,"acc":0.1458},{"lo":0.1,"w":0.014,"conf":0.154,"acc":0.153},{"lo":0.2,"w":0.0179,"conf":0.2546,"acc":0.1849},{"lo":0.3,"w":0.0309,"conf":0.3494,"acc":0.2869},{"lo":0.4,"w":0.0412,"conf":0.4563,"acc":0.4019},{"lo":0.5,"w":0.0873,"conf":0.5483,"acc":0.4513},{"lo":0.6,"w":0.0756,"conf":0.6497,"acc":0.5307},{"lo":0.7,"w":0.0784,"conf":0.7522,"acc":0.6265},{"lo":0.8,"w":0.097,"conf":0.8523,"acc":0.6912},{"lo":0.9,"w":0.553,"conf":0.9831,"acc":0.9291}],"areas":{"knowledge":{"ece":0.1007,"acc":0.6698,"conf":0.7699},"language":{"ece":0.0487,"acc":0.8449,"conf":0.8906},"retrieval":{"ece":0.0535,"acc":0.7484,"conf":0.8001},"tools":{"ece":0.0389,"acc":0.9419,"conf":0.9782},"arts":{"ece":0.1453,"acc":0.5699,"conf":0.7073}}},"gaps":null,"latency":{"median":107.5,"p95":1042.8,"mean":275.3},"results":{"1":{"score":0.9811,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":185.2},"2":{"score":0.6827,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2186},"3":{"score":0.8484,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":960.1},"4":{"score":0.7863,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":188.5},"5":{"score":0.8922,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":424.5},"6":{"score":0.7996,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":494.2},"9":{"score":0.6477,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":3535},"10":{"score":0.629,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":257.4},"11":{"score":0.8006,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":7251},"12":{"score":0.7491,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":93.9},"20":{"score":0.9426,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.9},"21":{"score":0.61,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":79.9},"22":{"score":0.272,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1363.4},"23":{"score":0.6526,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.3},"24":{"score":0.852,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":80.6},"25":{"score":0.5561,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":87.5},"26":{"score":0.992,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":79.6},"27":{"score":0.9684,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":79.6},"28":{"score":0.8729,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":79.7},"29":{"score":0.9616,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":93.2},"30":{"score":0.6615,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.173,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.9},"32":{"score":0.6489,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":230},"33":{"score":0.3376,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":789.3},"36":{"score":0.4973,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5032},"37":{"score":0.5846,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":125},"38":{"score":0.2102,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2060.3},"39":{"score":0.8712,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":133.2},"40":{"score":0.4974,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":90.8},"41":{"score":0.7613,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":90.7},"42":{"score":0.8463,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":133.9},"43":{"score":0.714,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":91},"44":{"score":0.7118,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":87.6},"45":{"score":0.1158,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":95.5},"50":{"score":0.3896,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":205},"56":{"score":0.7055,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":484.8},"57":{"score":0.6739,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":93.4},"58":{"score":0.7772,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":91},"59":{"score":0.812,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":195.4},"61":{"score":0.849,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":128.9},"62":{"score":0.8488,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":171.8},"64":{"score":0.7405,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":93.9}},"benchmarks":{"1":{"raw":0.9811,"skill":0.9745,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9329,"private_eq":0.9501},"2":{"raw":0.6827,"skill":0.6336,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7103,"private_eq":0.7511},"3":{"raw":0.8484,"skill":0.8455,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.7863,"skill":0.7836,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8643,"private_eq":0.8666},"5":{"raw":0.8922,"skill":0.8915,"coverage":1,"random":0.006,"tracks":[],"combined":0.733,"private_eq":0.7366},"6":{"raw":0.7996,"skill":0.5301,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6477,"skill":0.6477,"coverage":1,"random":0,"tracks":[],"combined":0.5812,"private_eq":0.585},"10":{"raw":0.629,"skill":0.3827,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8006,"skill":0.7117,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7491,"skill":0.6242,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5027,"private_eq":0.6711},"20":{"raw":0.9445,"skill":0.8891,"coverage":1,"random":0.5,"tracks":[],"combined":0.8782,"private_eq":0.9393},"21":{"raw":0.61,"skill":0.2199,"coverage":1,"random":0.5,"tracks":[],"combined":0.1962,"private_eq":0.6022},"22":{"raw":0.2678,"skill":0.2621,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2688,"private_eq":0.2818},"23":{"raw":0.6656,"skill":0.3311,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.852,"skill":0.8027,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5561,"skill":0.4082,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4711,"combined":0.2877},"26":{"raw":0.992,"skill":0.9893,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9684,"skill":0.9579,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8729,"skill":0.7458,"coverage":1,"random":0.5,"tracks":[],"combined":0.5319,"private_eq":0.7681},"29":{"raw":0.9616,"skill":0.9488,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6615,"skill":0.5963,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7521,"random":0.25},{"track":"10-choice","raw":0.5709,"random":0.1}],"combined":0.6886,"private_eq":0.7452},"31":{"raw":0.173,"skill":0.0993,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.6489,"skill":0.4418,"coverage":1,"random":0.371,"tracks":[],"combined":0.3047,"private_eq":0.567},"33":{"raw":0.3376,"skill":0.3288,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3126,"private_eq":0.3283},"36":{"raw":0.4973,"skill":0.4313,"coverage":1,"random":0.116,"tracks":[],"combined":0.398,"private_eq":0.473},"37":{"raw":0.5846,"skill":0.479,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5774,"private_eq":0.6661},"38":{"raw":0.2102,"skill":0.1849,"coverage":1,"random":0.031,"tracks":[],"combined":0.3094,"private_eq":0.3375},"39":{"raw":0.8712,"skill":0.8106,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5859,"private_eq":0.721},"40":{"raw":0.4974,"skill":0.3534,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3894,"headline":false},{"track":"A · English","score":0.4974,"headline":true},{"track":"C · Arabic pairs","score":0.675,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.5233,"private_eq":0.6329},"41":{"raw":0.7613,"skill":0.6419,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2658,"private_eq":0.5154},"42":{"raw":0.8463,"skill":0.701,"coverage":1,"random":0.486,"tracks":[],"combined":0.8235,"private_eq":0.9098},"43":{"raw":0.714,"skill":0.5463,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5494,"private_eq":0.7186},"44":{"raw":0.7118,"skill":0.4236,"coverage":1,"random":0.5,"tracks":[],"combined":0.3692,"private_eq":0.6877},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3896,"skill":0.1141,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.7055,"skill":0.411,"coverage":1,"random":0.5,"tracks":[],"combined":0.5165,"private_eq":0.7605},"57":{"raw":0.6739,"skill":0.6332,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5869,"combined":0.531},"58":{"raw":0.7772,"skill":0.6771,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4856,"combined":0.2468},"59":{"raw":0.812,"skill":0.6102,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3435,"private_eq":0.6865},"61":{"raw":0.849,"skill":0.698,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.8488,"skill":0.7984,"coverage":1,"random":0.25,"tracks":[],"combined":0.7067,"private_eq":0.7818},"64":{"raw":0.7405,"skill":0.6756,"coverage":1,"random":0.2,"tracks":[],"combined":0.6615,"private_eq":0.7315}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #110).","Submitted public 0.3 run adopted (kit rescore 58.92, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.36% on 831 rows (bar 98.5%).","Training data undisclosed; the run loaded a local directory, identity with the published shards rests on the agreement gate."],"scores":{"balanced_raw":68.83,"balanced_skill":59.65,"breadth_skill":57.77,"public_skill":58.92},"frozen_scores":{"balanced_raw":68.83,"balanced_skill":58.92,"breadth_skill":57.77},"categories":[{"id":"knowledge","raw":0.5702,"skill":0.4494,"combined":0.3921,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7289,"skill":0.6278,"combined":0.5543,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7197,"skill":0.6236,"combined":0.6234,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8101,"skill":0.7896,"combined":0.7718,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.603,"skill":0.4153,"combined":0.4416,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"linne-fern-1.0","pub":58.92,"same_skills":56.52,"new_domains":50.44,"v":59.65,"lab":false,"jev":false,"rank_v":53,"rank_pub":70,"name":"linne-fern 1.0 (27B)","eq":{"S":56.12,"O":64.94},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":33.67,"new_domains":25.98}}},{"engine":"qwen-decision-27b-noncausal","name":"Qwen Decision 27B noncausal LoRA","long_name":"Qwen Decision 27B noncausal LoRA","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/jsaurabh/qwen-decision-27b-noncausal-lora","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110177,"unsupported":24,"error":0},"lifted":false,"calibration":{"acc":0.7288,"conf":0.7793,"ece":0.0505,"brier":0.358,"over95":0.0083,"n":34407,"benchmarks":31,"rel":[{"lo":0,"w":0.0008,"conf":0.0789,"acc":0},{"lo":0.1,"w":0.0135,"conf":0.1586,"acc":0.1166},{"lo":0.2,"w":0.0201,"conf":0.2548,"acc":0.2185},{"lo":0.3,"w":0.038,"conf":0.3551,"acc":0.3054},{"lo":0.4,"w":0.0446,"conf":0.4529,"acc":0.3605},{"lo":0.5,"w":0.1112,"conf":0.5507,"acc":0.471},{"lo":0.6,"w":0.1037,"conf":0.6508,"acc":0.5107},{"lo":0.7,"w":0.112,"conf":0.7541,"acc":0.6881},{"lo":0.8,"w":0.1276,"conf":0.856,"acc":0.7991},{"lo":0.9,"w":0.4286,"conf":0.9704,"acc":0.9585}],"areas":{"knowledge":{"ece":0.0695,"acc":0.6639,"conf":0.7334},"language":{"ece":0.0133,"acc":0.8339,"conf":0.826},"retrieval":{"ece":0.1584,"acc":0.6433,"conf":0.8001},"tools":{"ece":0.0389,"acc":0.9424,"conf":0.9616},"arts":{"ece":0.0637,"acc":0.6092,"conf":0.6609}}},"gaps":null,"latency":{"median":126.6,"p95":1537.8,"mean":401.4},"results":{"1":{"score":0.9728,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":385.2},"2":{"score":0.681,"jev":null,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":3860.5},"3":{"score":0.8577,"jev":null,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":2135.9},"4":{"score":0.7894,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":239.9},"5":{"score":0.8963,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":399.4},"6":{"score":0.7981,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1240.3},"9":{"score":0.7727,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":5313.5},"10":{"score":0.2532,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":310.5},"11":{"score":0.7451,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":9215},"12":{"score":0.6991,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":84.5},"20":{"score":0.9098,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":86.4},"21":{"score":0.5909,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":74.5},"22":{"score":0.519,"jev":null,"metric":"accuracy","requests":2000,"answered":1975,"unsupported":25,"errors":0,"median_ms":1828.7},"23":{"score":0.6548,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":151.6},"24":{"score":0.8506,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":80.8},"25":{"score":0.5102,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":90.9},"26":{"score":0.9899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":75.2},"27":{"score":0.971,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":80},"28":{"score":0.8508,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":73.8},"29":{"score":0.9327,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":104.8},"30":{"score":0.5546,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1796,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":260.9},"32":{"score":0.6649,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":264.2},"33":{"score":0.2024,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":932.2},"36":{"score":0.4694,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":6981.2},"37":{"score":0.5859,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.1},"38":{"score":0.176,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2872.5},"39":{"score":0.8803,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":122.9},"40":{"score":0.705,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":73.6},"41":{"score":0.7418,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":85.8},"42":{"score":0.8252,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":150.5},"43":{"score":0.7719,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":85.9},"44":{"score":0.7388,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.9},"45":{"score":0.104,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":104.9},"50":{"score":0.398,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":232.2},"56":{"score":0.6095,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":651.2},"57":{"score":0.6479,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":90.8},"58":{"score":0.7696,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":86.5},"59":{"score":0.8121,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":208.1},"61":{"score":0.8087,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":128.4},"62":{"score":0.8116,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":206.5},"64":{"score":0.7027,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":89.2}},"benchmarks":{"1":{"raw":0.9728,"skill":0.9633,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9602,"private_eq":0.9701},"2":{"raw":0.679,"skill":0.6293,"coverage":0.9971,"random":0.1341,"tracks":[],"combined":0.6876,"private_eq":0.7317},"3":{"raw":0.8543,"skill":0.8515,"coverage":0.9961,"random":0.0189,"tracks":[],"combined":0.9118,"private_eq":0.9135},"4":{"raw":0.7894,"skill":0.7867,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8723,"private_eq":0.8744},"5":{"raw":0.8963,"skill":0.8957,"coverage":1,"random":0.006,"tracks":[],"combined":0.7673,"private_eq":0.7703},"6":{"raw":0.7981,"skill":0.5266,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7727,"skill":0.7727,"coverage":1,"random":0,"tracks":[],"combined":0.7254,"private_eq":0.7275},"10":{"raw":0.2532,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7451,"skill":0.6314,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6991,"skill":0.5493,"coverage":1,"random":0.3324,"tracks":[],"combined":0.404,"private_eq":0.606},"20":{"raw":0.9119,"skill":0.8239,"coverage":1,"random":0.5,"tracks":[],"combined":0.8493,"private_eq":0.925},"21":{"raw":0.5909,"skill":0.1819,"coverage":1,"random":0.5,"tracks":[],"combined":0.176,"private_eq":0.5922},"22":{"raw":0.4997,"skill":0.4958,"coverage":0.9875,"random":0.0078,"tracks":[],"combined":0.4215,"private_eq":0.4316},"23":{"raw":0.6638,"skill":0.3276,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8506,"skill":0.8008,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5102,"skill":0.3469,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4421,"combined":0.2486},"26":{"raw":0.9899,"skill":0.9865,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.971,"skill":0.9613,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8508,"skill":0.7016,"coverage":1,"random":0.5,"tracks":[],"combined":0.6731,"private_eq":0.8379},"29":{"raw":0.9327,"skill":0.9103,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.5546,"skill":0.4686,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6717,"random":0.25},{"track":"10-choice","raw":0.4375,"random":0.1}],"combined":0.6493,"private_eq":0.7131},"31":{"raw":0.1796,"skill":0.1065,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0821,"private_eq":0.166},"32":{"raw":0.6649,"skill":0.4672,"coverage":1,"random":0.371,"tracks":[],"combined":0.2892,"private_eq":0.5574},"33":{"raw":0.2024,"skill":0.1918,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1379,"private_eq":0.1579},"36":{"raw":0.4694,"skill":0.3998,"coverage":1,"random":0.116,"tracks":[],"combined":0.3981,"private_eq":0.4731},"37":{"raw":0.5859,"skill":0.4806,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5958,"private_eq":0.6806},"38":{"raw":0.176,"skill":0.1496,"coverage":1,"random":0.031,"tracks":[],"combined":0.2522,"private_eq":0.2827},"39":{"raw":0.8803,"skill":0.8239,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6005,"private_eq":0.7308},"40":{"raw":0.705,"skill":0.6205,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3191,"headline":false},{"track":"A · English","score":0.705,"headline":true},{"track":"C · Arabic pairs","score":0.69,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.6216,"private_eq":0.7084},"41":{"raw":0.7418,"skill":0.6127,"coverage":1,"random":0.3333,"tracks":[],"combined":0.6188,"private_eq":0.7481},"42":{"raw":0.8252,"skill":0.6599,"coverage":1,"random":0.486,"tracks":[],"combined":0.9131,"private_eq":0.9553},"43":{"raw":0.7719,"skill":0.6381,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5494,"private_eq":0.7186},"44":{"raw":0.7388,"skill":0.4776,"coverage":1,"random":0.5,"tracks":[],"combined":0.3852,"private_eq":0.6956},"45":{"raw":0.1038,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.398,"skill":0.1263,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.6095,"skill":0.219,"coverage":1,"random":0.5,"tracks":[],"combined":0.4473,"private_eq":0.7263},"57":{"raw":0.6479,"skill":0.604,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4766,"combined":0.4055},"58":{"raw":0.7696,"skill":0.666,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4755,"combined":0.232},"59":{"raw":0.8121,"skill":0.6104,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3559,"private_eq":0.6924},"61":{"raw":0.8087,"skill":0.6174,"coverage":1,"random":0.5,"tracks":[],"combined":0.6322,"private_eq":0.8177},"62":{"raw":0.8116,"skill":0.7488,"coverage":1,"random":0.25,"tracks":[],"combined":0.6384,"private_eq":0.7311},"64":{"raw":0.7027,"skill":0.6284,"coverage":1,"random":0.2,"tracks":[],"combined":0.6907,"private_eq":0.7546}},"coverage":0.9993,"pending":0.0007,"notes":["New in the 2026-10-10 PR batch (PR #114).","Submitted public 0.3 run adopted (kit rescore 57.70, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.65% on 825 rows (bar 98.5%).","Context cap lifted 8,192 to 262,144 for the private run; training corpus overlap with public benchmarks unaudited (author disclosure)."],"scores":{"balanced_raw":68.26,"balanced_skill":61.12,"breadth_skill":56.45,"public_skill":57.7},"frozen_scores":{"balanced_raw":68.26,"balanced_skill":57.7,"breadth_skill":56.45},"categories":[{"id":"knowledge","raw":0.5476,"skill":0.4277,"combined":0.367,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.728,"skill":0.6234,"combined":0.601,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6955,"skill":0.5781,"combined":0.6243,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8252,"skill":0.8016,"combined":0.7958,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.6278,"skill":0.4306,"combined":0.4609,"coverage":0.9979,"pending":0,"n":6}],"v03":{"engine":"qwen-decision-27b-noncausal","pub":57.7,"same_skills":57.71,"new_domains":51.96,"v":61.12,"lab":false,"jev":false,"rank_v":44,"rank_pub":83,"name":"Qwen Decision 27B noncausal LoRA","eq":{"S":57.32,"O":66.82},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":34.39,"new_domains":26.73}}},{"engine":"reflex-27b-pmscr","name":"reflex Qwen3.8-27B-FP8 (pmscr)","long_name":"reflex Qwen3.8-27B-FP8 (pmscr)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/kshetrajna12/reflex","base_model":"Qwen/Qwen3.8-27B-FP8","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B-FP8","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7327,"conf":0.7773,"ece":0.045,"brier":0.3737,"over95":0.0172,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0007,"conf":0.0821,"acc":0.194},{"lo":0.1,"w":0.0102,"conf":0.1599,"acc":0.1726},{"lo":0.2,"w":0.0231,"conf":0.2564,"acc":0.248},{"lo":0.3,"w":0.0403,"conf":0.353,"acc":0.333},{"lo":0.4,"w":0.0508,"conf":0.4526,"acc":0.3411},{"lo":0.5,"w":0.1082,"conf":0.5489,"acc":0.5203},{"lo":0.6,"w":0.1112,"conf":0.6492,"acc":0.6338},{"lo":0.7,"w":0.1017,"conf":0.7511,"acc":0.6855},{"lo":0.8,"w":0.117,"conf":0.8534,"acc":0.7701},{"lo":0.9,"w":0.437,"conf":0.9721,"acc":0.9334}],"areas":{"knowledge":{"ece":0.0632,"acc":0.6567,"conf":0.7197},"language":{"ece":0.0203,"acc":0.8438,"conf":0.8474},"retrieval":{"ece":0.1569,"acc":0.6579,"conf":0.8129},"tools":{"ece":0.0253,"acc":0.9402,"conf":0.9503},"arts":{"ece":0.0554,"acc":0.6201,"conf":0.6341}}},"gaps":null,"latency":{"median":148.5,"p95":1251.3,"mean":318.7},"results":{"1":{"score":0.9752,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6944,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7926,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9021,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7993,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.5909,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.2503,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7977,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.739,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9606,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6309,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.439,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6536,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8333,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4796,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9727,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8532,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9389,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.4094,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1674,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6516,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2867,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4946,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5868,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.3856,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.8785,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5772,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7282,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8318,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.6561,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6756,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1218,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4224,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.8605,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6147,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.6926,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.7353,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.744,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8122,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7235,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9752,"skill":0.9665,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9805,"private_eq":0.985},"2":{"raw":0.6944,"skill":0.647,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7212,"private_eq":0.7605},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8855,"private_eq":0.888},"4":{"raw":0.7926,"skill":0.7899,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8613,"private_eq":0.8636},"5":{"raw":0.9021,"skill":0.9015,"coverage":1,"random":0.006,"tracks":[],"combined":0.7378,"private_eq":0.7413},"6":{"raw":0.7993,"skill":0.5294,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5909,"skill":0.5909,"coverage":1,"random":0,"tracks":[],"combined":0.5914,"private_eq":0.595},"10":{"raw":0.2503,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7977,"skill":0.7075,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.739,"skill":0.609,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5571,"private_eq":0.707},"20":{"raw":0.962,"skill":0.9241,"coverage":1,"random":0.5,"tracks":[],"combined":0.9361,"private_eq":0.9679},"21":{"raw":0.6309,"skill":0.2618,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.432,"skill":0.4276,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3617,"private_eq":0.3729},"23":{"raw":0.6419,"skill":0.2838,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8333,"skill":0.7777,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4597,"combined":0.2723},"26":{"raw":0.9899,"skill":0.9865,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9727,"skill":0.9636,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8532,"skill":0.7064,"coverage":1,"random":0.5,"tracks":[],"combined":0.5873,"private_eq":0.7955},"29":{"raw":0.9389,"skill":0.9185,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.4094,"skill":0.2913,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5277,"random":0.25},{"track":"10-choice","raw":0.2911,"random":0.1}],"combined":0.4997,"private_eq":0.5911},"31":{"raw":0.1674,"skill":0.0932,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0939,"private_eq":0.1767},"32":{"raw":0.6516,"skill":0.4461,"coverage":1,"random":0.371,"tracks":[],"combined":0.3434,"private_eq":0.5911},"33":{"raw":0.2867,"skill":0.2772,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1738,"private_eq":0.1929},"36":{"raw":0.4946,"skill":0.4283,"coverage":1,"random":0.116,"tracks":[],"combined":0.3804,"private_eq":0.4576},"37":{"raw":0.5868,"skill":0.4818,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5097,"private_eq":0.6127},"38":{"raw":0.3856,"skill":0.3659,"coverage":1,"random":0.031,"tracks":[],"combined":0.555,"private_eq":0.5727},"39":{"raw":0.8785,"skill":0.8213,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5526,"private_eq":0.6986},"40":{"raw":0.5772,"skill":0.4561,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3939,"headline":false},{"track":"A · English","score":0.5772,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.4962,"private_eq":0.6121},"41":{"raw":0.7282,"skill":0.5923,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5903,"private_eq":0.7293},"42":{"raw":0.8318,"skill":0.6728,"coverage":1,"random":0.486,"tracks":[],"combined":0.5755,"private_eq":0.7838},"43":{"raw":0.6561,"skill":0.4544,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4701,"private_eq":0.6692},"44":{"raw":0.6756,"skill":0.3512,"coverage":1,"random":0.5,"tracks":[],"combined":0.4817,"private_eq":0.7433},"45":{"raw":0.1218,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4224,"skill":0.1618,"coverage":1,"random":0.311,"tracks":[],"combined":0.2322,"private_eq":0.4763},"56":{"raw":0.8605,"skill":0.721,"coverage":1,"random":0.5,"tracks":[],"combined":0.802,"private_eq":0.9016},"57":{"raw":0.6147,"skill":0.5666,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.459,"combined":0.3855},"58":{"raw":0.6926,"skill":0.5544,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4545,"combined":0.2012},"59":{"raw":0.7353,"skill":0.4512,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2867,"private_eq":0.6594},"61":{"raw":0.744,"skill":0.488,"coverage":1,"random":0.5,"tracks":[],"combined":0.5586,"private_eq":0.7813},"62":{"raw":0.8122,"skill":0.7496,"coverage":1,"random":0.25,"tracks":[],"combined":0.6999,"private_eq":0.7767},"64":{"raw":0.7235,"skill":0.6544,"coverage":1,"random":0.2,"tracks":[],"combined":0.6615,"private_eq":0.7315}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #81).","Replaces the earlier board entry reflex-27b-v2 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 57.34, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8 (Qwen official block-FP8 checkpoint), bf16 compute dtype and bf16 Mamba state); decision agreement with the submitted rows 98.44% on 900 rows (bar 96%).","No training: prompt scaffold and label-logprob readout on the stock Qwen3.8-27B-FP8 checkpoint.","FP8 checkpoint, so the quantized agreement bar applies: 98.44% vs 96%.","0.2.1 run resumed under 0.3 (only the rebuilt GSM8K rows newly run).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":67.88,"balanced_skill":58.77,"breadth_skill":55.73,"public_skill":57.34},"frozen_scores":{"balanced_raw":67.88,"balanced_skill":57.34,"breadth_skill":55.73},"categories":[{"id":"knowledge","raw":0.5134,"skill":0.3728,"combined":0.3517,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7391,"skill":0.6277,"combined":0.5864,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7301,"skill":0.6416,"combined":0.6432,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.795,"skill":0.7721,"combined":0.7874,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6354,"skill":0.4522,"combined":0.4816,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"reflex-27b-pmscr","pub":57.34,"same_skills":57.39,"new_domains":47.6,"v":58.77,"lab":false,"jev":false,"rank_v":61,"rank_pub":88,"name":"reflex Qwen3.8-27B-FP8 (pmscr)","eq":{"S":57,"O":61.43},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":34.2,"new_domains":24.57}}},{"engine":"rsi-jev-v6.1-vl-27b-auto","name":"RSI-Jev v6.1 VL 27B (effort auto)","long_name":"RSI-Jev v6.1 VL 27B (effort auto)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/shgao/rsi-jev-v6.1-vl-27b","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.772,"conf":0.8486,"ece":0.0771,"brier":0.3267,"over95":0.0281,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0021,"conf":0.0853,"acc":0.0924},{"lo":0.1,"w":0.0135,"conf":0.1535,"acc":0.1357},{"lo":0.2,"w":0.0143,"conf":0.2529,"acc":0.1885},{"lo":0.3,"w":0.0261,"conf":0.3569,"acc":0.2393},{"lo":0.4,"w":0.0387,"conf":0.4556,"acc":0.4605},{"lo":0.5,"w":0.0649,"conf":0.5483,"acc":0.5133},{"lo":0.6,"w":0.0379,"conf":0.6456,"acc":0.5324},{"lo":0.7,"w":0.0657,"conf":0.7461,"acc":0.6427},{"lo":0.8,"w":0.1308,"conf":0.8501,"acc":0.676},{"lo":0.9,"w":0.606,"conf":0.9828,"acc":0.9225}],"areas":{"knowledge":{"ece":0.0977,"acc":0.6819,"conf":0.7796},"language":{"ece":0.0279,"acc":0.8765,"conf":0.9028},"retrieval":{"ece":0.1679,"acc":0.6922,"conf":0.8591},"tools":{"ece":0.0166,"acc":0.9591,"conf":0.9691},"arts":{"ece":0.0988,"acc":0.7154,"conf":0.8085}}},"gaps":null,"latency":{"median":76.1,"p95":564.5,"mean":182.5},"results":{"1":{"score":0.9764,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":136.3},"2":{"score":0.6833,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1605.9},"3":{"score":0.8701,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":540},"4":{"score":0.9229,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":90.9},"5":{"score":0.9409,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":139.7},"6":{"score":0.7847,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":439.1},"9":{"score":0.7955,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":417.6},"10":{"score":0.2606,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":116.5},"11":{"score":0.8402,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":485.1},"12":{"score":0.7675,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31.9},"20":{"score":0.9764,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.4},"21":{"score":0.6084,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":24.9},"22":{"score":0.811,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":483.8},"23":{"score":0.6692,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.9},"24":{"score":0.8315,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":27.5},"25":{"score":0.5102,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":45},"26":{"score":0.9844,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":25.2},"27":{"score":0.9642,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.2},"28":{"score":0.914,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":24.7},"29":{"score":0.9638,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":34.3},"30":{"score":0.7472,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.26,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119},"32":{"score":0.6596,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":106.5},"33":{"score":0.3673,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":220.4},"36":{"score":0.4817,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2656.5},"37":{"score":0.5975,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.6},"38":{"score":0.3207,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":814.7},"39":{"score":0.9279,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":45.4},"40":{"score":0.6621,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":30.2},"41":{"score":0.781,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":33},"42":{"score":0.849,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":59.8},"43":{"score":0.786,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":31.9},"44":{"score":0.842,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.3},"45":{"score":0.0918,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":48.1},"50":{"score":0.5722,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":114.2},"56":{"score":0.7815,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":203.8},"57":{"score":0.6535,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":44.1},"58":{"score":0.8921,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":31.7},"59":{"score":0.8388,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":89.3},"61":{"score":0.8855,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":51.2},"62":{"score":0.8149,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":74.3},"64":{"score":0.8011,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":32.6}},"benchmarks":{"1":{"raw":0.9764,"skill":0.9681,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9192,"private_eq":0.9401},"2":{"raw":0.6833,"skill":0.6343,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6843,"private_eq":0.7289},"3":{"raw":0.8701,"skill":0.8676,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9066,"private_eq":0.9084},"4":{"raw":0.9229,"skill":0.9219,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8487,"private_eq":0.8513},"5":{"raw":0.9409,"skill":0.9405,"coverage":1,"random":0.006,"tracks":[],"combined":0.7409,"private_eq":0.7444},"6":{"raw":0.7847,"skill":0.4952,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7955,"skill":0.7955,"coverage":1,"random":0,"tracks":[],"combined":0.7734,"private_eq":0.775},"10":{"raw":0.2606,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8402,"skill":0.7689,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7675,"skill":0.6517,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5201,"private_eq":0.6826},"20":{"raw":0.9765,"skill":0.953,"coverage":1,"random":0.5,"tracks":[],"combined":0.9794,"private_eq":0.9893},"21":{"raw":0.6084,"skill":0.2169,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.8098,"skill":0.8083,"coverage":1,"random":0.0078,"tracks":[],"combined":0.8157,"private_eq":0.8182},"23":{"raw":0.6647,"skill":0.3295,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8315,"skill":0.7753,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5102,"skill":0.3469,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4607,"combined":0.2737},"26":{"raw":0.9844,"skill":0.9792,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9642,"skill":0.9523,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.914,"skill":0.828,"coverage":1,"random":0.5,"tracks":[],"combined":0.5622,"private_eq":0.7831},"29":{"raw":0.9638,"skill":0.9517,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7472,"skill":0.6958,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7908,"random":0.25},{"track":"10-choice","raw":0.7036,"random":0.1}],"combined":0.6931,"private_eq":0.7489},"31":{"raw":0.26,"skill":0.194,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0983,"private_eq":0.1807},"32":{"raw":0.6596,"skill":0.4588,"coverage":1,"random":0.371,"tracks":[],"combined":0.2815,"private_eq":0.5526},"33":{"raw":0.3673,"skill":0.3589,"coverage":1,"random":0.0131,"tracks":[],"combined":0.215,"private_eq":0.2331},"36":{"raw":0.4817,"skill":0.4136,"coverage":1,"random":0.116,"tracks":[],"combined":0.3841,"private_eq":0.4609},"37":{"raw":0.5975,"skill":0.4952,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5774,"private_eq":0.6661},"38":{"raw":0.3207,"skill":0.299,"coverage":1,"random":0.031,"tracks":[],"combined":0.6081,"private_eq":0.6236},"39":{"raw":0.9279,"skill":0.894,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5711,"private_eq":0.711},"40":{"raw":0.6621,"skill":0.5653,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5131,"headline":false},{"track":"A · English","score":0.6621,"headline":true},{"track":"C · Arabic pairs","score":0.955,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.3867,"private_eq":0.5279},"41":{"raw":0.781,"skill":0.6714,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2192,"private_eq":0.4847},"42":{"raw":0.849,"skill":0.7062,"coverage":1,"random":0.486,"tracks":[],"combined":0.8574,"private_eq":0.927},"43":{"raw":0.786,"skill":0.6604,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4952,"private_eq":0.6848},"44":{"raw":0.842,"skill":0.684,"coverage":1,"random":0.5,"tracks":[],"combined":0.669,"private_eq":0.8359},"45":{"raw":0.0918,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5722,"skill":0.3791,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.7815,"skill":0.563,"coverage":1,"random":0.5,"tracks":[],"combined":0.3694,"private_eq":0.6878},"57":{"raw":0.6535,"skill":0.6103,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4512,"combined":0.3766},"58":{"raw":0.8921,"skill":0.8436,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4689,"combined":0.2223},"59":{"raw":0.8388,"skill":0.6658,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3123,"private_eq":0.6716},"61":{"raw":0.8855,"skill":0.771,"coverage":1,"random":0.5,"tracks":[],"combined":0.6721,"private_eq":0.8374},"62":{"raw":0.8149,"skill":0.7532,"coverage":1,"random":0.25,"tracks":[],"combined":0.6451,"private_eq":0.7361},"64":{"raw":0.8011,"skill":0.7514,"coverage":1,"random":0.2,"tracks":[],"combined":0.7682,"private_eq":0.8159}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #106).","Submitted public 0.3 run adopted (kit rescore 65.64, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.76% on 895 rows (bar 98.5%).","Trained on Decision Index training pools; trunk selected among 9 full 0.3 public reads (author states it).","Effort auto run; the same package at unset effort reads 65.46 public."],"scores":{"balanced_raw":73.93,"balanced_skill":58.6,"breadth_skill":64.93,"public_skill":65.64},"frozen_scores":{"balanced_raw":73.93,"balanced_skill":65.64,"breadth_skill":64.93},"categories":[{"id":"knowledge","raw":0.6113,"skill":0.5172,"combined":0.3865,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7763,"skill":0.6909,"combined":0.569,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7696,"skill":0.691,"combined":0.6041,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8351,"skill":0.8122,"combined":0.7951,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7388,"skill":0.573,"combined":0.5995,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"rsi-jev-v6.1-vl-27b-auto","pub":65.64,"same_skills":58.19,"new_domains":46.27,"v":58.6,"lab":false,"jev":false,"rank_v":62,"rank_pub":30,"name":"RSI-Jev v6.1 VL 27B (effort auto)","eq":{"S":57.81,"O":59.79},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":34.69,"new_domains":23.91}}},{"engine":"anyjev-l0-gemma-4-31b","name":"AnyJev L0 · Gemma-4-31B","long_name":"AnyJev L0 · Gemma-4-31B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/MorrisZJ/anyjev-l0-gemma","base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.735,"conf":0.9753,"ece":0.2403,"brier":0.4979,"over95":0.201,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0,"conf":null,"acc":null},{"lo":0.2,"w":0.0001,"conf":0.2098,"acc":0},{"lo":0.3,"w":0.0015,"conf":0.3699,"acc":0.2081},{"lo":0.4,"w":0.0037,"conf":0.4629,"acc":0.2304},{"lo":0.5,"w":0.0148,"conf":0.5464,"acc":0.3027},{"lo":0.6,"w":0.0127,"conf":0.6483,"acc":0.3654},{"lo":0.7,"w":0.016,"conf":0.7484,"acc":0.3415},{"lo":0.8,"w":0.026,"conf":0.854,"acc":0.2819},{"lo":0.9,"w":0.9252,"conf":0.9971,"acc":0.7695}],"areas":{"knowledge":{"ece":0.2727,"acc":0.6894,"conf":0.9621},"language":{"ece":0.1567,"acc":0.8356,"conf":0.992},"retrieval":{"ece":0.3824,"acc":0.6096,"conf":0.9921},"tools":{"ece":0.0611,"acc":0.9361,"conf":0.9972},"arts":{"ece":0.3205,"acc":0.6246,"conf":0.9451}}},"gaps":null,"latency":{"median":97.9,"p95":1080.3,"mean":260},"results":{"1":{"score":0.9799,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":731},"2":{"score":0.6101,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2072.5},"3":{"score":0.8504,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3495.5},"4":{"score":0.79,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1043.2},"5":{"score":0.915,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1205.2},"6":{"score":0.8005,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1101.5},"9":{"score":0.6477,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1819},"10":{"score":0.015,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":711.1},"11":{"score":0.7489,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2017},"12":{"score":0.7402,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":355.3},"20":{"score":0.9816,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":351.2},"21":{"score":0.653,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":313.7},"22":{"score":0.335,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2358.2},"23":{"score":0.6662,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":449.4},"24":{"score":0.868,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":219.2},"25":{"score":0.5255,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":445.8},"26":{"score":0.9912,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":339},"27":{"score":0.9718,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":356.7},"28":{"score":0.854,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":356.8},"29":{"score":0.9237,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":314.7},"30":{"score":0.7589,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2446,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":801.2},"32":{"score":0.6596,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":524.4},"33":{"score":0.297,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":746.4},"36":{"score":0.454,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3013.4},"37":{"score":0.541,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":416.6},"38":{"score":0.2145,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3008.9},"39":{"score":0.9347,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":444.1},"40":{"score":0.5414,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":385},"41":{"score":0.7411,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":340.8},"42":{"score":0.8327,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":466.8},"43":{"score":0.8053,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":352.9},"44":{"score":0.7688,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":353.7},"45":{"score":0.1397,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":490.1},"50":{"score":0.4869,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":548},"56":{"score":0.878,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":640.3},"57":{"score":0.7056,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":276.1},"58":{"score":0.7587,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":347},"59":{"score":0.7877,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":480.4},"61":{"score":0.7552,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":403.7},"62":{"score":0.8089,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":609.8},"64":{"score":0.7576,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":443.4}},"benchmarks":{"1":{"raw":0.9799,"skill":0.9729,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9329,"private_eq":0.9501},"2":{"raw":0.6101,"skill":0.5497,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6569,"private_eq":0.7054},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9171,"private_eq":0.9186},"4":{"raw":0.79,"skill":0.7873,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8871,"private_eq":0.8888},"5":{"raw":0.915,"skill":0.9145,"coverage":1,"random":0.006,"tracks":[],"combined":0.7396,"private_eq":0.7431},"6":{"raw":0.8005,"skill":0.5322,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6477,"skill":0.6477,"coverage":1,"random":0,"tracks":[],"combined":0.6546,"private_eq":0.6575},"10":{"raw":0.015,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7489,"skill":0.6368,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7402,"skill":0.6108,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4663,"private_eq":0.6471},"20":{"raw":0.9819,"skill":0.9639,"coverage":1,"random":0.5,"tracks":[],"combined":0.965,"private_eq":0.9822},"21":{"raw":0.653,"skill":0.3059,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.335,"skill":0.3298,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3068,"private_eq":0.319},"23":{"raw":0.6651,"skill":0.3301,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.868,"skill":0.824,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5255,"skill":0.3673,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5096,"combined":0.3396},"26":{"raw":0.9912,"skill":0.9883,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9718,"skill":0.9624,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.854,"skill":0.708,"coverage":1,"random":0.5,"tracks":[],"combined":0.6581,"private_eq":0.8305},"29":{"raw":0.9237,"skill":0.8983,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7589,"skill":0.7127,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8249,"random":0.25},{"track":"10-choice","raw":0.6929,"random":0.1}],"combined":0.7437,"private_eq":0.7901},"31":{"raw":0.2446,"skill":0.1773,"coverage":1,"random":0.0819,"tracks":[],"combined":0.095,"private_eq":0.1777},"32":{"raw":0.6596,"skill":0.4588,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.297,"skill":0.2877,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4154,"private_eq":0.4286},"36":{"raw":0.454,"skill":0.3823,"coverage":1,"random":0.116,"tracks":[],"combined":0.3819,"private_eq":0.4589},"37":{"raw":0.541,"skill":0.4243,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5501,"private_eq":0.6446},"38":{"raw":0.2145,"skill":0.1894,"coverage":1,"random":0.031,"tracks":[],"combined":0.3476,"private_eq":0.3741},"39":{"raw":0.9347,"skill":0.904,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5836,"private_eq":0.7194},"40":{"raw":0.5414,"skill":0.41,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5461,"headline":false},{"track":"A · English","score":0.5414,"headline":true},{"track":"C · Arabic pairs","score":0.9,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.4826,"private_eq":0.6016},"41":{"raw":0.7411,"skill":0.6116,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4709,"private_eq":0.6506},"42":{"raw":0.8327,"skill":0.6745,"coverage":1,"random":0.486,"tracks":[],"combined":0.8475,"private_eq":0.922},"43":{"raw":0.8053,"skill":0.691,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6038,"private_eq":0.7525},"44":{"raw":0.7688,"skill":0.5376,"coverage":1,"random":0.5,"tracks":[],"combined":0.471,"private_eq":0.738},"45":{"raw":0.1397,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4869,"skill":0.2553,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.878,"skill":0.756,"coverage":1,"random":0.5,"tracks":[],"combined":0.7069,"private_eq":0.8546},"57":{"raw":0.7056,"skill":0.6689,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.542,"combined":0.48},"58":{"raw":0.7587,"skill":0.6502,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4871,"combined":0.249},"59":{"raw":0.7877,"skill":0.5598,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3647,"private_eq":0.6966},"61":{"raw":0.7552,"skill":0.5104,"coverage":1,"random":0.5,"tracks":[],"combined":0.5414,"private_eq":0.7728},"62":{"raw":0.8089,"skill":0.7452,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.7576,"skill":0.697,"coverage":1,"random":0.2,"tracks":[],"combined":0.7392,"private_eq":0.7929}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #109).","Submitted public 0.3 run adopted (kit rescore 59.79, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (vLLM --quantization fp8 (dynamic FP8 at load of the bf16 checkpoint)); decision agreement with the submitted rows 98.18% on 827 rows (bar 96%).","No training: prompt, restricted label readout, rotation averaging and a label-free running prior on stock gemma-4-31B-it.","Served in FP8 (vLLM dynamic quantization at load), so the quantized agreement bar applies: 98.18% vs 96%.","Running prior is per process and history-dependent; submitted environment provenance does not match the published engine."],"scores":{"balanced_raw":69.37,"balanced_skill":63.65,"breadth_skill":59.06,"public_skill":59.79},"frozen_scores":{"balanced_raw":69.37,"balanced_skill":59.79,"breadth_skill":59.06},"categories":[{"id":"knowledge","raw":0.5929,"skill":0.4782,"combined":0.4429,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7232,"skill":0.6167,"combined":0.5783,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.722,"skill":0.6351,"combined":0.6376,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7895,"skill":0.7643,"combined":0.7646,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6466,"skill":0.4803,"combined":0.5088,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"anyjev-l0-gemma-4-31b","pub":59.79,"same_skills":59.13,"new_domains":55.34,"v":63.65,"lab":false,"jev":false,"rank_v":36,"rank_pub":61,"name":"AnyJev L0 · Gemma-4-31B","eq":{"S":58.76,"O":70.99},"rank_tie":8,"tied":true,"parts":{"public":0,"same_skills":35.26,"new_domains":28.39}}},{"engine":"decisio-0.6.0-gemma-4-31b","name":"decisio 0.6.0 (Gemma 4 31B)","long_name":"decisio 0.6.0 (Gemma 4 31B)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/aminry/decisio","base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7203,"conf":0.8624,"ece":0.1423,"brier":0.4354,"over95":0.0854,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0037,"conf":0.0774,"acc":0.1086},{"lo":0.1,"w":0.014,"conf":0.1561,"acc":0.1231},{"lo":0.2,"w":0.0183,"conf":0.2487,"acc":0.1738},{"lo":0.3,"w":0.0199,"conf":0.3559,"acc":0.2115},{"lo":0.4,"w":0.0258,"conf":0.4503,"acc":0.2361},{"lo":0.5,"w":0.0364,"conf":0.5487,"acc":0.3836},{"lo":0.6,"w":0.0426,"conf":0.6521,"acc":0.504},{"lo":0.7,"w":0.0752,"conf":0.756,"acc":0.7109},{"lo":0.8,"w":0.099,"conf":0.8497,"acc":0.6722},{"lo":0.9,"w":0.6651,"conf":0.9741,"acc":0.8257}],"areas":{"knowledge":{"ece":0.1477,"acc":0.6818,"conf":0.8295},"language":{"ece":0.1227,"acc":0.8212,"conf":0.9439},"retrieval":{"ece":0.2978,"acc":0.6043,"conf":0.8428},"tools":{"ece":0.0383,"acc":0.9335,"conf":0.8996},"arts":{"ece":0.2021,"acc":0.5804,"conf":0.7747}}},"gaps":null,"latency":{"median":53.9,"p95":475.2,"mean":151.6},"results":{"1":{"score":0.9793,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":170.5},"2":{"score":0.6132,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1671.6},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1163.3},"4":{"score":0.7845,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":100.3},"5":{"score":0.9015,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":128.2},"6":{"score":0.8005,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":403.8},"9":{"score":0.6136,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":683.9},"10":{"score":0.01,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":175.6},"11":{"score":0.7264,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":900.3},"12":{"score":0.7374,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":42.2},"20":{"score":0.8908,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35},"21":{"score":0.6252,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":32.6},"22":{"score":0.262,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":779.1},"23":{"score":0.6508,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39},"24":{"score":0.8672,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":34.2},"25":{"score":0.5204,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":50.4},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.2},"27":{"score":0.9718,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":33.4},"28":{"score":0.8524,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":32.6},"29":{"score":0.9232,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":53.6},"30":{"score":0.7176,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2198,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.1},"32":{"score":0.6396,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":116.7},"33":{"score":0.323,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":285.6},"36":{"score":0.4386,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2226.4},"37":{"score":0.5399,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.6},"38":{"score":0.2147,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1992.1},"39":{"score":0.9314,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":93.5},"40":{"score":0.5056,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":32.6},"41":{"score":0.6869,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":43.1},"42":{"score":0.8351,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":59.2},"43":{"score":0.8316,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":41.9},"44":{"score":0.764,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.7},"45":{"score":0.1497,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":52.8},"50":{"score":0.4845,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":128.9},"56":{"score":0.8745,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":314.7},"57":{"score":0.6939,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":49.3},"58":{"score":0.7619,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":40.9},"59":{"score":0.7835,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":67.7},"61":{"score":0.7562,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":72.1},"62":{"score":0.7725,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":69.3},"64":{"score":0.7386,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":35.8}},"benchmarks":{"1":{"raw":0.9793,"skill":0.9721,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6132,"skill":0.5534,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6621,"private_eq":0.7099},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9276,"private_eq":0.9288},"4":{"raw":0.7845,"skill":0.7817,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8431,"private_eq":0.8459},"5":{"raw":0.9015,"skill":0.9009,"coverage":1,"random":0.006,"tracks":[],"combined":0.7615,"private_eq":0.7646},"6":{"raw":0.8005,"skill":0.5322,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6136,"skill":0.6136,"coverage":1,"random":0,"tracks":[],"combined":0.652,"private_eq":0.655},"10":{"raw":0.01,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7264,"skill":0.6043,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7374,"skill":0.6067,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4918,"private_eq":0.6639},"20":{"raw":0.8943,"skill":0.7885,"coverage":1,"random":0.5,"tracks":[],"combined":0.8566,"private_eq":0.9286},"21":{"raw":0.6252,"skill":0.2504,"coverage":1,"random":0.5,"tracks":[],"combined":0.176,"private_eq":0.5922},"22":{"raw":0.2504,"skill":0.2446,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2681,"private_eq":0.2811},"23":{"raw":0.639,"skill":0.2779,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8672,"skill":0.8229,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5204,"skill":0.3605,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5189,"combined":0.3522},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9718,"skill":0.9624,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8524,"skill":0.7048,"coverage":1,"random":0.5,"tracks":[],"combined":0.6581,"private_eq":0.8305},"29":{"raw":0.9232,"skill":0.8976,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7176,"skill":0.6631,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7923,"random":0.25},{"track":"10-choice","raw":0.6429,"random":0.1}],"combined":0.7347,"private_eq":0.7828},"31":{"raw":0.2198,"skill":0.1502,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0853,"private_eq":0.1689},"32":{"raw":0.6396,"skill":0.427,"coverage":1,"random":0.371,"tracks":[],"combined":0.3899,"private_eq":0.62},"33":{"raw":0.323,"skill":0.314,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4359,"private_eq":0.4486},"36":{"raw":0.4386,"skill":0.3649,"coverage":1,"random":0.116,"tracks":[],"combined":0.3922,"private_eq":0.4679},"37":{"raw":0.5399,"skill":0.4229,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5527,"private_eq":0.6466},"38":{"raw":0.2147,"skill":0.1896,"coverage":1,"random":0.031,"tracks":[],"combined":0.3594,"private_eq":0.3854},"39":{"raw":0.9314,"skill":0.8991,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5955,"private_eq":0.7274},"40":{"raw":0.5056,"skill":0.364,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5008,"headline":false},{"track":"A · English","score":0.5056,"headline":true},{"track":"C · Arabic pairs","score":0.91,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.4126,"private_eq":0.5478},"41":{"raw":0.6869,"skill":0.5304,"coverage":1,"random":0.3333,"tracks":[],"combined":0.458,"private_eq":0.6421},"42":{"raw":0.8351,"skill":0.6792,"coverage":1,"random":0.486,"tracks":[],"combined":0.8186,"private_eq":0.9073},"43":{"raw":0.8316,"skill":0.7328,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5704,"private_eq":0.7317},"44":{"raw":0.764,"skill":0.528,"coverage":1,"random":0.5,"tracks":[],"combined":0.562,"private_eq":0.783},"45":{"raw":0.1497,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4845,"skill":0.2518,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.8745,"skill":0.749,"coverage":1,"random":0.5,"tracks":[],"combined":0.6723,"private_eq":0.8375},"57":{"raw":0.6939,"skill":0.6557,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5128,"combined":0.4467},"58":{"raw":0.7619,"skill":0.6549,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4907,"combined":0.2543},"59":{"raw":0.7835,"skill":0.5511,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3758,"private_eq":0.7019},"61":{"raw":0.7562,"skill":0.5124,"coverage":1,"random":0.5,"tracks":[],"combined":0.5869,"private_eq":0.7953},"62":{"raw":0.7725,"skill":0.6967,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"64":{"raw":0.7386,"skill":0.6732,"coverage":1,"random":0.2,"tracks":[],"combined":0.7198,"private_eq":0.7776}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #115).","Submitted public 0.3 run adopted (kit rescore 58.09, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (vLLM 0.30.0 online FP8 at load); decision agreement with the submitted rows 97.89% on 895 rows (bar 96%).","No training: prompt, summed letter-logit readout and fitted temperatures on frozen gemma-4-31B-it.","Served in FP8 (vLLM online quantization at load), so the quantized agreement bar applies: 97.89% vs 96%.","Temperatures fitted on a suite containing 650 Decision Index public-pool questions; base choice and rendering rules made after index measurements (author-declared)."],"scores":{"balanced_raw":68.18,"balanced_skill":62.68,"breadth_skill":57.25,"public_skill":58.09},"frozen_scores":{"balanced_raw":68.18,"balanced_skill":58.09,"breadth_skill":57.25},"categories":[{"id":"knowledge","raw":0.5891,"skill":0.4722,"combined":0.4474,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7102,"skill":0.5982,"combined":0.5728,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7152,"skill":0.6275,"combined":0.6376,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7777,"skill":0.7504,"combined":0.7588,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6053,"skill":0.4144,"combined":0.4534,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"decisio-0.6.0-gemma-4-31b","pub":58.09,"same_skills":58.5,"new_domains":54.15,"v":62.68,"lab":false,"jev":false,"rank_v":39,"rank_pub":76,"name":"decisio 0.6.0 (Gemma 4 31B)","eq":{"S":58.12,"O":69.52},"rank_tie":9,"tied":false,"parts":{"public":0,"same_skills":34.87,"new_domains":27.81}}},{"engine":"decider-31b","name":"Decider 31B (NVFP4)","long_name":"Decider 31B (NVFP4)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":32680000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Mapika/decider-31b","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.763,"conf":0.7427,"ece":0.0243,"brier":0.3349,"over95":0.0009,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0008,"conf":0.0945,"acc":0},{"lo":0.1,"w":0.0086,"conf":0.1595,"acc":0.0958},{"lo":0.2,"w":0.02,"conf":0.2539,"acc":0.2991},{"lo":0.3,"w":0.0369,"conf":0.3495,"acc":0.3186},{"lo":0.4,"w":0.0522,"conf":0.4528,"acc":0.4477},{"lo":0.5,"w":0.1019,"conf":0.5508,"acc":0.5525},{"lo":0.6,"w":0.1016,"conf":0.6469,"acc":0.6577},{"lo":0.7,"w":0.2099,"conf":0.7471,"acc":0.755},{"lo":0.8,"w":0.2291,"conf":0.8531,"acc":0.8999},{"lo":0.9,"w":0.239,"conf":0.9435,"acc":0.9759}],"areas":{"knowledge":{"ece":0.0463,"acc":0.6991,"conf":0.6868},"language":{"ece":0.1165,"acc":0.8585,"conf":0.7421},"retrieval":{"ece":0.1297,"acc":0.6328,"conf":0.7619},"tools":{"ece":0.0471,"acc":0.947,"conf":0.9281},"arts":{"ece":0.0358,"acc":0.7123,"conf":0.729}}},"gaps":null,"latency":{"median":72.3,"p95":614.9,"mean":155.5},"results":{"1":{"score":0.9782,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.648,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8563,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8016,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9545,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7999,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0886,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7565,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7563,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.968,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6537,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.8165,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6584,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8732,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5051,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9701,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8524,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9347,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.8666,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.234,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6755,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2509,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.486,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5404,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.3236,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9112,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.7118,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7968,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8407,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8596,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8008,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1198,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4737,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.8625,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7285,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7937,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.7963,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.783,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8245,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7841,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9782,"skill":0.9705,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.648,"skill":0.5935,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6886,"private_eq":0.7326},"3":{"raw":0.8563,"skill":0.8535,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9276,"private_eq":0.9288},"4":{"raw":0.8016,"skill":0.799,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8695,"private_eq":0.8716},"5":{"raw":0.9545,"skill":0.9542,"coverage":1,"random":0.006,"tracks":[],"combined":0.7626,"private_eq":0.7657},"6":{"raw":0.7999,"skill":0.5308,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8973,"private_eq":0.8975},"10":{"raw":0.0886,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7565,"skill":0.6478,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7563,"skill":0.635,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4851,"private_eq":0.6595},"20":{"raw":0.9689,"skill":0.9377,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.6537,"skill":0.3075,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.8098,"skill":0.8083,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5801,"private_eq":0.5871},"23":{"raw":0.6457,"skill":0.2914,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8732,"skill":0.8309,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5262,"combined":0.362},"26":{"raw":0.9907,"skill":0.9876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8524,"skill":0.7048,"coverage":1,"random":0.5,"tracks":[],"combined":0.648,"private_eq":0.8255},"29":{"raw":0.9347,"skill":0.9129,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8666,"skill":0.8406,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8999,"random":0.25},{"track":"10-choice","raw":0.8332,"random":0.1}],"combined":0.8353,"private_eq":0.8648},"31":{"raw":0.234,"skill":0.1657,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0907,"private_eq":0.1738},"32":{"raw":0.6755,"skill":0.4841,"coverage":1,"random":0.371,"tracks":[],"combined":0.3666,"private_eq":0.6055},"33":{"raw":0.2509,"skill":0.241,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2868,"private_eq":0.3032},"36":{"raw":0.486,"skill":0.4185,"coverage":1,"random":0.116,"tracks":[],"combined":0.4015,"private_eq":0.4761},"37":{"raw":0.5404,"skill":0.4236,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5685,"private_eq":0.6591},"38":{"raw":0.3236,"skill":0.302,"coverage":1,"random":0.031,"tracks":[],"combined":0.4504,"private_eq":0.4725},"39":{"raw":0.9112,"skill":0.8694,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5569,"private_eq":0.7015},"40":{"raw":0.7118,"skill":0.6292,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5463,"headline":false},{"track":"A · English","score":0.7118,"headline":true},{"track":"C · Arabic pairs","score":0.905,"headline":false},{"track":"C · English pairs","score":0.99,"headline":false}],"combined":0.5912,"private_eq":0.6851},"41":{"raw":0.7968,"skill":0.6953,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3146,"private_eq":0.5476},"42":{"raw":0.8407,"skill":0.6901,"coverage":1,"random":0.486,"tracks":[],"combined":0.8951,"private_eq":0.9462},"43":{"raw":0.8596,"skill":0.7773,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6666,"private_eq":0.7916},"44":{"raw":0.8008,"skill":0.6016,"coverage":1,"random":0.5,"tracks":[],"combined":0.5458,"private_eq":0.775},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4737,"skill":0.2362,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.8625,"skill":0.725,"coverage":1,"random":0.5,"tracks":[],"combined":0.6895,"private_eq":0.846},"57":{"raw":0.7285,"skill":0.6946,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6182,"combined":0.5666},"58":{"raw":0.7937,"skill":0.701,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5183,"combined":0.2948},"59":{"raw":0.7963,"skill":0.5776,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4035,"private_eq":0.7151},"61":{"raw":0.783,"skill":0.566,"coverage":1,"random":0.5,"tracks":[],"combined":0.6152,"private_eq":0.8093},"62":{"raw":0.8245,"skill":0.766,"coverage":1,"random":0.25,"tracks":[],"combined":0.5356,"private_eq":0.6549},"64":{"raw":0.7841,"skill":0.7301,"coverage":1,"random":0.2,"tracks":[],"combined":0.8167,"private_eq":0.8542}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #94).","Submitted public 0.3 run adopted (kit rescore 64.27, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (ModelOpt NVFP4 (MLP, group 16), bf16 attention, FP8 KV cache); decision agreement with the submitted rows 96.99% on 827 rows (bar 96%).","NVFP4 checkpoint (ModelOpt), so the quantized agreement bar applies: 96.99% vs 96%.","Trained on train splits of 7 suite benchmarks (plus lineage on 12 more), decontaminated by the author; data unpublished.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":73.24,"balanced_skill":66.03,"breadth_skill":63.41,"public_skill":64.27},"frozen_scores":{"balanced_raw":73.24,"balanced_skill":64.27,"breadth_skill":63.41},"categories":[{"id":"knowledge","raw":0.6072,"skill":0.5032,"combined":0.4664,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7624,"skill":0.6669,"combined":0.5947,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7389,"skill":0.6546,"combined":0.6535,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8655,"skill":0.8423,"combined":0.8048,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7226,"skill":0.5519,"combined":0.5718,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"decider-31b","pub":64.27,"same_skills":61.79,"new_domains":56.88,"v":66.03,"lab":false,"jev":false,"rank_v":25,"rank_pub":42,"name":"Decider 31B (NVFP4)","eq":{"S":61.45,"O":72.9},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":36.87,"new_domains":29.16}}},{"engine":"vllm-sr-d3-v310","name":"vLLM-SR d3 (27B)","long_name":"vLLM-SR d3 (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3/tree/v3.1.0","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7771,"conf":0.8189,"ece":0.0428,"brier":0.3008,"over95":0.0133,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0017,"conf":0.0847,"acc":0.1276},{"lo":0.1,"w":0.0178,"conf":0.1566,"acc":0.1795},{"lo":0.2,"w":0.0206,"conf":0.251,"acc":0.2436},{"lo":0.3,"w":0.0328,"conf":0.3553,"acc":0.3078},{"lo":0.4,"w":0.0373,"conf":0.4483,"acc":0.376},{"lo":0.5,"w":0.0806,"conf":0.5484,"acc":0.4835},{"lo":0.6,"w":0.0758,"conf":0.649,"acc":0.5655},{"lo":0.7,"w":0.0768,"conf":0.7512,"acc":0.6389},{"lo":0.8,"w":0.1002,"conf":0.8529,"acc":0.7894},{"lo":0.9,"w":0.5565,"conf":0.981,"acc":0.9607}],"areas":{"knowledge":{"ece":0.091,"acc":0.6942,"conf":0.7848},"language":{"ece":0.0063,"acc":0.8865,"conf":0.8928},"retrieval":{"ece":0.058,"acc":0.7463,"conf":0.8031},"tools":{"ece":0.0293,"acc":0.952,"conf":0.9775},"arts":{"ece":0.081,"acc":0.6658,"conf":0.6718}}},"gaps":null,"latency":{"median":97.1,"p95":1144.3,"mean":318.9},"results":{"1":{"score":0.9734,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":190},"2":{"score":0.6909,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1779.9},"3":{"score":0.8642,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":786.2},"4":{"score":0.9205,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":117.9},"5":{"score":0.9693,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":183.1},"6":{"score":0.7981,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":488.8},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2054.1},"10":{"score":0.573,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":141.4},"11":{"score":0.7851,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3382.4},"12":{"score":0.76,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":39.7},"20":{"score":0.9582,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.3},"21":{"score":0.637,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":33.2},"22":{"score":0.629,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":675.7},"23":{"score":0.656,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.1},"24":{"score":0.885,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":35.1},"25":{"score":0.5663,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":45.3},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.3},"27":{"score":0.9744,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":33.8},"28":{"score":0.9542,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":32.8},"29":{"score":0.9821,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":48.8},"30":{"score":0.8518,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1758,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":120.2},"32":{"score":0.6928,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":122.5},"33":{"score":0.3842,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":381.3},"36":{"score":0.4927,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3013.7},"37":{"score":0.5901,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.8},"38":{"score":0.3005,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1315.9},"39":{"score":0.957,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":61.6},"40":{"score":0.668,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":33.6},"41":{"score":0.7477,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":40.7},"42":{"score":0.86,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":72.5},"43":{"score":0.8807,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":39.9},"44":{"score":0.8124,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.3},"45":{"score":0.1397,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":48.1},"50":{"score":0.4165,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":111},"56":{"score":0.739,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":288.8},"57":{"score":0.7304,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":44.4},"58":{"score":0.8565,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":40.4},"59":{"score":0.8212,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":103},"61":{"score":0.8355,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":68.8},"62":{"score":0.8636,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":102.2},"64":{"score":0.8201,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":43.5}},"benchmarks":{"1":{"raw":0.9734,"skill":0.9641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6909,"skill":0.643,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7183,"private_eq":0.758},"3":{"raw":0.8642,"skill":0.8616,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9118,"private_eq":0.9135},"4":{"raw":0.9205,"skill":0.9195,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9074,"private_eq":0.9086},"5":{"raw":0.9693,"skill":0.9691,"coverage":1,"random":0.006,"tracks":[],"combined":0.8535,"private_eq":0.855},"6":{"raw":0.7981,"skill":0.5266,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.7608,"private_eq":0.7625},"10":{"raw":0.573,"skill":0.2895,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7851,"skill":0.6892,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.76,"skill":0.6405,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5269,"private_eq":0.6871},"20":{"raw":0.9596,"skill":0.9192,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.637,"skill":0.274,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.6208,"skill":0.6178,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5667,"private_eq":0.574},"23":{"raw":0.6612,"skill":0.3223,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.885,"skill":0.8467,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5663,"skill":0.4218,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5366,"combined":0.376},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9744,"skill":0.9659,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9542,"skill":0.9084,"coverage":1,"random":0.5,"tracks":[],"combined":0.6429,"private_eq":0.823},"29":{"raw":0.9821,"skill":0.9761,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8518,"skill":0.8228,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.887,"random":0.25},{"track":"10-choice","raw":0.8165,"random":0.1}],"combined":0.7824,"private_eq":0.8217},"31":{"raw":0.1758,"skill":0.1023,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.6928,"skill":0.5116,"coverage":1,"random":0.371,"tracks":[],"combined":0.3666,"private_eq":0.6055},"33":{"raw":0.3842,"skill":0.376,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4565,"private_eq":0.4687},"36":{"raw":0.4927,"skill":0.4261,"coverage":1,"random":0.116,"tracks":[],"combined":0.4303,"private_eq":0.5012},"37":{"raw":0.5901,"skill":0.4858,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5821,"private_eq":0.6698},"38":{"raw":0.3005,"skill":0.2781,"coverage":1,"random":0.031,"tracks":[],"combined":0.4469,"private_eq":0.4692},"39":{"raw":0.957,"skill":0.9368,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5255,"private_eq":0.6804},"40":{"raw":0.668,"skill":0.5729,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.6047,"headline":false},{"track":"A · English","score":0.668,"headline":true},{"track":"C · Arabic pairs","score":0.89,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.554,"private_eq":0.6565},"41":{"raw":0.7477,"skill":0.6216,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5128,"private_eq":0.6782},"42":{"raw":0.86,"skill":0.7276,"coverage":1,"random":0.486,"tracks":[],"combined":0.9418,"private_eq":0.9699},"43":{"raw":0.8807,"skill":0.8107,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5955,"private_eq":0.7473},"44":{"raw":0.8124,"skill":0.6248,"coverage":1,"random":0.5,"tracks":[],"combined":0.5673,"private_eq":0.7856},"45":{"raw":0.1397,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4165,"skill":0.1531,"coverage":1,"random":0.311,"tracks":[],"combined":0.2756,"private_eq":0.5059},"56":{"raw":0.739,"skill":0.478,"coverage":1,"random":0.5,"tracks":[],"combined":0.6462,"private_eq":0.8246},"57":{"raw":0.7304,"skill":0.6968,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5547,"combined":0.4944},"58":{"raw":0.8565,"skill":0.792,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.528,"combined":0.309},"59":{"raw":0.8212,"skill":0.6293,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3334,"private_eq":0.6817},"61":{"raw":0.8355,"skill":0.671,"coverage":1,"random":0.5,"tracks":[],"combined":0.6834,"private_eq":0.843},"62":{"raw":0.8636,"skill":0.8181,"coverage":1,"random":0.25,"tracks":[],"combined":0.5903,"private_eq":0.6955},"64":{"raw":0.8201,"skill":0.7751,"coverage":1,"random":0.2,"tracks":[],"combined":0.7975,"private_eq":0.839}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #128).","Replaces the earlier board entry vllm-sr-d3 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 65.45, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.74% on 827 rows (bar 98.5%).","Training method and data undisclosed (author asserts no index items).","Tag v3.1.0; submitted on AMD MI325X/ROCm with the author's ROCm fast path, the card ran the plain CUDA path."],"scores":{"balanced_raw":74.01,"balanced_skill":67.15,"breadth_skill":64.79,"public_skill":65.45},"frozen_scores":{"balanced_raw":74.01,"balanced_skill":65.45,"breadth_skill":64.79},"categories":[{"id":"knowledge","raw":0.6412,"skill":0.5526,"combined":0.4665,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7692,"skill":0.6804,"combined":0.6129,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7612,"skill":0.6686,"combined":0.688,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8456,"skill":0.8257,"combined":0.7977,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6859,"skill":0.5102,"combined":0.5714,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-v310","pub":65.45,"same_skills":62.83,"new_domains":57.86,"v":67.15,"lab":false,"jev":false,"rank_v":19,"rank_pub":34,"name":"vLLM-SR d3 (27B)","eq":{"S":62.5,"O":74.11},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.5,"new_domains":29.64}}},{"engine":"vllm-sr-d3-flash","name":"vLLM-SR d3-flash (9B)","long_name":"vLLM-SR d3-flash (9B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":8392695024,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3-flash","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.737,"conf":0.7821,"ece":0.0559,"brier":0.3467,"over95":0.0119,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.024,"conf":0.0739,"acc":0.1687},{"lo":0.1,"w":0.0309,"conf":0.1416,"acc":0.2184},{"lo":0.2,"w":0.0129,"conf":0.2493,"acc":0.3114},{"lo":0.3,"w":0.0245,"conf":0.354,"acc":0.2948},{"lo":0.4,"w":0.0416,"conf":0.4569,"acc":0.337},{"lo":0.5,"w":0.0911,"conf":0.5502,"acc":0.4629},{"lo":0.6,"w":0.0883,"conf":0.6484,"acc":0.5561},{"lo":0.7,"w":0.0871,"conf":0.7517,"acc":0.6736},{"lo":0.8,"w":0.109,"conf":0.8533,"acc":0.7666},{"lo":0.9,"w":0.4907,"conf":0.9767,"acc":0.9528}],"areas":{"knowledge":{"ece":0.1025,"acc":0.6473,"conf":0.7487},"language":{"ece":0.0156,"acc":0.8499,"conf":0.8635},"retrieval":{"ece":0.0383,"acc":0.7668,"conf":0.8051},"tools":{"ece":0.0261,"acc":0.9293,"conf":0.9447},"arts":{"ece":0.1,"acc":0.5742,"conf":0.5861}}},"gaps":null,"latency":{"median":53.7,"p95":353.2,"mean":113.1},"results":{"1":{"score":0.9616,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":119.8},"2":{"score":0.6511,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1030.3},"3":{"score":0.8484,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":424.2},"4":{"score":0.9271,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":68},"5":{"score":0.9643,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":99.9},"6":{"score":0.7993,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":273.4},"9":{"score":0.5341,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1125},"10":{"score":0.5794,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":90.4},"11":{"score":0.7388,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1545.3},"12":{"score":0.7004,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.7},"20":{"score":0.9414,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.4},"21":{"score":0.6104,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":20.4},"22":{"score":0.3185,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":367.8},"23":{"score":0.6292,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":42.7},"24":{"score":0.8332,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":20},"25":{"score":0.4847,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":22.9},"26":{"score":0.9912,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.9},"27":{"score":0.9625,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":19.1},"28":{"score":0.9219,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":18.7},"29":{"score":0.9701,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":26.7},"30":{"score":0.7278,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1506,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":64.8},"32":{"score":0.6104,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":74.3},"33":{"score":0.1958,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":223.9},"36":{"score":0.4733,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1675.1},"37":{"score":0.537,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":42.8},"38":{"score":0.2651,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":746.5},"39":{"score":0.943,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":38.1},"40":{"score":0.5157,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":27.4},"41":{"score":0.6879,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":25.6},"42":{"score":0.8363,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":41.2},"43":{"score":0.7105,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":23.8},"44":{"score":0.7272,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.1},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":29.7},"50":{"score":0.3735,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":50.9},"56":{"score":0.689,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":154.8},"57":{"score":0.6415,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":25.2},"58":{"score":0.7594,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":24.3},"59":{"score":0.763,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":56.2},"61":{"score":0.8435,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":42.6},"62":{"score":0.8656,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":54.3},"64":{"score":0.803,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":33.3}},"benchmarks":{"1":{"raw":0.9616,"skill":0.9482,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6511,"skill":0.5971,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7008,"private_eq":0.743},"3":{"raw":0.8484,"skill":0.8455,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.9271,"skill":0.9262,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8918,"private_eq":0.8934},"5":{"raw":0.9643,"skill":0.9641,"coverage":1,"random":0.006,"tracks":[],"combined":0.7757,"private_eq":0.7786},"6":{"raw":0.7993,"skill":0.5294,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5341,"skill":0.5341,"coverage":1,"random":0,"tracks":[],"combined":0.5332,"private_eq":0.5375},"10":{"raw":0.5794,"skill":0.3002,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7388,"skill":0.6222,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7004,"skill":0.5512,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5345,"private_eq":0.6921},"20":{"raw":0.9445,"skill":0.889,"coverage":1,"random":0.5,"tracks":[],"combined":0.9432,"private_eq":0.9714},"21":{"raw":0.6104,"skill":0.2207,"coverage":1,"random":0.5,"tracks":[],"combined":0.2365,"private_eq":0.6221},"22":{"raw":0.306,"skill":0.3006,"coverage":1,"random":0.0078,"tracks":[],"combined":0.187,"private_eq":0.2015},"23":{"raw":0.6299,"skill":0.2599,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8332,"skill":0.7776,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4847,"skill":0.3129,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4472,"combined":0.2555},"26":{"raw":0.9912,"skill":0.9883,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9625,"skill":0.95,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9219,"skill":0.8438,"coverage":1,"random":0.5,"tracks":[],"combined":0.5774,"private_eq":0.7906},"29":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7278,"skill":0.6742,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.79,"random":0.25},{"track":"10-choice","raw":0.6657,"random":0.1}],"combined":0.6441,"private_eq":0.7089},"31":{"raw":0.1506,"skill":0.0749,"coverage":1,"random":0.0819,"tracks":[],"combined":0.081,"private_eq":0.165},"32":{"raw":0.6104,"skill":0.3806,"coverage":1,"random":0.371,"tracks":[],"combined":0.2428,"private_eq":0.5285},"33":{"raw":0.1958,"skill":0.1851,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0556,"private_eq":0.0776},"36":{"raw":0.4733,"skill":0.4042,"coverage":1,"random":0.116,"tracks":[],"combined":0.4106,"private_eq":0.484},"37":{"raw":0.537,"skill":0.4192,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4778,"private_eq":0.5876},"38":{"raw":0.2651,"skill":0.2416,"coverage":1,"random":0.031,"tracks":[],"combined":0.3662,"private_eq":0.3919},"39":{"raw":0.943,"skill":0.9162,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4367,"private_eq":0.6207},"40":{"raw":0.5157,"skill":0.3769,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4609,"headline":false},{"track":"A · English","score":0.5157,"headline":true},{"track":"C · Arabic pairs","score":0.885,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"combined":0.2589,"private_eq":0.4297},"41":{"raw":0.6879,"skill":0.5319,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3846,"private_eq":0.5937},"42":{"raw":0.8363,"skill":0.6815,"coverage":1,"random":0.486,"tracks":[],"combined":0.8969,"private_eq":0.9471},"43":{"raw":0.7105,"skill":0.5407,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4952,"private_eq":0.6848},"44":{"raw":0.7272,"skill":0.4544,"coverage":1,"random":0.5,"tracks":[],"combined":0.4977,"private_eq":0.7512},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3735,"skill":0.0908,"coverage":1,"random":0.311,"tracks":[],"combined":0.1596,"private_eq":0.4269},"56":{"raw":0.689,"skill":0.378,"coverage":1,"random":0.5,"tracks":[],"combined":0.4732,"private_eq":0.7391},"57":{"raw":0.6415,"skill":0.5968,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4531,"combined":0.3788},"58":{"raw":0.7594,"skill":0.6513,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4646,"combined":0.216},"59":{"raw":0.763,"skill":0.5086,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3678,"private_eq":0.6981},"61":{"raw":0.8435,"skill":0.687,"coverage":1,"random":0.5,"tracks":[],"combined":0.6891,"private_eq":0.8458},"62":{"raw":0.8656,"skill":0.8208,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.803,"skill":0.7537,"coverage":1,"random":0.2,"tracks":[],"combined":0.7198,"private_eq":0.7776}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #128).","Submitted public 0.3 run adopted (kit rescore 57.79, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.69% on 827 rows (bar 98.5%).","Training method and data undisclosed (author asserts no index items).","Tag v3.1.0; submitted on AMD MI325X/ROCm with the author's ROCm fast path, the card ran the plain CUDA path."],"scores":{"balanced_raw":68.14,"balanced_skill":54.27,"breadth_skill":56.67,"public_skill":57.79},"frozen_scores":{"balanced_raw":68.14,"balanced_skill":57.79,"breadth_skill":56.67},"categories":[{"id":"knowledge","raw":0.5523,"skill":0.433,"combined":0.3453,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7186,"skill":0.6056,"combined":0.5421,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7435,"skill":0.6421,"combined":0.6263,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.782,"skill":0.7601,"combined":0.7424,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6112,"skill":0.4191,"combined":0.4492,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-flash","pub":57.79,"same_skills":54.63,"new_domains":41.87,"v":54.27,"lab":false,"jev":false,"rank_v":81,"rank_pub":82,"name":"vLLM-SR d3-flash (9B)","eq":{"S":54.21,"O":54.36},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":32.52,"new_domains":21.74}}},{"engine":"vllm-sr-d3-mini","name":"vLLM-SR d3-mini (4B)","long_name":"vLLM-SR d3-mini (4B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4539265536,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3-mini","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7257,"conf":0.7759,"ece":0.0561,"brier":0.3657,"over95":0.0145,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0224,"conf":0.078,"acc":0.1563},{"lo":0.1,"w":0.0284,"conf":0.1394,"acc":0.1819},{"lo":0.2,"w":0.0142,"conf":0.2483,"acc":0.175},{"lo":0.3,"w":0.0297,"conf":0.3555,"acc":0.308},{"lo":0.4,"w":0.0443,"conf":0.4544,"acc":0.3353},{"lo":0.5,"w":0.0948,"conf":0.549,"acc":0.4795},{"lo":0.6,"w":0.0874,"conf":0.6529,"acc":0.5597},{"lo":0.7,"w":0.095,"conf":0.7507,"acc":0.6801},{"lo":0.8,"w":0.115,"conf":0.8525,"acc":0.7866},{"lo":0.9,"w":0.4687,"conf":0.9761,"acc":0.9411}],"areas":{"knowledge":{"ece":0.1021,"acc":0.6386,"conf":0.7394},"language":{"ece":0.0161,"acc":0.8502,"conf":0.8564},"retrieval":{"ece":0.0753,"acc":0.7403,"conf":0.8153},"tools":{"ece":0.0307,"acc":0.9208,"conf":0.9495},"arts":{"ece":0.0629,"acc":0.5473,"conf":0.5685}}},"gaps":null,"latency":{"median":50.4,"p95":237.1,"mean":87.2},"results":{"1":{"score":0.9551,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":68.1},"2":{"score":0.6442,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":609},"3":{"score":0.8386,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":242.3},"4":{"score":0.9275,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":52.4},"5":{"score":0.9628,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":65.2},"6":{"score":0.7785,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":168.9},"9":{"score":0.375,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":675.1},"10":{"score":0.4727,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":61.7},"11":{"score":0.7469,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":878.9},"12":{"score":0.6788,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":18.6},"20":{"score":0.8946,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.3},"21":{"score":0.5974,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":18.5},"22":{"score":0.217,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":187.5},"23":{"score":0.6182,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.7},"24":{"score":0.801,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":18},"25":{"score":0.449,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":18.2},"26":{"score":0.9886,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":17.8},"27":{"score":0.9548,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":17.8},"28":{"score":0.9021,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":17.7},"29":{"score":0.9599,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":18.6},"30":{"score":0.696,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1412,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40},"32":{"score":0.6077,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":44.4},"33":{"score":0.1897,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":139.5},"36":{"score":0.4509,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":945.8},"37":{"score":0.5197,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.9},"38":{"score":0.1738,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":446.4},"39":{"score":0.9352,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":27.1},"40":{"score":0.5022,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":22.2},"41":{"score":0.7055,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":18.6},"42":{"score":0.8256,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":26},"43":{"score":0.6456,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":18.2},"44":{"score":0.7282,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.1},"45":{"score":0.1078,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":19.6},"50":{"score":0.4117,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":33.1},"56":{"score":0.7235,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":86.3},"57":{"score":0.5887,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":18.6},"58":{"score":0.7145,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":18.3},"59":{"score":0.7155,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.6},"61":{"score":0.8153,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":25.8},"62":{"score":0.8453,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":33.6},"64":{"score":0.7614,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":19.1}},"benchmarks":{"1":{"raw":0.9551,"skill":0.9394,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8989,"private_eq":0.9252},"2":{"raw":0.6442,"skill":0.5891,"coverage":1,"random":0.1341,"tracks":[],"combined":0.695,"private_eq":0.738},"3":{"raw":0.8386,"skill":0.8355,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8487,"private_eq":0.8523},"4":{"raw":0.9275,"skill":0.9266,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8427,"private_eq":0.8455},"5":{"raw":0.9628,"skill":0.9626,"coverage":1,"random":0.006,"tracks":[],"combined":0.8035,"private_eq":0.8059},"6":{"raw":0.7785,"skill":0.4807,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.375,"skill":0.375,"coverage":1,"random":0,"tracks":[],"combined":0.427,"private_eq":0.4325},"10":{"raw":0.4727,"skill":0.1226,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7469,"skill":0.634,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6788,"skill":0.5189,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4918,"private_eq":0.6639},"20":{"raw":0.8974,"skill":0.7948,"coverage":1,"random":0.5,"tracks":[],"combined":0.8782,"private_eq":0.9393},"21":{"raw":0.5974,"skill":0.1948,"coverage":1,"random":0.5,"tracks":[],"combined":0.1258,"private_eq":0.5674},"22":{"raw":0.2128,"skill":0.2067,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2371,"private_eq":0.2507},"23":{"raw":0.6078,"skill":0.2156,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.801,"skill":0.7347,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.449,"skill":0.2653,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4088,"combined":0.2037},"26":{"raw":0.9886,"skill":0.9848,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9548,"skill":0.9397,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9021,"skill":0.8042,"coverage":1,"random":0.5,"tracks":[],"combined":0.5721,"private_eq":0.788},"29":{"raw":0.9599,"skill":0.9465,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.696,"skill":0.6355,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7597,"random":0.25},{"track":"10-choice","raw":0.6323,"random":0.1}],"combined":0.6341,"private_eq":0.7007},"31":{"raw":0.1412,"skill":0.0646,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0842,"private_eq":0.1679},"32":{"raw":0.6077,"skill":0.3763,"coverage":1,"random":0.371,"tracks":[],"combined":0.2042,"private_eq":0.5045},"33":{"raw":0.1897,"skill":0.1789,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2098,"private_eq":0.228},"36":{"raw":0.4509,"skill":0.3788,"coverage":1,"random":0.116,"tracks":[],"combined":0.4256,"private_eq":0.4971},"37":{"raw":0.5197,"skill":0.3976,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5166,"private_eq":0.6182},"38":{"raw":0.1738,"skill":0.1474,"coverage":1,"random":0.031,"tracks":[],"combined":0.2679,"private_eq":0.2977},"39":{"raw":0.9352,"skill":0.9047,"coverage":1,"random":0.3201,"tracks":[],"combined":0.448,"private_eq":0.6283},"40":{"raw":0.5022,"skill":0.3596,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4775,"headline":false},{"track":"A · English","score":0.5022,"headline":true},{"track":"C · Arabic pairs","score":0.74,"headline":false},{"track":"C · English pairs","score":0.94,"headline":false}],"combined":0.3137,"private_eq":0.4718},"41":{"raw":0.7055,"skill":0.5583,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1614,"private_eq":0.4466},"42":{"raw":0.8256,"skill":0.6607,"coverage":1,"random":0.486,"tracks":[],"combined":0.8863,"private_eq":0.9417},"43":{"raw":0.6456,"skill":0.4377,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3864,"private_eq":0.617},"44":{"raw":0.7282,"skill":0.4564,"coverage":1,"random":0.5,"tracks":[],"combined":0.4603,"private_eq":0.7327},"45":{"raw":0.1078,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4117,"skill":0.1462,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.7235,"skill":0.447,"coverage":1,"random":0.5,"tracks":[],"combined":0.4299,"private_eq":0.7177},"57":{"raw":0.5887,"skill":0.5374,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4639,"combined":0.3911},"58":{"raw":0.7145,"skill":0.5862,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.43,"combined":0.1653},"59":{"raw":0.7155,"skill":0.4101,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3014,"private_eq":0.6664},"61":{"raw":0.8153,"skill":0.6306,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.8453,"skill":0.7937,"coverage":1,"random":0.25,"tracks":[],"combined":0.5698,"private_eq":0.6803},"64":{"raw":0.7614,"skill":0.7017,"coverage":1,"random":0.2,"tracks":[],"combined":0.6907,"private_eq":0.7546}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #128).","Submitted public 0.3 run adopted (kit rescore 54.90, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.43% on 827 rows (bar 98.5%).","Training method and data undisclosed (author asserts no index items).","Tag v3.1.0; submitted on AMD MI325X/ROCm with the author's ROCm fast path, the card ran the plain CUDA path."],"scores":{"balanced_raw":65.87,"balanced_skill":49.97,"breadth_skill":53.64,"public_skill":54.9},"frozen_scores":{"balanced_raw":65.87,"balanced_skill":54.9,"breadth_skill":53.64},"categories":[{"id":"knowledge","raw":0.5269,"skill":0.3984,"combined":0.3312,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6991,"skill":0.578,"combined":0.4964,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7376,"skill":0.6359,"combined":0.615,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7439,"skill":0.7199,"combined":0.7016,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5814,"skill":0.3766,"combined":0.4241,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-mini","pub":54.9,"same_skills":51.9,"new_domains":36.52,"v":49.97,"lab":false,"jev":false,"rank_v":93,"rank_pub":100,"name":"vLLM-SR d3-mini (4B)","eq":{"S":51.45,"O":47.75},"rank_tie":21,"tied":true,"parts":{"public":0,"same_skills":30.87,"new_domains":19.1}}},{"engine":"vllm-sr-d3-nano","name":"vLLM-SR d3-nano (2B)","long_name":"vLLM-SR d3-nano (2B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2213241664,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3-nano","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6542,"conf":0.764,"ece":0.112,"brier":0.4525,"over95":0.0265,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0482,"conf":0.0634,"acc":0.0856},{"lo":0.1,"w":0.0145,"conf":0.1313,"acc":0.1289},{"lo":0.2,"w":0.0072,"conf":0.2563,"acc":0.1741},{"lo":0.3,"w":0.0216,"conf":0.3581,"acc":0.1899},{"lo":0.4,"w":0.0421,"conf":0.4546,"acc":0.3563},{"lo":0.5,"w":0.1099,"conf":0.5518,"acc":0.4004},{"lo":0.6,"w":0.0979,"conf":0.6482,"acc":0.5082},{"lo":0.7,"w":0.098,"conf":0.7514,"acc":0.5992},{"lo":0.8,"w":0.1115,"conf":0.8525,"acc":0.6867},{"lo":0.9,"w":0.4489,"conf":0.9745,"acc":0.8882}],"areas":{"knowledge":{"ece":0.1669,"acc":0.5521,"conf":0.719},"language":{"ece":0.0648,"acc":0.7761,"conf":0.8409},"retrieval":{"ece":0.0931,"acc":0.7745,"conf":0.8668},"tools":{"ece":0.1019,"acc":0.8065,"conf":0.9032},"arts":{"ece":0.1183,"acc":0.4511,"conf":0.5497}}},"gaps":null,"latency":{"median":38,"p95":101.1,"mean":52.6},"results":{"1":{"score":0.9215,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":35.5},"2":{"score":0.6124,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":265.6},"3":{"score":0.563,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":115.5},"4":{"score":0.9133,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":20.8},"5":{"score":0.9511,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":28.3},"6":{"score":0.7903,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":72},"9":{"score":0.125,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":283},"10":{"score":0.6317,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":25.2},"11":{"score":0.6885,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":390.7},"12":{"score":0.5596,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":13.7},"20":{"score":0.5658,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.5},"21":{"score":0.6054,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.5},"22":{"score":0.1095,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":102.8},"23":{"score":0.6108,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.3},"24":{"score":0.6712,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13.3},"25":{"score":0.3214,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":13.6},"26":{"score":0.9642,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":13.3},"27":{"score":0.8882,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":13.3},"28":{"score":0.7987,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":13.2},"29":{"score":0.9096,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":13.6},"30":{"score":0.5178,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0874,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.1},"32":{"score":0.5957,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":22.4},"33":{"score":0.2976,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":62.4},"36":{"score":0.3902,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":417.1},"37":{"score":0.4714,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.8},"38":{"score":0.1295,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":203.1},"39":{"score":0.8757,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":14.1},"40":{"score":0.4006,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":13.5},"41":{"score":0.5703,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":13.5},"42":{"score":0.7159,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.5},"43":{"score":0.4175,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":13.4},"44":{"score":0.6034,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.4},"45":{"score":0.1397,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":13.8},"50":{"score":0.3669,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":17.2},"56":{"score":0.67,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":47},"57":{"score":0.4352,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":13.5},"58":{"score":0.5439,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.4},"59":{"score":0.295,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":18.1},"61":{"score":0.799,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":14.3},"62":{"score":0.8234,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":17.5},"64":{"score":0.7064,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":13.6}},"benchmarks":{"1":{"raw":0.9215,"skill":0.894,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8581,"private_eq":0.8953},"2":{"raw":0.6124,"skill":0.5524,"coverage":1,"random":0.1341,"tracks":[],"combined":0.683,"private_eq":0.7278},"3":{"raw":0.563,"skill":0.5546,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5758,"private_eq":0.5876},"4":{"raw":0.9133,"skill":0.9122,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8311,"private_eq":0.8341},"5":{"raw":0.9511,"skill":0.9508,"coverage":1,"random":0.006,"tracks":[],"combined":0.7504,"private_eq":0.7537},"6":{"raw":0.7903,"skill":0.5083,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1,"random":0,"tracks":[],"combined":0.1564,"private_eq":0.165},"10":{"raw":0.6317,"skill":0.3872,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6885,"skill":0.5496,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5596,"skill":0.3403,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4269,"private_eq":0.6211},"20":{"raw":0.5718,"skill":0.1435,"coverage":1,"random":0.5,"tracks":[],"combined":0.3431,"private_eq":0.6748},"21":{"raw":0.6054,"skill":0.2108,"coverage":1,"random":0.5,"tracks":[],"combined":0.1962,"private_eq":0.6022},"22":{"raw":0.1017,"skill":0.0947,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1051,"private_eq":0.1212},"23":{"raw":0.5916,"skill":0.1833,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6712,"skill":0.5616,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3319,"combined":0.0999},"26":{"raw":0.9642,"skill":0.9523,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8882,"skill":0.8509,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7987,"skill":0.5974,"coverage":1,"random":0.5,"tracks":[],"combined":0.3805,"private_eq":0.6933},"29":{"raw":0.9096,"skill":0.8795,"coverage":1,"random":0.25,"tracks":[],"combined":0.7653,"private_eq":0.8252},"30":{"raw":0.5178,"skill":0.4218,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6179,"random":0.25},{"track":"10-choice","raw":0.4177,"random":0.1}],"combined":0.5166,"private_eq":0.6049},"31":{"raw":0.0874,"skill":0.006,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0019,"private_eq":0.0932},"32":{"raw":0.5957,"skill":0.3572,"coverage":1,"random":0.371,"tracks":[],"combined":0.1112,"private_eq":0.4467},"33":{"raw":0.2976,"skill":0.2883,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3433,"private_eq":0.3583},"36":{"raw":0.3902,"skill":0.3101,"coverage":1,"random":0.116,"tracks":[],"combined":0.31,"private_eq":0.3961},"37":{"raw":0.4714,"skill":0.337,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4763,"private_eq":0.5864},"38":{"raw":0.1295,"skill":0.1017,"coverage":1,"random":0.031,"tracks":[],"combined":0.2225,"private_eq":0.2542},"39":{"raw":0.8757,"skill":0.8172,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4964,"private_eq":0.6608},"40":{"raw":0.4006,"skill":0.2288,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4072,"headline":false},{"track":"A · English","score":0.4006,"headline":true},{"track":"C · Arabic pairs","score":0.8,"headline":false},{"track":"C · English pairs","score":0.875,"headline":false}],"combined":0.1438,"private_eq":0.3413},"41":{"raw":0.5703,"skill":0.3555,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2081,"private_eq":0.4774},"42":{"raw":0.7159,"skill":0.4473,"coverage":1,"random":0.486,"tracks":[],"combined":0.8501,"private_eq":0.9233},"43":{"raw":0.4175,"skill":0.0759,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2819,"private_eq":0.5519},"44":{"raw":0.6034,"skill":0.2068,"coverage":1,"random":0.5,"tracks":[],"combined":0.2138,"private_eq":0.6109},"45":{"raw":0.1397,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3669,"skill":0.0812,"coverage":1,"random":0.311,"tracks":[],"combined":0.174,"private_eq":0.4367},"56":{"raw":0.67,"skill":0.34,"coverage":1,"random":0.5,"tracks":[],"combined":0.404,"private_eq":0.7049},"57":{"raw":0.4352,"skill":0.3648,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3428,"combined":0.2533},"58":{"raw":0.5439,"skill":0.3389,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3639,"combined":0.0684},"59":{"raw":0.295,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.799,"skill":0.598,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"62":{"raw":0.8234,"skill":0.7645,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.7064,"skill":0.633,"coverage":1,"random":0.2,"tracks":[],"combined":0.6326,"private_eq":0.7086}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #128).","Submitted public 0.3 run adopted (kit rescore 42.22, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.48% on 827 rows (bar 98.5%).","Training method and data undisclosed (author asserts no index items).","Tag v3.1.0; submitted on AMD MI325X/ROCm with the author's ROCm fast path, the card ran the plain CUDA path."],"scores":{"balanced_raw":56.48,"balanced_skill":34.3,"breadth_skill":40.05,"public_skill":42.22},"frozen_scores":{"balanced_raw":56.48,"balanced_skill":42.22,"breadth_skill":40.05},"categories":[{"id":"knowledge","raw":0.4338,"skill":0.2526,"combined":0.2251,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5786,"skill":0.4217,"combined":0.3977,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7039,"skill":0.5883,"combined":0.5739,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6189,"skill":0.5889,"combined":0.5821,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4906,"skill":0.2244,"combined":0.2902,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-nano","pub":42.22,"same_skills":42.47,"new_domains":16.4,"v":34.3,"lab":false,"jev":false,"rank_v":154,"rank_pub":145,"name":"vLLM-SR d3-nano (2B)","eq":{"S":41.91,"O":22.89},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":25.15,"new_domains":9.16}}},{"engine":"vllm-sr-d3-lite","name":"vLLM-SR d3-lite (0.8B)","long_name":"vLLM-SR d3-lite (0.8B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":852985920,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3-lite","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6137,"conf":0.7045,"ece":0.0943,"brier":0.5076,"over95":0.0218,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0566,"conf":0.0534,"acc":0.0386},{"lo":0.1,"w":0.0081,"conf":0.145,"acc":0.1088},{"lo":0.2,"w":0.0096,"conf":0.2527,"acc":0.2419},{"lo":0.3,"w":0.0445,"conf":0.3606,"acc":0.3996},{"lo":0.4,"w":0.0676,"conf":0.4519,"acc":0.352},{"lo":0.5,"w":0.1406,"conf":0.5502,"acc":0.512},{"lo":0.6,"w":0.1192,"conf":0.6486,"acc":0.576},{"lo":0.7,"w":0.1106,"conf":0.7475,"acc":0.6025},{"lo":0.8,"w":0.1285,"conf":0.8532,"acc":0.6725},{"lo":0.9,"w":0.3147,"conf":0.9671,"acc":0.8677}],"areas":{"knowledge":{"ece":0.1322,"acc":0.5337,"conf":0.6505},"language":{"ece":0.098,"acc":0.6734,"conf":0.7694},"retrieval":{"ece":0.0592,"acc":0.7623,"conf":0.8205},"tools":{"ece":0.0977,"acc":0.7527,"conf":0.8365},"arts":{"ece":0.0633,"acc":0.4643,"conf":0.5239}}},"gaps":null,"latency":{"median":37,"p95":77.1,"mean":47.5},"results":{"1":{"score":0.8601,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":22.8},"2":{"score":0.6143,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":164.6},"3":{"score":0.5689,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":76.1},"4":{"score":0.9029,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":14.9},"5":{"score":0.9395,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":19.6},"6":{"score":0.7452,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":45.9},"9":{"score":0.1136,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":176.5},"10":{"score":0.6427,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":17.3},"11":{"score":0.5423,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":256.1},"12":{"score":0.4934,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":13.7},"20":{"score":0.7132,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.5},"21":{"score":0.5734,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.4},"22":{"score":0.016,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":66.9},"23":{"score":0.5866,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14},"24":{"score":0.5622,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13.4},"25":{"score":0.3316,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":13.6},"26":{"score":0.9036,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":13.3},"27":{"score":0.7756,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":13.3},"28":{"score":0.7088,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":13.3},"29":{"score":0.8268,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":13.6},"30":{"score":0.4162,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0834,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.4},"32":{"score":0.5785,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":15.7},"33":{"score":0.1964,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":39.8},"36":{"score":0.3674,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":240.1},"37":{"score":0.4128,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.2},"38":{"score":0.0645,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":131.8},"39":{"score":0.8809,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":13.8},"40":{"score":0.3292,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":13.4},"41":{"score":0.5542,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":13.4},"42":{"score":0.6607,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14},"43":{"score":0.3368,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":13.3},"44":{"score":0.6106,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.4},"45":{"score":0.1437,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":13.6},"50":{"score":0.3795,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":14.4},"56":{"score":0.5485,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":33.1},"57":{"score":0.3246,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":13.5},"58":{"score":0.4656,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.4},"59":{"score":0.2971,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":14.6},"61":{"score":0.8177,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":14},"62":{"score":0.796,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":14.3},"64":{"score":0.6155,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":13.6}},"benchmarks":{"1":{"raw":0.8601,"skill":0.8111,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5174,"private_eq":0.6458},"2":{"raw":0.6143,"skill":0.5546,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6765,"private_eq":0.7222},"3":{"raw":0.5689,"skill":0.5606,"coverage":1,"random":0.0189,"tracks":[],"combined":0.345,"private_eq":0.3637},"4":{"raw":0.9029,"skill":0.9017,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7916,"private_eq":0.7956},"5":{"raw":0.9395,"skill":0.9391,"coverage":1,"random":0.006,"tracks":[],"combined":0.7412,"private_eq":0.7447},"6":{"raw":0.7452,"skill":0.4026,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1136,"skill":0.1136,"coverage":1,"random":0,"tracks":[],"combined":0.0148,"private_eq":0.025},"10":{"raw":0.6427,"skill":0.4055,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5423,"skill":0.3381,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4934,"skill":0.2412,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4346,"private_eq":0.6262},"20":{"raw":0.7162,"skill":0.4324,"coverage":1,"random":0.5,"tracks":[],"combined":0.4226,"private_eq":0.7141},"21":{"raw":0.5734,"skill":0.1469,"coverage":1,"random":0.5,"tracks":[],"combined":0.0051,"private_eq":0.5077},"22":{"raw":0.0171,"skill":0.0094,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.564,"skill":0.1281,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5622,"skill":0.4163,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3316,"skill":0.1088,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3059,"combined":0.0649},"26":{"raw":0.9036,"skill":0.8714,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7756,"skill":0.7007,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7088,"skill":0.4176,"coverage":1,"random":0.5,"tracks":[],"combined":0.2492,"private_eq":0.6284},"29":{"raw":0.8268,"skill":0.7691,"coverage":1,"random":0.25,"tracks":[],"combined":0.5029,"private_eq":0.6307},"30":{"raw":0.4162,"skill":0.2989,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5277,"random":0.25},{"track":"10-choice","raw":0.3048,"random":0.1}],"combined":0.3328,"private_eq":0.455},"31":{"raw":0.0834,"skill":0.0017,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0074,"private_eq":0.0982},"32":{"raw":0.5785,"skill":0.3299,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1964,"skill":0.1857,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0146,"private_eq":0.0376},"36":{"raw":0.3674,"skill":0.2844,"coverage":1,"random":0.116,"tracks":[],"combined":0.268,"private_eq":0.3594},"37":{"raw":0.4128,"skill":0.2635,"coverage":1,"random":0.2027,"tracks":[],"combined":0.413,"private_eq":0.5365},"38":{"raw":0.0645,"skill":0.0346,"coverage":1,"random":0.031,"tracks":[],"combined":0.0691,"private_eq":0.1073},"39":{"raw":0.8809,"skill":0.8248,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3745,"private_eq":0.5789},"40":{"raw":0.3292,"skill":0.137,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3729,"headline":false},{"track":"A · English","score":0.3292,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.83,"headline":false}],"combined":0.1103,"private_eq":0.3155},"41":{"raw":0.5542,"skill":0.3313,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3882,"private_eq":0.5961},"42":{"raw":0.6607,"skill":0.3399,"coverage":1,"random":0.486,"tracks":[],"combined":0.6816,"private_eq":0.8377},"43":{"raw":0.3368,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1941,"private_eq":0.4972},"44":{"raw":0.6106,"skill":0.2212,"coverage":1,"random":0.5,"tracks":[],"combined":0.3049,"private_eq":0.6559},"45":{"raw":0.1437,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3795,"skill":0.0994,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.5485,"skill":0.097,"coverage":1,"random":0.5,"tracks":[],"combined":0.4991,"private_eq":0.7519},"57":{"raw":0.3246,"skill":0.2404,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2627,"combined":0.1622},"58":{"raw":0.4656,"skill":0.2254,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3367,"combined":0.0285},"59":{"raw":0.2971,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8177,"skill":0.6354,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.796,"skill":0.728,"coverage":1,"random":0.25,"tracks":[],"combined":0.6108,"private_eq":0.7107},"64":{"raw":0.6155,"skill":0.5194,"coverage":1,"random":0.2,"tracks":[],"combined":0.5355,"private_eq":0.6318}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #128).","Submitted public 0.3 run adopted (kit rescore 36.93, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.91% on 827 rows (bar 98.5%).","Training method and data undisclosed (author asserts no index items).","Tag v3.1.0; submitted on AMD MI325X/ROCm with the author's ROCm fast path, the card ran the plain CUDA path."],"scores":{"balanced_raw":52.58,"balanced_skill":26.16,"breadth_skill":34.41,"public_skill":36.93},"frozen_scores":{"balanced_raw":52.58,"balanced_skill":36.93,"breadth_skill":34.41},"categories":[{"id":"knowledge","raw":0.3826,"skill":0.1857,"combined":0.1342,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5227,"skill":0.3423,"combined":0.326,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6713,"skill":0.5373,"combined":0.5631,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5998,"skill":0.5634,"combined":0.4328,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4776,"skill":0.2226,"combined":0.2312,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-lite","pub":36.93,"same_skills":34.81,"new_domains":9.33,"v":26.16,"lab":false,"jev":false,"rank_v":178,"rank_pub":172,"name":"vLLM-SR d3-lite (0.8B)","eq":{"S":34.16,"O":14.17},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":20.49,"new_domains":5.67}}},{"engine":"vllm-sr-d3-edge","name":"vLLM-SR d3-edge (0.6B)","long_name":"vLLM-SR d3-edge (0.6B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":594321600,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3-edge","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5453,"conf":0.6861,"ece":0.1409,"brier":0.5853,"over95":0.0261,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0497,"conf":0.0554,"acc":0.025},{"lo":0.1,"w":0.0136,"conf":0.1387,"acc":0.0917},{"lo":0.2,"w":0.0122,"conf":0.2617,"acc":0.149},{"lo":0.3,"w":0.0382,"conf":0.3582,"acc":0.2649},{"lo":0.4,"w":0.0833,"conf":0.4522,"acc":0.3304},{"lo":0.5,"w":0.1656,"conf":0.5489,"acc":0.4552},{"lo":0.6,"w":0.1224,"conf":0.6475,"acc":0.5213},{"lo":0.7,"w":0.1157,"conf":0.7483,"acc":0.5481},{"lo":0.8,"w":0.1303,"conf":0.8523,"acc":0.6112},{"lo":0.9,"w":0.269,"conf":0.9636,"acc":0.822}],"areas":{"knowledge":{"ece":0.1617,"acc":0.4507,"conf":0.6124},"language":{"ece":0.1122,"acc":0.6305,"conf":0.7427},"retrieval":{"ece":0.067,"acc":0.7452,"conf":0.8101},"tools":{"ece":0.2392,"acc":0.6066,"conf":0.8445},"arts":{"ece":0.1534,"acc":0.3841,"conf":0.5376}}},"gaps":null,"latency":{"median":19.2,"p95":45.3,"mean":25.7},"results":{"1":{"score":0.7249,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":12.4},"2":{"score":0.5701,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":85.9},"3":{"score":0.4016,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":43.6},"4":{"score":0.8997,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":8.3},"5":{"score":0.9191,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":10.8},"6":{"score":0.5162,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":24.8},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":91.7},"10":{"score":0.5127,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":9.4},"11":{"score":0.5649,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":148.1},"12":{"score":0.4078,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":6.5},"20":{"score":0.549,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.5},"21":{"score":0.5335,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":6.5},"22":{"score":0.0115,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":37.8},"23":{"score":0.5206,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.8},"24":{"score":0.412,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":6.4},"25":{"score":0.2806,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":6.6},"26":{"score":0.7458,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6.3},"27":{"score":0.5802,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":6.3},"28":{"score":0.618,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":6.3},"29":{"score":0.5895,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":6.6},"30":{"score":0.3495,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0742,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.9},"32":{"score":0.4987,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":8.8},"33":{"score":0.1139,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":23},"36":{"score":0.3115,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":136},"37":{"score":0.2955,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7},"38":{"score":0.0778,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":70.1},"39":{"score":0.8051,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":6.9},"40":{"score":0.3173,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":6.4},"41":{"score":0.4529,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":6.5},"42":{"score":0.5671,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":7},"43":{"score":0.4316,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":6.4},"44":{"score":0.5406,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.4},"45":{"score":0.1677,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":6.6},"50":{"score":0.3407,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":7.8},"56":{"score":0.6785,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":17.8},"57":{"score":0.2311,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":6.5},"58":{"score":0.3831,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":6.4},"59":{"score":0.3569,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":7.7},"61":{"score":0.8103,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":7},"62":{"score":0.6837,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":7.5},"64":{"score":0.5663,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":6.5}},"benchmarks":{"1":{"raw":0.7249,"skill":0.6286,"coverage":1,"random":0.2592,"tracks":[],"combined":0.3744,"private_eq":0.5411},"2":{"raw":0.5701,"skill":0.5036,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6452,"private_eq":0.6954},"3":{"raw":0.4016,"skill":0.3901,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2243,"private_eq":0.2466},"4":{"raw":0.8997,"skill":0.8984,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6917,"private_eq":0.6981},"5":{"raw":0.9191,"skill":0.9186,"coverage":1,"random":0.006,"tracks":[],"combined":0.6328,"private_eq":0.6382},"6":{"raw":0.5162,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.5127,"skill":0.1892,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5649,"skill":0.3708,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4078,"skill":0.1129,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2915,"private_eq":0.5317},"20":{"raw":0.5443,"skill":0.0886,"coverage":1,"random":0.5,"tracks":[],"combined":0.1262,"private_eq":0.5676},"21":{"raw":0.5335,"skill":0.067,"coverage":1,"random":0.5,"tracks":[],"combined":0.0856,"private_eq":0.5475},"22":{"raw":0.0114,"skill":0.0037,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.525,"skill":0.0501,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.412,"skill":0.216,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2806,"skill":0.0408,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2914,"combined":0.0453},"26":{"raw":0.7458,"skill":0.661,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5802,"skill":0.4401,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.618,"skill":0.236,"coverage":1,"random":0.5,"tracks":[],"combined":0.1384,"private_eq":0.5736},"29":{"raw":0.5895,"skill":0.4527,"coverage":1,"random":0.25,"tracks":[],"combined":0.2703,"private_eq":0.4582},"30":{"raw":0.3495,"skill":0.2159,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4481,"random":0.25},{"track":"10-choice","raw":0.2509,"random":0.1}],"combined":0.2031,"private_eq":0.3492},"31":{"raw":0.0742,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.4987,"skill":0.203,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1139,"skill":0.1021,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0351,"private_eq":0.0576},"36":{"raw":0.3115,"skill":0.2212,"coverage":1,"random":0.116,"tracks":[],"combined":0.1844,"private_eq":0.2863},"37":{"raw":0.2955,"skill":0.1163,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3506,"private_eq":0.4873},"38":{"raw":0.0778,"skill":0.0483,"coverage":1,"random":0.031,"tracks":[],"combined":0.081,"private_eq":0.1187},"39":{"raw":0.8051,"skill":0.7133,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4181,"private_eq":0.6082},"40":{"raw":0.3173,"skill":0.1217,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3358,"headline":false},{"track":"A · English","score":0.3173,"headline":true},{"track":"C · Arabic pairs","score":0.61,"headline":false},{"track":"C · English pairs","score":0.79,"headline":false}],"combined":0.0588,"private_eq":0.276},"41":{"raw":0.4529,"skill":0.1793,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5671,"skill":0.1578,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4316,"skill":0.0981,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.5406,"skill":0.0812,"coverage":1,"random":0.5,"tracks":[],"combined":0.1712,"private_eq":0.5898},"45":{"raw":0.1677,"skill":0.0043,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3407,"skill":0.0431,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.6785,"skill":0.357,"coverage":1,"random":0.5,"tracks":[],"combined":0.3866,"private_eq":0.6963},"57":{"raw":0.2311,"skill":0.1352,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2217,"combined":0.1155},"58":{"raw":0.3831,"skill":0.1058,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3286,"combined":0.0166},"59":{"raw":0.3569,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8103,"skill":0.6206,"coverage":1,"random":0.5,"tracks":[],"combined":0.5642,"private_eq":0.7841},"62":{"raw":0.6837,"skill":0.5783,"coverage":1,"random":0.25,"tracks":[],"combined":0.4056,"private_eq":0.5585},"64":{"raw":0.5663,"skill":0.4579,"coverage":1,"random":0.2,"tracks":[],"combined":0.5452,"private_eq":0.6395}},"coverage":1,"pending":0,"notes":["New in the 2026-10-10 PR batch (PR #128).","Submitted public 0.3 run adopted (kit rescore 28.82, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.58% on 827 rows (bar 98.5%).","Training method and data undisclosed (author asserts no index items).","Tag v3.1.0; submitted on AMD MI325X/ROCm with the author's ROCm fast path, the card ran the plain CUDA path."],"scores":{"balanced_raw":46.79,"balanced_skill":17.69,"breadth_skill":25.44,"public_skill":28.82},"frozen_scores":{"balanced_raw":46.79,"balanced_skill":28.82,"breadth_skill":25.44},"categories":[{"id":"knowledge","raw":0.3354,"skill":0.1106,"combined":0.0701,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4616,"skill":0.2415,"combined":0.1429,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6577,"skill":0.5363,"combined":0.4715,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4825,"skill":0.4267,"combined":0.3257,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4202,"skill":0.1184,"combined":0.1914,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-edge","pub":28.82,"same_skills":24.34,"new_domains":5.03,"v":17.69,"lab":false,"jev":false,"rank_v":204,"rank_pub":194,"name":"vLLM-SR d3-edge (0.6B)","eq":{"S":23.57,"O":8.86},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":14.14,"new_domains":3.54}}},{"engine":"wald-26b-a4b","name":"Wald-26B-A4B","long_name":"Wald-26B-A4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/org2ai/Wald-26B-A4B","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"org2ai","name":"ORG2 AI","logo":"data/orgs/org2ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7468,"conf":0.79,"ece":0.0432,"brier":0.3334,"over95":0.0103,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0015,"conf":0.0797,"acc":0.0518},{"lo":0.1,"w":0.0137,"conf":0.1504,"acc":0.1089},{"lo":0.2,"w":0.0198,"conf":0.2538,"acc":0.181},{"lo":0.3,"w":0.035,"conf":0.3497,"acc":0.2505},{"lo":0.4,"w":0.0508,"conf":0.4556,"acc":0.4091},{"lo":0.5,"w":0.1035,"conf":0.5519,"acc":0.4748},{"lo":0.6,"w":0.0876,"conf":0.6505,"acc":0.5594},{"lo":0.7,"w":0.0982,"conf":0.7534,"acc":0.7039},{"lo":0.8,"w":0.1276,"conf":0.8555,"acc":0.8117},{"lo":0.9,"w":0.4623,"conf":0.9738,"acc":0.9546}],"areas":{"knowledge":{"ece":0.0748,"acc":0.6558,"conf":0.7307},"language":{"ece":0.0171,"acc":0.8415,"conf":0.8382},"retrieval":{"ece":0.0873,"acc":0.7028,"conf":0.7901},"tools":{"ece":0.0375,"acc":0.9439,"conf":0.9677},"arts":{"ece":0.04,"acc":0.6753,"conf":0.7153}}},"gaps":null,"latency":{"median":90,"p95":339.9,"mean":167.8},"results":{"1":{"score":0.961,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6697,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8504,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8642,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.8693,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7737,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.8864,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4196,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7632,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7297,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9216,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6252,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.6485,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6264,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8358,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4745,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9878,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.959,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8414,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9204,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.7104,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1986,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6862,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3994,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4517,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5644,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.2973,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.8727,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6923,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7474,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.7869,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.6912,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7632,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.6008,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.9295,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6095,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7727,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8537,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.856,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8787,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.6913,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.961,"skill":0.9474,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8376,"private_eq":0.8803},"2":{"raw":0.6697,"skill":0.6185,"coverage":1,"random":0.1341,"tracks":[],"combined":0.689,"private_eq":0.7329},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8908,"private_eq":0.8931},"4":{"raw":0.8642,"skill":0.8625,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8566,"private_eq":0.859},"5":{"raw":0.8693,"skill":0.8685,"coverage":1,"random":0.006,"tracks":[],"combined":0.7495,"private_eq":0.7528},"6":{"raw":0.7737,"skill":0.4694,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8864,"skill":0.8864,"coverage":1,"random":0,"tracks":[],"combined":0.7077,"private_eq":0.71},"10":{"raw":0.4196,"skill":0.0343,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7632,"skill":0.6575,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7297,"skill":0.5951,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3993,"private_eq":0.6029},"20":{"raw":0.923,"skill":0.846,"coverage":1,"random":0.5,"tracks":[],"combined":0.907,"private_eq":0.9535},"21":{"raw":0.6252,"skill":0.2504,"coverage":1,"random":0.5,"tracks":[],"combined":0.1861,"private_eq":0.5972},"22":{"raw":0.6556,"skill":0.653,"coverage":1,"random":0.0078,"tracks":[],"combined":0.47,"private_eq":0.4791},"23":{"raw":0.6337,"skill":0.2673,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8358,"skill":0.7811,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4745,"skill":0.2993,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4462,"combined":0.2541},"26":{"raw":0.9878,"skill":0.9837,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.959,"skill":0.9453,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8414,"skill":0.6828,"coverage":1,"random":0.5,"tracks":[],"combined":0.4714,"private_eq":0.7382},"29":{"raw":0.9204,"skill":0.8939,"coverage":1,"random":0.25,"tracks":[],"combined":0.9336,"private_eq":0.95},"30":{"raw":0.7104,"skill":0.6529,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7718,"random":0.25},{"track":"10-choice","raw":0.649,"random":0.1}],"combined":0.7139,"private_eq":0.7658},"31":{"raw":0.1986,"skill":0.1272,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0734,"private_eq":0.1581},"32":{"raw":0.6862,"skill":0.5011,"coverage":1,"random":0.371,"tracks":[],"combined":0.3356,"private_eq":0.5862},"33":{"raw":0.3994,"skill":0.3914,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4359,"private_eq":0.4486},"36":{"raw":0.4517,"skill":0.3797,"coverage":1,"random":0.116,"tracks":[],"combined":0.3769,"private_eq":0.4546},"37":{"raw":0.5644,"skill":0.4536,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4277,"private_eq":0.5481},"38":{"raw":0.2973,"skill":0.2748,"coverage":1,"random":0.031,"tracks":[],"combined":0.3068,"private_eq":0.335},"39":{"raw":0.8727,"skill":0.8128,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6319,"private_eq":0.7519},"40":{"raw":0.6923,"skill":0.6042,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5867,"headline":false},{"track":"A · English","score":0.6923,"headline":true},{"track":"C · Arabic pairs","score":0.87,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.5588,"private_eq":0.6602},"41":{"raw":0.7474,"skill":0.6211,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4271,"private_eq":0.6217},"42":{"raw":0.7869,"skill":0.5854,"coverage":1,"random":0.486,"tracks":[],"combined":0.7584,"private_eq":0.8767},"43":{"raw":0.6912,"skill":0.5101,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5452,"private_eq":0.716},"44":{"raw":0.7632,"skill":0.5264,"coverage":1,"random":0.5,"tracks":[],"combined":0.5191,"private_eq":0.7618},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6008,"skill":0.4207,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.9295,"skill":0.859,"coverage":1,"random":0.5,"tracks":[],"combined":0.5596,"private_eq":0.7818},"57":{"raw":0.6095,"skill":0.5608,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5742,"combined":0.5166},"58":{"raw":0.7727,"skill":0.6705,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5078,"combined":0.2794},"59":{"raw":0.8537,"skill":0.6967,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2777,"private_eq":0.6551},"61":{"raw":0.856,"skill":0.712,"coverage":1,"random":0.5,"tracks":[],"combined":0.6096,"private_eq":0.8065},"62":{"raw":0.8787,"skill":0.8383,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.6913,"skill":0.6141,"coverage":1,"random":0.2,"tracks":[],"combined":0.7101,"private_eq":0.7699}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #129).","Submitted public 0.3 run adopted (kit rescore 62.30, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.66% on 831 rows (bar 98.5%).","About 17% of training tokens come from train splits of index source benchmarks; part of the data was written or labelled by frontier models (author-declared).","Scored at the PR revision bc28ab1; the repo head adds a serving floor that changes outputs and was not used.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":71.5,"balanced_skill":55.35,"breadth_skill":61.03,"public_skill":62.3},"frozen_scores":{"balanced_raw":71.5,"balanced_skill":62.3,"breadth_skill":61.03},"categories":[{"id":"knowledge","raw":0.5686,"skill":0.4471,"combined":0.4119,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.744,"skill":0.6425,"combined":0.5429,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7534,"skill":0.6905,"combined":0.6025,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8534,"skill":0.8328,"combined":0.7508,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6883,"skill":0.5086,"combined":0.4982,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"wald-26b-a4b","pub":62.3,"same_skills":56.38,"new_domains":41.91,"v":55.35,"lab":false,"jev":false,"rank_v":75,"rank_pub":48,"name":"Wald-26B-A4B","eq":{"S":55.98,"O":54.41},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":33.59,"new_domains":21.76}}},{"engine":"synack-decide-26b-a4b-v7","name":"SynACK Decide 26B-A4B v7","long_name":"SynACK Decide 26B-A4B v7","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":25805933872,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/SkyPanther/synack-decide-26b-a4b","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7358,"conf":0.8048,"ece":0.0691,"brier":0.3597,"over95":0.014,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0001,"conf":0.0836,"acc":0.7179},{"lo":0.1,"w":0.0057,"conf":0.1682,"acc":0.0658},{"lo":0.2,"w":0.0145,"conf":0.2534,"acc":0.11},{"lo":0.3,"w":0.0291,"conf":0.3545,"acc":0.2402},{"lo":0.4,"w":0.045,"conf":0.4567,"acc":0.3177},{"lo":0.5,"w":0.0974,"conf":0.5493,"acc":0.4827},{"lo":0.6,"w":0.0946,"conf":0.6506,"acc":0.5205},{"lo":0.7,"w":0.0983,"conf":0.7512,"acc":0.6418},{"lo":0.8,"w":0.1339,"conf":0.8535,"acc":0.739},{"lo":0.9,"w":0.4814,"conf":0.9683,"acc":0.9435}],"areas":{"knowledge":{"ece":0.1205,"acc":0.6442,"conf":0.7647},"language":{"ece":0.0189,"acc":0.8453,"conf":0.8639},"retrieval":{"ece":0.1387,"acc":0.6836,"conf":0.8223},"tools":{"ece":0.0301,"acc":0.9256,"conf":0.9505},"arts":{"ece":0.0398,"acc":0.6501,"conf":0.6772}}},"gaps":null,"latency":{"median":129.1,"p95":363.7,"mean":174.1},"results":{"1":{"score":0.954,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":252.7},"2":{"score":0.6716,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1665},"3":{"score":0.8248,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":583.2},"4":{"score":0.8479,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":138.9},"5":{"score":0.9215,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":139.6},"6":{"score":0.8009,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":335.1},"9":{"score":0.75,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":337.1},"10":{"score":0.3723,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":150.3},"11":{"score":0.7707,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":553.7},"12":{"score":0.7341,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":127.8},"20":{"score":0.9094,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.1},"21":{"score":0.6404,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":122.3},"22":{"score":0.346,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":408.9},"23":{"score":0.6618,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":130.9},"24":{"score":0.8347,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":124.1},"25":{"score":0.4847,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":126.7},"26":{"score":0.9886,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":122.1},"27":{"score":0.959,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":124.3},"28":{"score":0.8358,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":123},"29":{"score":0.9427,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":129.9},"30":{"score":0.6395,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2172,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":133},"32":{"score":0.6343,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":147.7},"33":{"score":0.3455,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":262.2},"36":{"score":0.4823,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1952.4},"37":{"score":0.5684,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":134.2},"38":{"score":0.2494,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":500.6},"39":{"score":0.9332,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":244},"40":{"score":0.5954,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":122.2},"41":{"score":0.773,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":128.9},"42":{"score":0.8063,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":137.2},"43":{"score":0.7123,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":124.2},"44":{"score":0.7178,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129.4},"45":{"score":0.1158,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":130},"50":{"score":0.6301,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":142.9},"56":{"score":0.7865,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":269.1},"57":{"score":0.6587,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":126.7},"58":{"score":0.8073,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":128.5},"59":{"score":0.7998,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":144.3},"61":{"score":0.735,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":136.5},"62":{"score":0.8428,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":143.2},"64":{"score":0.7955,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":131.2}},"benchmarks":{"1":{"raw":0.954,"skill":0.9378,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8581,"private_eq":0.8953},"2":{"raw":0.6716,"skill":0.6208,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6834,"private_eq":0.7281},"3":{"raw":0.8248,"skill":0.8214,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8593,"private_eq":0.8625},"4":{"raw":0.8479,"skill":0.8459,"coverage":1,"random":0.0127,"tracks":[],"combined":0.883,"private_eq":0.8848},"5":{"raw":0.9215,"skill":0.921,"coverage":1,"random":0.006,"tracks":[],"combined":0.74,"private_eq":0.7435},"6":{"raw":0.8009,"skill":0.5332,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.75,"skill":0.75,"coverage":1,"random":0,"tracks":[],"combined":0.5711,"private_eq":0.575},"10":{"raw":0.3723,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7707,"skill":0.6684,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7341,"skill":0.6017,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4395,"private_eq":0.6294},"20":{"raw":0.9112,"skill":0.8224,"coverage":1,"random":0.5,"tracks":[],"combined":0.8926,"private_eq":0.9464},"21":{"raw":0.6404,"skill":0.2808,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.3436,"skill":0.3385,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3847,"private_eq":0.3955},"23":{"raw":0.6794,"skill":0.3587,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8347,"skill":0.7796,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4847,"skill":0.3129,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5033,"combined":0.3311},"26":{"raw":0.9886,"skill":0.9848,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.959,"skill":0.9453,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8358,"skill":0.6716,"coverage":1,"random":0.5,"tracks":[],"combined":0.5572,"private_eq":0.7806},"29":{"raw":0.9427,"skill":0.9236,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6395,"skill":0.5713,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7468,"random":0.25},{"track":"10-choice","raw":0.5322,"random":0.1}],"combined":0.6397,"private_eq":0.7053},"31":{"raw":0.2172,"skill":0.1474,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1091,"private_eq":0.1905},"32":{"raw":0.6343,"skill":0.4186,"coverage":1,"random":0.371,"tracks":[],"combined":0.2582,"private_eq":0.5381},"33":{"raw":0.3455,"skill":0.3368,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4102,"private_eq":0.4236},"36":{"raw":0.4823,"skill":0.4144,"coverage":1,"random":0.116,"tracks":[],"combined":0.3693,"private_eq":0.4479},"37":{"raw":0.5684,"skill":0.4587,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5491,"private_eq":0.6438},"38":{"raw":0.2494,"skill":0.2254,"coverage":1,"random":0.031,"tracks":[],"combined":0.4352,"private_eq":0.458},"39":{"raw":0.9332,"skill":0.9018,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4805,"private_eq":0.6501},"40":{"raw":0.5954,"skill":0.4795,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5212,"headline":false},{"track":"A · English","score":0.5954,"headline":true},{"track":"C · Arabic pairs","score":0.81,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.4616,"private_eq":0.5855},"41":{"raw":0.773,"skill":0.6595,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4613,"private_eq":0.6443},"42":{"raw":0.8063,"skill":0.6232,"coverage":1,"random":0.486,"tracks":[],"combined":0.7448,"private_eq":0.8698},"43":{"raw":0.7123,"skill":0.5435,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5746,"private_eq":0.7343},"44":{"raw":0.7178,"skill":0.4356,"coverage":1,"random":0.5,"tracks":[],"combined":0.4548,"private_eq":0.73},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6301,"skill":0.4631,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.7865,"skill":0.573,"coverage":1,"random":0.5,"tracks":[],"combined":0.4732,"private_eq":0.7391},"57":{"raw":0.6587,"skill":0.6161,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.585,"combined":0.5289},"58":{"raw":0.8073,"skill":0.7207,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4918,"combined":0.2559},"59":{"raw":0.7998,"skill":0.5849,"coverage":1,"random":0.5177,"tracks":[],"combined":0.32,"private_eq":0.6753},"61":{"raw":0.735,"skill":0.47,"coverage":1,"random":0.5,"tracks":[],"combined":0.5414,"private_eq":0.7728},"62":{"raw":0.8428,"skill":0.7904,"coverage":1,"random":0.25,"tracks":[],"combined":0.6246,"private_eq":0.7209},"64":{"raw":0.7955,"skill":0.7444,"coverage":1,"random":0.2,"tracks":[],"combined":0.7975,"private_eq":0.839}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #71).","Public 0.3 score 59.68: the submitted 0.2.1 rows were adopted where the request is unchanged in 0.3 and the lab ran the rest with the author's engine (0.2.1 claim 59.54). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 100.00% on 731 rows (bar 98.5%).","Training includes train splits of about 19 board-related benchmarks, decontaminated per author; distillation targets from Kimi-K3 and Gemma-4-31B.","The commit the run loaded was removed by a history squash; every surviving commit has identical weights."],"scores":{"balanced_raw":69.72,"balanced_skill":56.22,"breadth_skill":58.61,"public_skill":59.68},"frozen_scores":{"balanced_raw":69.72,"balanced_skill":59.68,"breadth_skill":58.61},"categories":[{"id":"knowledge","raw":0.5624,"skill":0.4366,"combined":0.4124,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7383,"skill":0.6354,"combined":0.5514,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7261,"skill":0.6241,"combined":0.5991,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8146,"skill":0.7912,"combined":0.7296,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6667,"skill":0.5013,"combined":0.5173,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"synack-decide-26b-a4b-v7","pub":59.68,"same_skills":56.29,"new_domains":43.79,"v":56.22,"lab":false,"jev":false,"rank_v":71,"rank_pub":63,"name":"SynACK Decide 26B-A4B v7","eq":{"S":55.89,"O":56.72},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":33.53,"new_domains":22.69}}},{"engine":"ajev-gemma26-lora1","name":"AJev lora1 (Gemma 4 26B-A4B)","long_name":"AJev lora1 (Gemma 4 26B-A4B)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/andyzhang232/ajev-gemma4-26b-a4b-lora1","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7324,"conf":0.78,"ece":0.0476,"brier":0.3584,"over95":0.0084,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0041,"conf":0.0865,"acc":0.0675},{"lo":0.1,"w":0.0174,"conf":0.1455,"acc":0.1269},{"lo":0.2,"w":0.0152,"conf":0.2539,"acc":0.1715},{"lo":0.3,"w":0.0299,"conf":0.3561,"acc":0.3547},{"lo":0.4,"w":0.0514,"conf":0.4541,"acc":0.3503},{"lo":0.5,"w":0.0911,"conf":0.5462,"acc":0.4982},{"lo":0.6,"w":0.106,"conf":0.6444,"acc":0.5557},{"lo":0.7,"w":0.1029,"conf":0.7513,"acc":0.6379},{"lo":0.8,"w":0.1392,"conf":0.856,"acc":0.7776},{"lo":0.9,"w":0.4427,"conf":0.9594,"acc":0.9498}],"areas":{"knowledge":{"ece":0.0659,"acc":0.6467,"conf":0.7127},"language":{"ece":0.0167,"acc":0.8289,"conf":0.8315},"retrieval":{"ece":0.0978,"acc":0.7033,"conf":0.8007},"tools":{"ece":0.0264,"acc":0.9108,"conf":0.9367},"arts":{"ece":0.0664,"acc":0.6462,"conf":0.7117}}},"gaps":null,"latency":{"median":88.8,"p95":247,"mean":94.3},"results":{"1":{"score":0.9534,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":277.4},"2":{"score":0.6323,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2847.5},"3":{"score":0.8169,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2447.9},"4":{"score":0.7999,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":395.1},"5":{"score":0.883,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":521.6},"6":{"score":0.7992,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1406.5},"9":{"score":0.4773,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":745.5},"10":{"score":0.5316,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":286.3},"11":{"score":0.7978,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1622.8},"12":{"score":0.7061,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":92.3},"20":{"score":0.8872,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.2},"21":{"score":0.6465,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":70.5},"22":{"score":0.6285,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1988.5},"23":{"score":0.6232,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":107.5},"24":{"score":0.8278,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":84.1},"25":{"score":0.4541,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":108},"26":{"score":0.987,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":78},"27":{"score":0.9633,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":80.5},"28":{"score":0.8279,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":74.2},"29":{"score":0.9271,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":117.4},"30":{"score":0.6941,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1788,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":264.6},"32":{"score":0.6596,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":231.1},"33":{"score":0.3558,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":278.8},"36":{"score":0.4701,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4309.3},"37":{"score":0.5489,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":184.6},"38":{"score":0.2838,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1778.7},"39":{"score":0.8257,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":122.3},"40":{"score":0.651,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":110.5},"41":{"score":0.7912,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":126.5},"42":{"score":0.7709,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":132},"43":{"score":0.6579,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":119.6},"44":{"score":0.7178,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":106.3},"45":{"score":0.1158,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":38.3},"50":{"score":0.4266,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":274.5},"56":{"score":0.781,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":278.1},"57":{"score":0.639,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":143.8},"58":{"score":0.7514,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":120.9},"59":{"score":0.7951,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":181.6},"61":{"score":0.8505,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":188.9},"62":{"score":0.8505,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":181.1},"64":{"score":0.7443,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":123.4}},"benchmarks":{"1":{"raw":0.9534,"skill":0.937,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8648,"private_eq":0.9002},"2":{"raw":0.6323,"skill":0.5754,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6725,"private_eq":0.7188},"3":{"raw":0.8169,"skill":0.8134,"coverage":1,"random":0.0189,"tracks":[],"combined":0.812,"private_eq":0.8167},"4":{"raw":0.7999,"skill":0.7973,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8545,"private_eq":0.857},"5":{"raw":0.883,"skill":0.8823,"coverage":1,"random":0.006,"tracks":[],"combined":0.7502,"private_eq":0.7535},"6":{"raw":0.7992,"skill":0.5292,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4773,"skill":0.4773,"coverage":1,"random":0,"tracks":[],"combined":0.3309,"private_eq":0.3375},"10":{"raw":0.5316,"skill":0.2206,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7978,"skill":0.7076,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7061,"skill":0.5598,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4415,"private_eq":0.6307},"20":{"raw":0.8908,"skill":0.7815,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.6465,"skill":0.293,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.6243,"skill":0.6214,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4887,"private_eq":0.4975},"23":{"raw":0.6318,"skill":0.2637,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8278,"skill":0.7704,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4541,"skill":0.2721,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4649,"combined":0.2793},"26":{"raw":0.987,"skill":0.9827,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9633,"skill":0.9511,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8279,"skill":0.6558,"coverage":1,"random":0.5,"tracks":[],"combined":0.4309,"private_eq":0.7182},"29":{"raw":0.9271,"skill":0.9028,"coverage":1,"random":0.25,"tracks":[],"combined":0.9879,"private_eq":0.9903},"30":{"raw":0.6941,"skill":0.6345,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7695,"random":0.25},{"track":"10-choice","raw":0.6187,"random":0.1}],"combined":0.7189,"private_eq":0.7699},"31":{"raw":0.1788,"skill":0.1056,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0821,"private_eq":0.166},"32":{"raw":0.6596,"skill":0.4588,"coverage":1,"random":0.371,"tracks":[],"combined":0.2428,"private_eq":0.5285},"33":{"raw":0.3558,"skill":0.3472,"coverage":1,"random":0.0131,"tracks":[],"combined":0.405,"private_eq":0.4185},"36":{"raw":0.4701,"skill":0.4006,"coverage":1,"random":0.116,"tracks":[],"combined":0.3443,"private_eq":0.4261},"37":{"raw":0.5489,"skill":0.4342,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4933,"private_eq":0.5998},"38":{"raw":0.2838,"skill":0.2609,"coverage":1,"random":0.031,"tracks":[],"combined":0.4634,"private_eq":0.485},"39":{"raw":0.8257,"skill":0.7436,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5069,"private_eq":0.6679},"40":{"raw":0.651,"skill":0.551,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5385,"headline":false},{"track":"A · English","score":0.651,"headline":true},{"track":"C · Arabic pairs","score":0.785,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.6255,"private_eq":0.7114},"41":{"raw":0.7912,"skill":0.6867,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3172,"private_eq":0.5493},"42":{"raw":0.7709,"skill":0.5543,"coverage":1,"random":0.486,"tracks":[],"combined":0.7782,"private_eq":0.8868},"43":{"raw":0.6579,"skill":0.4572,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5955,"private_eq":0.7473},"44":{"raw":0.7178,"skill":0.4356,"coverage":1,"random":0.5,"tracks":[],"combined":0.5031,"private_eq":0.7539},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4266,"skill":0.1678,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.781,"skill":0.562,"coverage":1,"random":0.5,"tracks":[],"combined":0.5252,"private_eq":0.7648},"57":{"raw":0.639,"skill":0.594,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5274,"combined":0.4633},"58":{"raw":0.7514,"skill":0.6397,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.467,"combined":0.2195},"59":{"raw":0.7951,"skill":0.5752,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3399,"private_eq":0.6848},"61":{"raw":0.8505,"skill":0.701,"coverage":1,"random":0.5,"tracks":[],"combined":0.6096,"private_eq":0.8065},"62":{"raw":0.8505,"skill":0.8007,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.7443,"skill":0.6804,"coverage":1,"random":0.2,"tracks":[],"combined":0.7198,"private_eq":0.7776}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #68).","Public 0.3 score 57.67: the submitted 0.2.1 rows were adopted where the request is unchanged in 0.3 and the lab ran the rest with the author's engine (0.2.1 claim 57.42). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.58% on 727 rows (bar 98.5%).","Training includes train splits of seven board benchmarks (author-declared)."],"scores":{"balanced_raw":68.03,"balanced_skill":55.29,"breadth_skill":56.67,"public_skill":57.67},"frozen_scores":{"balanced_raw":68.03,"balanced_skill":57.67,"breadth_skill":56.67},"categories":[{"id":"knowledge","raw":0.5508,"skill":0.4194,"combined":0.3906,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7314,"skill":0.6207,"combined":0.5651,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7218,"skill":0.6354,"combined":0.601,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7564,"skill":0.7322,"combined":0.6665,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6607,"skill":0.468,"combined":0.5197,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"ajev-gemma26-lora1","pub":57.67,"same_skills":54.92,"new_domains":43.58,"v":55.29,"lab":false,"jev":false,"rank_v":76,"rank_pub":84,"name":"AJev lora1 (Gemma 4 26B-A4B)","eq":{"S":54.5,"O":56.47},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":32.7,"new_domains":22.59}}},{"engine":"drex-v2.0","name":"Nace.AI Drex v2.0 (35B-A3B)","long_name":"Nace.AI Drex v2.0 (35B-A3B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":35107181936,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/nace-ai/drex-v2.0","code_url":null,"base_model":"Qwen/Qwen3.6-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","weights_repo":null,"served_checkpoint":null,"org":{"id":"nace-ai","name":"Nace.AI","logo":"data/orgs/nace-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7708,"conf":0.7624,"ece":0.0183,"brier":0.3168,"over95":0.0047,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.015,"conf":0.0711,"acc":0.1436},{"lo":0.1,"w":0.0138,"conf":0.1432,"acc":0.1952},{"lo":0.2,"w":0.0126,"conf":0.2576,"acc":0.3023},{"lo":0.3,"w":0.034,"conf":0.3579,"acc":0.3925},{"lo":0.4,"w":0.0594,"conf":0.4533,"acc":0.4946},{"lo":0.5,"w":0.1129,"conf":0.5488,"acc":0.5704},{"lo":0.6,"w":0.1169,"conf":0.6502,"acc":0.6672},{"lo":0.7,"w":0.1023,"conf":0.7497,"acc":0.7778},{"lo":0.8,"w":0.1301,"conf":0.8529,"acc":0.8162},{"lo":0.9,"w":0.403,"conf":0.9713,"acc":0.9708}],"areas":{"knowledge":{"ece":0.0405,"acc":0.6958,"conf":0.7164},"language":{"ece":0.0515,"acc":0.8731,"conf":0.8225},"retrieval":{"ece":0.0723,"acc":0.6924,"conf":0.7634},"tools":{"ece":0.0273,"acc":0.9306,"conf":0.9361},"arts":{"ece":0.0663,"acc":0.7031,"conf":0.6413}}},"gaps":null,"latency":{"median":275.9,"p95":529.3,"mean":329.7},"results":{"1":{"score":0.9664,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":1882.6},"2":{"score":0.673,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":6306.3},"3":{"score":0.8327,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3937.9},"4":{"score":0.8051,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":858.8},"5":{"score":0.8911,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1009.2},"6":{"score":0.792,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1054.4},"9":{"score":0.5455,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1468.2},"10":{"score":0.4743,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":1088.3},"11":{"score":0.8339,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1013.6},"12":{"score":0.7048,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":652.3},"20":{"score":0.8896,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":597.7},"21":{"score":0.6355,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":1521.6},"22":{"score":0.7975,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1705.5},"23":{"score":0.5922,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":1003.9},"24":{"score":0.8527,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":884},"25":{"score":0.5153,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":914.8},"26":{"score":0.9861,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":957.9},"27":{"score":0.9684,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":955.9},"28":{"score":0.8611,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":939.1},"29":{"score":0.9589,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":621.3},"30":{"score":0.6683,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2344,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":787},"32":{"score":0.5957,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":1211.6},"33":{"score":0.2412,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1415.3},"36":{"score":0.4721,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1691.8},"37":{"score":0.5459,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":1550.5},"38":{"score":0.2388,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1834.4},"39":{"score":0.9081,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":1645.3},"40":{"score":0.6366,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":1281.1},"41":{"score":0.7837,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":760.7},"42":{"score":0.8189,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":683},"43":{"score":0.8456,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":629.8},"44":{"score":0.8454,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":601.1},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":603.9},"50":{"score":0.6241,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":609.4},"56":{"score":0.7495,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":618.9},"57":{"score":0.7284,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":594},"58":{"score":0.7835,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":689.6},"59":{"score":0.8248,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":767.4},"61":{"score":0.8575,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":730.5},"62":{"score":0.8656,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":759.9},"64":{"score":0.7765,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":764.4}},"benchmarks":{"1":{"raw":0.9664,"skill":0.9546,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.673,"skill":0.6224,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7015,"private_eq":0.7436},"3":{"raw":0.8327,"skill":0.8295,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8699,"private_eq":0.8728},"4":{"raw":0.8051,"skill":0.8026,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8387,"private_eq":0.8416},"5":{"raw":0.8911,"skill":0.8904,"coverage":1,"random":0.006,"tracks":[],"combined":0.6892,"private_eq":0.6936},"6":{"raw":0.792,"skill":0.5123,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5455,"skill":0.5455,"coverage":1,"random":0,"tracks":[],"combined":0.4826,"private_eq":0.4875},"10":{"raw":0.4743,"skill":0.1253,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8339,"skill":0.7598,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7048,"skill":0.5578,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4784,"private_eq":0.6551},"20":{"raw":0.8906,"skill":0.7812,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.6355,"skill":0.2709,"coverage":1,"random":0.5,"tracks":[],"combined":0.3975,"private_eq":0.7017},"22":{"raw":0.7966,"skill":0.795,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5869,"private_eq":0.5938},"23":{"raw":0.6058,"skill":0.2116,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8527,"skill":0.8036,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5153,"skill":0.3537,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4856,"combined":0.3072},"26":{"raw":0.9861,"skill":0.9815,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9684,"skill":0.9579,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8611,"skill":0.7222,"coverage":1,"random":0.5,"tracks":[],"combined":0.542,"private_eq":0.7731},"29":{"raw":0.9589,"skill":0.9452,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6683,"skill":0.603,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7437,"random":0.25},{"track":"10-choice","raw":0.5929,"random":0.1}],"combined":0.7167,"private_eq":0.7681},"31":{"raw":0.2344,"skill":0.1661,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.5957,"skill":0.3572,"coverage":1,"random":0.371,"tracks":[],"combined":0.2505,"private_eq":0.5333},"33":{"raw":0.2412,"skill":0.2311,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0917,"private_eq":0.1128},"36":{"raw":0.4721,"skill":0.4029,"coverage":1,"random":0.116,"tracks":[],"combined":0.3882,"private_eq":0.4644},"37":{"raw":0.5459,"skill":0.4304,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5075,"private_eq":0.611},"38":{"raw":0.2388,"skill":0.2144,"coverage":1,"random":0.031,"tracks":[],"combined":0.3135,"private_eq":0.3414},"39":{"raw":0.9081,"skill":0.8648,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5426,"private_eq":0.6919},"40":{"raw":0.6366,"skill":0.5324,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5273,"headline":false},{"track":"A · English","score":0.6366,"headline":true},{"track":"C · Arabic pairs","score":0.91,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.6347,"private_eq":0.7185},"41":{"raw":0.7837,"skill":0.6756,"coverage":1,"random":0.3333,"tracks":[],"combined":0.088,"private_eq":0.3982},"42":{"raw":0.8189,"skill":0.6477,"coverage":1,"random":0.486,"tracks":[],"combined":0.8585,"private_eq":0.9276},"43":{"raw":0.8456,"skill":0.7551,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4241,"private_eq":0.6405},"44":{"raw":0.8454,"skill":0.6908,"coverage":1,"random":0.5,"tracks":[],"combined":0.6423,"private_eq":0.8227},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6241,"skill":0.4545,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.7495,"skill":0.499,"coverage":1,"random":0.5,"tracks":[],"combined":0.4299,"private_eq":0.7177},"57":{"raw":0.7284,"skill":0.6945,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4278,"combined":0.35},"58":{"raw":0.7835,"skill":0.6862,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4553,"combined":0.2024},"59":{"raw":0.8248,"skill":0.6367,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2129,"private_eq":0.6242},"61":{"raw":0.8575,"skill":0.715,"coverage":1,"random":0.5,"tracks":[],"combined":0.6209,"private_eq":0.8121},"62":{"raw":0.8656,"skill":0.8208,"coverage":1,"random":0.25,"tracks":[],"combined":0.5903,"private_eq":0.6955},"64":{"raw":0.7765,"skill":0.7206,"coverage":1,"random":0.2,"tracks":[],"combined":0.7392,"private_eq":0.7929}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #156).","Submitted public 0.3 run adopted (kit rescore 61.25, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the public transformers code; decision agreement with the submitted rows 98.66% on 831 rows (bar 98.5%).","The submitted run used an unpublished SGLang plugin and router; the card ran the public transformers code (published after the PR) on the PR's weights, so identity rests on the agreement gate.","Training data and decontamination undisclosed."],"scores":{"balanced_raw":70.55,"balanced_skill":52.76,"breadth_skill":60.56,"public_skill":61.25},"frozen_scores":{"balanced_raw":70.55,"balanced_skill":61.25,"breadth_skill":60.56},"categories":[{"id":"knowledge","raw":0.5875,"skill":0.4793,"combined":0.3602,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7491,"skill":0.6527,"combined":0.5268,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7204,"skill":0.6302,"combined":0.5845,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7857,"skill":0.7647,"combined":0.7247,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7215,"skill":0.539,"combined":0.5523,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"drex-v2.0","pub":61.25,"same_skills":54.3,"new_domains":39.22,"v":52.76,"lab":false,"jev":false,"rank_v":86,"rank_pub":54,"name":"Nace.AI Drex v2.0 (35B-A3B)","eq":{"S":53.87,"O":51.08},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":32.32,"new_domains":20.43}}},{"engine":"decision-4b-v1.6","name":"FlyMy.AI Decision 4B v1.6","long_name":"FlyMy.AI Decision 4B v1.6","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/flymy-ai/decision-4b-v1.6","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"flymy-ai","name":"FlyMy.AI","logo":"data/orgs/flymy-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6709,"conf":0.7164,"ece":0.0459,"brier":0.4329,"over95":0.0053,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.016,"conf":0.0785,"acc":0.0908},{"lo":0.1,"w":0.0156,"conf":0.1399,"acc":0.1399},{"lo":0.2,"w":0.018,"conf":0.2581,"acc":0.188},{"lo":0.3,"w":0.0466,"conf":0.3517,"acc":0.345},{"lo":0.4,"w":0.0656,"conf":0.4523,"acc":0.3418},{"lo":0.5,"w":0.1352,"conf":0.548,"acc":0.5337},{"lo":0.6,"w":0.1274,"conf":0.6489,"acc":0.586},{"lo":0.7,"w":0.1322,"conf":0.7517,"acc":0.6838},{"lo":0.8,"w":0.1675,"conf":0.8513,"acc":0.7703},{"lo":0.9,"w":0.2759,"conf":0.9551,"acc":0.9392}],"areas":{"knowledge":{"ece":0.0698,"acc":0.579,"conf":0.6476},"language":{"ece":0.038,"acc":0.7398,"conf":0.7732},"retrieval":{"ece":0.053,"acc":0.6867,"conf":0.675},"tools":{"ece":0.0379,"acc":0.8531,"conf":0.8661},"arts":{"ece":0.0909,"acc":0.6085,"conf":0.6948}}},"gaps":null,"latency":{"median":27.2,"p95":269.2,"mean":98.8},"results":{"1":{"score":0.8442,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":137.4},"2":{"score":0.627,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1168.8},"3":{"score":0.6949,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2288.1},"4":{"score":0.6958,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":156.1},"5":{"score":0.7861,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":286.7},"6":{"score":0.7704,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":464.7},"9":{"score":0.2841,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1911.2},"10":{"score":0.6344,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":107.1},"11":{"score":0.7218,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2454.9},"12":{"score":0.6244,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.1},"20":{"score":0.8174,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.1},"21":{"score":0.6058,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":15.9},"22":{"score":0.6235,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1861.8},"23":{"score":0.6136,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46},"24":{"score":0.7353,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.1},"25":{"score":0.3673,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":27.8},"26":{"score":0.9798,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.9},"27":{"score":0.9317,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.9},"28":{"score":0.7451,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":15.8},"29":{"score":0.8209,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":30.4},"30":{"score":0.3321,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1158,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":170.1},"32":{"score":0.5319,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":106.9},"33":{"score":0.3127,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":294.4},"36":{"score":0.4179,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1551.6},"37":{"score":0.4891,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.6},"38":{"score":0.1614,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1223.9},"39":{"score":0.8383,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":43.7},"40":{"score":0.4496,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":16},"41":{"score":0.4615,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":22.2},"42":{"score":0.7622,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":46.1},"43":{"score":0.4982,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":22.1},"44":{"score":0.6332,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.1},"45":{"score":0.1218,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":30.4},"50":{"score":0.4553,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":69.3},"56":{"score":0.5775,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":240.4},"57":{"score":0.5125,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":30.3},"58":{"score":0.6464,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":22.3},"59":{"score":0.711,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":69},"61":{"score":0.6455,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":37},"62":{"score":0.7024,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":69},"64":{"score":0.5852,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":27.9}},"benchmarks":{"1":{"raw":0.8442,"skill":0.7896,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6536,"private_eq":0.7456},"2":{"raw":0.627,"skill":0.5692,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6679,"private_eq":0.7148},"3":{"raw":0.6949,"skill":0.689,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6232,"private_eq":0.6335},"4":{"raw":0.6958,"skill":0.6919,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7358,"private_eq":0.7411},"5":{"raw":0.7861,"skill":0.7848,"coverage":1,"random":0.006,"tracks":[],"combined":0.7057,"private_eq":0.7098},"6":{"raw":0.7704,"skill":0.4617,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2841,"skill":0.2841,"coverage":1,"random":0,"tracks":[],"combined":0.1564,"private_eq":0.165},"10":{"raw":0.6344,"skill":0.3917,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7218,"skill":0.5977,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6244,"skill":0.4374,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4528,"private_eq":0.6382},"20":{"raw":0.8195,"skill":0.6391,"coverage":1,"random":0.5,"tracks":[],"combined":0.7119,"private_eq":0.8571},"21":{"raw":0.6058,"skill":0.2116,"coverage":1,"random":0.5,"tracks":[],"combined":0.1258,"private_eq":0.5674},"22":{"raw":0.6112,"skill":0.6081,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4694,"private_eq":0.4785},"23":{"raw":0.595,"skill":0.19,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7353,"skill":0.6471,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3673,"skill":0.1565,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3485,"combined":0.1223},"26":{"raw":0.9798,"skill":0.9731,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9317,"skill":0.9089,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7451,"skill":0.4902,"coverage":1,"random":0.5,"tracks":[],"combined":0.3654,"private_eq":0.6858},"29":{"raw":0.8209,"skill":0.7612,"coverage":1,"random":0.25,"tracks":[],"combined":0.7059,"private_eq":0.7812},"30":{"raw":0.3321,"skill":0.1969,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4511,"random":0.25},{"track":"10-choice","raw":0.213,"random":0.1}],"combined":0.3649,"private_eq":0.4812},"31":{"raw":0.1158,"skill":0.037,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0679,"private_eq":0.1531},"32":{"raw":0.5319,"skill":0.2558,"coverage":1,"random":0.371,"tracks":[],"combined":0.15,"private_eq":0.4708},"33":{"raw":0.3127,"skill":0.3036,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3279,"private_eq":0.3433},"36":{"raw":0.4179,"skill":0.3415,"coverage":1,"random":0.116,"tracks":[],"combined":0.3224,"private_eq":0.4069},"37":{"raw":0.4891,"skill":0.3591,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4078,"private_eq":0.5324},"38":{"raw":0.1614,"skill":0.1346,"coverage":1,"random":0.031,"tracks":[],"combined":0.2499,"private_eq":0.2805},"39":{"raw":0.8383,"skill":0.7622,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5577,"private_eq":0.702},"40":{"raw":0.4496,"skill":0.2919,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3404,"headline":false},{"track":"A · English","score":0.4496,"headline":true},{"track":"C · Arabic pairs","score":0.67,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"combined":0.2033,"private_eq":0.387},"41":{"raw":0.4615,"skill":0.1923,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5029,"private_eq":0.6717},"42":{"raw":0.7622,"skill":0.5374,"coverage":1,"random":0.486,"tracks":[],"combined":0.6039,"private_eq":0.7982},"43":{"raw":0.4982,"skill":0.2039,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4032,"private_eq":0.6275},"44":{"raw":0.6332,"skill":0.2664,"coverage":1,"random":0.5,"tracks":[],"combined":0.3852,"private_eq":0.6956},"45":{"raw":0.1218,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4553,"skill":0.2094,"coverage":1,"random":0.311,"tracks":[],"combined":0.1596,"private_eq":0.4269},"56":{"raw":0.5775,"skill":0.155,"coverage":1,"random":0.5,"tracks":[],"combined":0.2654,"private_eq":0.6364},"57":{"raw":0.5125,"skill":0.4517,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5684,"combined":0.51},"58":{"raw":0.6464,"skill":0.4875,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3947,"combined":0.1135},"59":{"raw":0.711,"skill":0.4008,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2204,"private_eq":0.6278},"61":{"raw":0.6455,"skill":0.291,"coverage":1,"random":0.5,"tracks":[],"combined":0.3712,"private_eq":0.6887},"62":{"raw":0.7024,"skill":0.6032,"coverage":1,"random":0.25,"tracks":[],"combined":0.563,"private_eq":0.6752},"64":{"raw":0.5852,"skill":0.4815,"coverage":1,"random":0.2,"tracks":[],"combined":0.5161,"private_eq":0.6165}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #142).","Submitted public 0.3 run adopted (kit rescore 42.69, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.51% on 831 rows (bar 98.5%).","ContractNLI training items built from 27 train-split documents share hypotheses and boilerplate with the suite requests (author-declared).","Supersedes Decision 4B v1.5 (PR #130), which is not listed."],"scores":{"balanced_raw":57.41,"balanced_skill":39.8,"breadth_skill":41.43,"public_skill":42.7},"frozen_scores":{"balanced_raw":57.41,"balanced_skill":42.7,"breadth_skill":41.43},"categories":[{"id":"knowledge","raw":0.4413,"skill":0.2639,"combined":0.2808,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6213,"skill":0.4633,"combined":0.4439,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6048,"skill":0.4526,"combined":0.4789,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6408,"skill":0.5983,"combined":0.5406,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.612,"skill":0.3899,"combined":0.3965,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"decision-4b-v1.6","pub":42.7,"same_skills":43.09,"new_domains":26.75,"v":39.8,"lab":false,"jev":false,"rank_v":135,"rank_pub":143,"name":"FlyMy.AI Decision 4B v1.6","eq":{"S":42.54,"O":35.68},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":25.52,"new_domains":14.27}}},{"engine":"sieve-0.8b","name":"Sieve 0.8B","long_name":"Sieve 0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":590000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/Sieve-0.8B","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"}},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.5785,"conf":0.6658,"ece":0.0882,"brier":0.541,"over95":0.027,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0552,"conf":0.0688,"acc":0.0242},{"lo":0.1,"w":0.0102,"conf":0.1407,"acc":0.1838},{"lo":0.2,"w":0.016,"conf":0.2631,"acc":0.2342},{"lo":0.3,"w":0.0598,"conf":0.3536,"acc":0.3254},{"lo":0.4,"w":0.087,"conf":0.4511,"acc":0.4142},{"lo":0.5,"w":0.181,"conf":0.5472,"acc":0.5128},{"lo":0.6,"w":0.1292,"conf":0.6486,"acc":0.5695},{"lo":0.7,"w":0.1035,"conf":0.7481,"acc":0.622},{"lo":0.8,"w":0.0983,"conf":0.8514,"acc":0.6704},{"lo":0.9,"w":0.2599,"conf":0.9698,"acc":0.8439}],"areas":{"knowledge":{"ece":0.0671,"acc":0.5047,"conf":0.5718},"language":{"ece":0.0726,"acc":0.6653,"conf":0.7365},"retrieval":{"ece":0.1502,"acc":0.6955,"conf":0.8351},"tools":{"ece":0.1332,"acc":0.6599,"conf":0.7923},"arts":{"ece":0.0935,"acc":0.4271,"conf":0.5154}}},"gaps":null,"latency":{"median":51.8,"p95":109.9,"mean":76.5},"results":{"1":{"score":0.843,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":149.4},"2":{"score":0.6156,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":668.6},"3":{"score":0.5492,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":363.7},"4":{"score":0.8564,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":197.1},"5":{"score":0.9061,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":320.8},"6":{"score":0.7811,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":311},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":633.5},"10":{"score":0.4097,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":128.7},"11":{"score":0.7414,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1341.7},"12":{"score":0.4324,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":111.5},"20":{"score":0.561,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":108},"21":{"score":0.583,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":100.5},"22":{"score":0.014,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1990},"23":{"score":0.5724,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":97.3},"24":{"score":0.5057,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":87.2},"25":{"score":0.3622,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":86.4},"26":{"score":0.867,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":85.4},"27":{"score":0.7295,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":85.8},"28":{"score":0.6425,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":84.3},"29":{"score":0.7562,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":91.4},"30":{"score":0.3688,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0922,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":120.1},"32":{"score":0.5545,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":113},"33":{"score":0.1418,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":200.4},"36":{"score":0.275,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":885},"37":{"score":0.4122,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.5},"38":{"score":0.1012,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":474.7},"39":{"score":0.6522,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":95.3},"40":{"score":0.1834,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":90.3},"41":{"score":0.3715,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":85.1},"42":{"score":0.6227,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":92.3},"43":{"score":0.3298,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":83.4},"44":{"score":0.5324,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":82},"45":{"score":0.1297,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":92.7},"50":{"score":0.4099,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":101.4},"56":{"score":0.9265,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":141.9},"57":{"score":0.2689,"jev":null,"metric":"accuracy","requests":12032,"answered":12031,"unsupported":1,"errors":0,"median_ms":93.9},"58":{"score":0.4511,"jev":null,"metric":"accuracy","requests":5507,"answered":5506,"unsupported":1,"errors":0,"median_ms":92.5},"59":{"score":0.4976,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":99.8},"61":{"score":0.5855,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":96.3},"62":{"score":0.7117,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":96.8},"64":{"score":0.392,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":97.1}},"benchmarks":{"1":{"raw":0.843,"skill":0.788,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6332,"private_eq":0.7306},"2":{"raw":0.6156,"skill":0.5561,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6385,"private_eq":0.6897},"3":{"raw":0.5492,"skill":0.5405,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2452,"private_eq":0.2669},"4":{"raw":0.8564,"skill":0.8546,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7342,"private_eq":0.7396},"5":{"raw":0.9061,"skill":0.9055,"coverage":1,"random":0.006,"tracks":[],"combined":0.5559,"private_eq":0.5626},"6":{"raw":0.7811,"skill":0.4868,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0.005},"10":{"raw":0.4097,"skill":0.0178,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7414,"skill":0.626,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4324,"skill":0.1498,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4375,"private_eq":0.6281},"20":{"raw":0.5607,"skill":0.1214,"coverage":1,"random":0.5,"tracks":[],"combined":0.2492,"private_eq":0.6284},"21":{"raw":0.583,"skill":0.1659,"coverage":1,"random":0.5,"tracks":[],"combined":0.1157,"private_eq":0.5624},"22":{"raw":0.0136,"skill":0.0059,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5677,"skill":0.1354,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5057,"skill":0.3409,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3622,"skill":0.1497,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2924,"combined":0.0467},"26":{"raw":0.867,"skill":0.8226,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7295,"skill":0.6392,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6425,"skill":0.285,"coverage":1,"random":0.5,"tracks":[],"combined":0.0931,"private_eq":0.5512},"29":{"raw":0.7562,"skill":0.6749,"coverage":1,"random":0.25,"tracks":[],"combined":0.5226,"private_eq":0.6453},"30":{"raw":0.3688,"skill":0.24,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4716,"random":0.25},{"track":"10-choice","raw":0.2661,"random":0.1}],"combined":0.2441,"private_eq":0.3827},"31":{"raw":0.0922,"skill":0.0113,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0387,"private_eq":0.1266},"32":{"raw":0.5545,"skill":0.2917,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1418,"skill":0.1304,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.275,"skill":0.1799,"coverage":1,"random":0.116,"tracks":[],"combined":0.215,"private_eq":0.3131},"37":{"raw":0.4122,"skill":0.2627,"coverage":1,"random":0.2027,"tracks":[],"combined":0.2945,"private_eq":0.4431},"38":{"raw":0.1012,"skill":0.0724,"coverage":1,"random":0.031,"tracks":[],"combined":0.1066,"private_eq":0.1432},"39":{"raw":0.6522,"skill":0.4885,"coverage":1,"random":0.3201,"tracks":[],"combined":0.2936,"private_eq":0.5245},"40":{"raw":0.1834,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3397,"headline":false},{"track":"A · English","score":0.1834,"headline":true},{"track":"C · Arabic pairs","score":0.885,"headline":false},{"track":"C · English pairs","score":0.83,"headline":false}],"combined":0.0387,"private_eq":0.2605},"41":{"raw":0.3715,"skill":0.0572,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0298,"private_eq":0.3599},"42":{"raw":0.6227,"skill":0.266,"coverage":1,"random":0.486,"tracks":[],"combined":0.0859,"private_eq":0.535},"43":{"raw":0.3298,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1063,"private_eq":0.4425},"44":{"raw":0.5324,"skill":0.0648,"coverage":1,"random":0.5,"tracks":[],"combined":0.1712,"private_eq":0.5898},"45":{"raw":0.1297,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4099,"skill":0.1436,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.9265,"skill":0.853,"coverage":1,"random":0.5,"tracks":[],"combined":0.4299,"private_eq":0.7177},"57":{"raw":0.2689,"skill":0.1777,"coverage":0.9999,"random":0.1109,"tracks":[],"private_eq":0.2569,"combined":0.1556},"58":{"raw":0.4511,"skill":0.2044,"coverage":0.9998,"random":0.3101,"tracks":[],"private_eq":0.3101,"combined":0},"59":{"raw":0.4976,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5855,"skill":0.171,"coverage":1,"random":0.5,"tracks":[],"combined":0.2861,"private_eq":0.6466},"62":{"raw":0.7117,"skill":0.6156,"coverage":1,"random":0.25,"tracks":[],"combined":0.4672,"private_eq":0.6042},"64":{"raw":0.392,"skill":0.24,"coverage":1,"random":0.2,"tracks":[],"combined":0.1573,"private_eq":0.3327}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #148).","Submitted public 0.3 run adopted (kit rescore 32.26, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.47% on 895 rows (bar 98.5%).","Training includes train/dev splits of 11 suite datasets (16.9% of questions), disclosed per dataset.","0.2.1 run resumed under 0.3 (only the rebuilt GSM8K rows newly run)."],"scores":{"balanced_raw":49.4,"balanced_skill":20.82,"breadth_skill":28.9,"public_skill":32.26},"frozen_scores":{"balanced_raw":49.4,"balanced_skill":32.26,"breadth_skill":28.9},"categories":[{"id":"knowledge","raw":0.3537,"skill":0.1432,"combined":0.0816,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.489,"skill":0.2659,"combined":0.2019,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.662,"skill":0.5477,"combined":0.4268,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5552,"skill":0.5122,"combined":0.399,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4211,"skill":0.1354,"combined":0.1314,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-0.8b","pub":32.26,"same_skills":26.01,"new_domains":9.33,"v":20.82,"lab":false,"jev":false,"rank_v":194,"rank_pub":183,"name":"Sieve 0.8B","eq":{"S":25.26,"O":14.17},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":15.16,"new_domains":5.67}}},{"engine":"jiwo-0.8b-v2-r2","name":"jiwo 0.8B v2","long_name":"jiwo 0.8B v2","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":752393024,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/eljiwo/jiwo-0.8b","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5581,"conf":0.6257,"ece":0.0676,"brier":0.5381,"over95":0.0022,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0582,"conf":0.0527,"acc":0.0239},{"lo":0.1,"w":0.0271,"conf":0.15,"acc":0.1271},{"lo":0.2,"w":0.0348,"conf":0.2581,"acc":0.2515},{"lo":0.3,"w":0.0629,"conf":0.3519,"acc":0.2668},{"lo":0.4,"w":0.0828,"conf":0.4514,"acc":0.3619},{"lo":0.5,"w":0.1668,"conf":0.5465,"acc":0.4943},{"lo":0.6,"w":0.1411,"conf":0.649,"acc":0.5734},{"lo":0.7,"w":0.1251,"conf":0.7491,"acc":0.6404},{"lo":0.8,"w":0.132,"conf":0.8518,"acc":0.7331},{"lo":0.9,"w":0.1693,"conf":0.9527,"acc":0.9309}],"areas":{"knowledge":{"ece":0.0788,"acc":0.4768,"conf":0.5556},"language":{"ece":0.0843,"acc":0.6516,"conf":0.7329},"retrieval":{"ece":0.0441,"acc":0.6594,"conf":0.6445},"tools":{"ece":0.0475,"acc":0.6535,"conf":0.6767},"arts":{"ece":0.1133,"acc":0.4144,"conf":0.5276}}},"gaps":null,"latency":{"median":36.3,"p95":75.3,"mean":61.6},"results":{"1":{"score":0.8388,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":111},"2":{"score":0.6107,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":344.2},"3":{"score":0.1909,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":163.4},"4":{"score":0.6073,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":53.2},"5":{"score":0.8157,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":55.2},"6":{"score":0.6668,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":75.7},"9":{"score":0.0795,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":316},"10":{"score":0.4876,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":50.3},"11":{"score":0.7854,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":501.5},"12":{"score":0.4712,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":46.2},"20":{"score":0.506,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.9},"21":{"score":0.5358,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":41.9},"22":{"score":0.005,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":126.9},"23":{"score":0.6168,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":42.1},"24":{"score":0.5073,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":41.5},"25":{"score":0.2602,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":42},"26":{"score":0.8573,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":41.8},"27":{"score":0.6792,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":41.6},"28":{"score":0.5533,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":45.7},"29":{"score":0.7435,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":47.8},"30":{"score":0.2324,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0766,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.9},"32":{"score":0.5691,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":47.7},"33":{"score":0.2206,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":121.7},"36":{"score":0.3876,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":504.9},"37":{"score":0.4941,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.4},"38":{"score":0.0981,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":184.1},"39":{"score":0.8156,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":49.6},"40":{"score":0.2585,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":47.4},"41":{"score":0.4234,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":45.6},"42":{"score":0.6279,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":48.9},"43":{"score":0.3,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":52.6},"44":{"score":0.5144,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.7},"45":{"score":0.1357,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":46.8},"50":{"score":0.3825,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":49.6},"56":{"score":0.656,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":62.9},"57":{"score":0.2901,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":45.9},"58":{"score":0.4787,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":46.4},"59":{"score":0.716,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":50},"61":{"score":0.586,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":47.5},"62":{"score":0.546,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":49.1},"64":{"score":0.4489,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":46}},"benchmarks":{"1":{"raw":0.8388,"skill":0.7824,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5992,"private_eq":0.7057},"2":{"raw":0.6107,"skill":0.5504,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6416,"private_eq":0.6923},"3":{"raw":0.1909,"skill":0.1753,"coverage":1,"random":0.0189,"tracks":[],"combined":0.1507,"private_eq":0.1753},"4":{"raw":0.6073,"skill":0.6022,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7102,"private_eq":0.7161},"5":{"raw":0.8157,"skill":0.8146,"coverage":1,"random":0.006,"tracks":[],"combined":0.5684,"private_eq":0.5749},"6":{"raw":0.6668,"skill":0.2188,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0795,"skill":0.0795,"coverage":1,"random":0,"tracks":[],"combined":0.0274,"private_eq":0.0375},"10":{"raw":0.4876,"skill":0.1474,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7854,"skill":0.6896,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4712,"skill":0.2079,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3334,"private_eq":0.5594},"20":{"raw":0.5033,"skill":0.0066,"coverage":1,"random":0.5,"tracks":[],"combined":0.1695,"private_eq":0.589},"21":{"raw":0.5358,"skill":0.0715,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0056,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0011,"private_eq":0.0192},"23":{"raw":0.6204,"skill":0.2407,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5073,"skill":0.3431,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3059,"combined":0.0649},"26":{"raw":0.8573,"skill":0.8097,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.6792,"skill":0.5722,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5533,"skill":0.1066,"coverage":1,"random":0.5,"tracks":[],"combined":0.1283,"private_eq":0.5686},"29":{"raw":0.7435,"skill":0.658,"coverage":1,"random":0.25,"tracks":[],"combined":0.4583,"private_eq":0.5976},"30":{"raw":0.2324,"skill":0.071,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3154,"random":0.25},{"track":"10-choice","raw":0.1494,"random":0.1}],"combined":0.1665,"private_eq":0.3194},"31":{"raw":0.0766,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0863},"32":{"raw":0.5691,"skill":0.3149,"coverage":1,"random":0.371,"tracks":[],"combined":0.0107,"private_eq":0.3842},"33":{"raw":0.2206,"skill":0.2103,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2355,"private_eq":0.2531},"36":{"raw":0.3876,"skill":0.3072,"coverage":1,"random":0.116,"tracks":[],"combined":0.2068,"private_eq":0.3059},"37":{"raw":0.4941,"skill":0.3655,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3386,"private_eq":0.4779},"38":{"raw":0.0981,"skill":0.0692,"coverage":1,"random":0.031,"tracks":[],"combined":0.1364,"private_eq":0.1717},"39":{"raw":0.8156,"skill":0.7288,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5426,"private_eq":0.6919},"40":{"raw":0.2585,"skill":0.046,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2841,"headline":false},{"track":"A · English","score":0.2585,"headline":true},{"track":"C · Arabic pairs","score":0.485,"headline":false},{"track":"C · English pairs","score":0.505,"headline":false}],"combined":0.0272,"private_eq":0.2517},"41":{"raw":0.4234,"skill":0.1351,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3413,"private_eq":0.5652},"42":{"raw":0.6279,"skill":0.2761,"coverage":1,"random":0.486,"tracks":[],"combined":0.732,"private_eq":0.8633},"43":{"raw":0.3,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2191,"private_eq":0.5128},"44":{"raw":0.5144,"skill":0.0288,"coverage":1,"random":0.5,"tracks":[],"combined":0.1121,"private_eq":0.5606},"45":{"raw":0.1357,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3825,"skill":0.1037,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.656,"skill":0.312,"coverage":1,"random":0.5,"tracks":[],"combined":0.4991,"private_eq":0.7519},"57":{"raw":0.2901,"skill":0.2016,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.377,"combined":0.2922},"58":{"raw":0.4787,"skill":0.2444,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.348,"combined":0.0451},"59":{"raw":0.716,"skill":0.4112,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2372,"private_eq":0.6358},"61":{"raw":0.586,"skill":0.172,"coverage":1,"random":0.5,"tracks":[],"combined":0.3316,"private_eq":0.6691},"62":{"raw":0.546,"skill":0.3947,"coverage":1,"random":0.25,"tracks":[],"combined":0.4192,"private_eq":0.5686},"64":{"raw":0.4489,"skill":0.3111,"coverage":1,"random":0.2,"tracks":[],"combined":0.3319,"private_eq":0.4708}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #113).","Replaces the earlier board entry jiwo-0.8b-v2 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 29.58, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.85% on 895 rows (bar 98.5%).","The weights behind the v2 tag were replaced after the lab's first run; this entry scores the new weights (71b7911).","Author check found shared source text with six index benchmarks.","0.2.1 run resumed under 0.3 (only the rebuilt GSM8K rows newly run)."],"scores":{"balanced_raw":47.2,"balanced_skill":25.81,"breadth_skill":26.75,"public_skill":29.58},"frozen_scores":{"balanced_raw":47.2,"balanced_skill":29.58,"breadth_skill":26.75},"categories":[{"id":"knowledge","raw":0.3312,"skill":0.1087,"combined":0.1273,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5513,"skill":0.3612,"combined":0.3532,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5922,"skill":0.4422,"combined":0.4472,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4577,"skill":0.4026,"combined":0.3681,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4161,"skill":0.1223,"combined":0.1274,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jiwo-0.8b-v2-r2","pub":29.58,"same_skills":30.66,"new_domains":13.71,"v":25.81,"lab":false,"jev":false,"rank_v":180,"rank_pub":191,"name":"jiwo 0.8B v2","eq":{"S":29.96,"O":19.57},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":17.98,"new_domains":7.83}}},{"engine":"ajev-lora7","name":"AJev lora7 (Gemma 4 12B)","long_name":"AJev lora7 (Gemma 4 12B)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":11960000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/andyzhang232/ajev-gemma4-12b-lora7","code_url":null,"base_model":"google/gemma-4-12B-it","base_url":"https://huggingface.co/google/gemma-4-12B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6949,"conf":0.7772,"ece":0.0823,"brier":0.4064,"over95":0.0118,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0119,"conf":0.0735,"acc":0.0538},{"lo":0.1,"w":0.0222,"conf":0.1501,"acc":0.0797},{"lo":0.2,"w":0.0175,"conf":0.2568,"acc":0.1435},{"lo":0.3,"w":0.0327,"conf":0.3481,"acc":0.1858},{"lo":0.4,"w":0.0439,"conf":0.452,"acc":0.3643},{"lo":0.5,"w":0.0869,"conf":0.5456,"acc":0.45},{"lo":0.6,"w":0.0945,"conf":0.6416,"acc":0.5587},{"lo":0.7,"w":0.0941,"conf":0.7455,"acc":0.5976},{"lo":0.8,"w":0.1341,"conf":0.8545,"acc":0.6976},{"lo":0.9,"w":0.4622,"conf":0.9618,"acc":0.9223}],"areas":{"knowledge":{"ece":0.102,"acc":0.6202,"conf":0.7222},"language":{"ece":0.0332,"acc":0.8425,"conf":0.8647},"retrieval":{"ece":0.2097,"acc":0.6201,"conf":0.8292},"tools":{"ece":0.063,"acc":0.8372,"conf":0.8912},"arts":{"ece":0.0668,"acc":0.5531,"conf":0.6199}}},"gaps":null,"latency":{"median":69.1,"p95":453,"mean":117.2},"results":{"1":{"score":0.9669,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":608.7},"2":{"score":0.652,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":13481.8},"3":{"score":0.5827,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":4906.5},"4":{"score":0.8069,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":828.1},"5":{"score":0.888,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1077.5},"6":{"score":0.7991,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":2909.3},"9":{"score":0.4432,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1745.5},"10":{"score":0.1099,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":594.7},"11":{"score":0.7947,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":4004.8},"12":{"score":0.6741,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":135.6},"20":{"score":0.9154,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":104.1},"21":{"score":0.6514,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":97.8},"22":{"score":0.0975,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":4271.4},"23":{"score":0.6312,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":130.2},"24":{"score":0.7794,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":132.8},"25":{"score":0.4235,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":209.8},"26":{"score":0.9827,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":159.4},"27":{"score":0.9445,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":160.1},"28":{"score":0.8477,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":158.5},"29":{"score":0.9267,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":237.4},"30":{"score":0.5694,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1586,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":566},"32":{"score":0.6383,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":482.1},"33":{"score":0.38,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":620.3},"36":{"score":0.4706,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":12221.4},"37":{"score":0.5514,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":322},"38":{"score":0.239,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":4084.9},"39":{"score":0.8247,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":238.1},"40":{"score":0.7154,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":129.5},"41":{"score":0.7968,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":201},"42":{"score":0.7961,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":145.8},"43":{"score":0.5825,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":153.4},"44":{"score":0.7044,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111.5},"45":{"score":0.1078,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":305.7},"50":{"score":0.4457,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":579.6},"56":{"score":0.778,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":783.6},"57":{"score":0.5615,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":211.3},"58":{"score":0.7481,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":200.3},"59":{"score":0.8043,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":312.9},"61":{"score":0.826,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":327.3},"62":{"score":0.8639,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":307.2},"64":{"score":0.7689,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":200.3}},"benchmarks":{"1":{"raw":0.9669,"skill":0.9554,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.652,"skill":0.5981,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6986,"private_eq":0.7411},"3":{"raw":0.5827,"skill":0.5747,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4552,"private_eq":0.4706},"4":{"raw":0.8069,"skill":0.8044,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8585,"private_eq":0.8609},"5":{"raw":0.888,"skill":0.8873,"coverage":1,"random":0.006,"tracks":[],"combined":0.6919,"private_eq":0.6962},"6":{"raw":0.7991,"skill":0.529,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4432,"skill":0.4432,"coverage":1,"random":0,"tracks":[],"combined":0.389,"private_eq":0.395},"10":{"raw":0.1099,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7947,"skill":0.7031,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6741,"skill":0.5118,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5042,"private_eq":0.6721},"20":{"raw":0.9171,"skill":0.8342,"coverage":1,"random":0.5,"tracks":[],"combined":0.8493,"private_eq":0.925},"21":{"raw":0.6514,"skill":0.3029,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.093,"skill":0.0859,"coverage":1,"random":0.0078,"tracks":[],"combined":0.093,"private_eq":0.1093},"23":{"raw":0.629,"skill":0.258,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7794,"skill":0.7059,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4235,"skill":0.2313,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3879,"combined":0.1755},"26":{"raw":0.9827,"skill":0.9769,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9445,"skill":0.926,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8477,"skill":0.6954,"coverage":1,"random":0.5,"tracks":[],"combined":0.4208,"private_eq":0.7132},"29":{"raw":0.9267,"skill":0.9023,"coverage":1,"random":0.25,"tracks":[],"combined":0.8346,"private_eq":0.8766},"30":{"raw":0.5694,"skill":0.4837,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6596,"random":0.25},{"track":"10-choice","raw":0.4792,"random":0.1}],"combined":0.6105,"private_eq":0.6815},"31":{"raw":0.1586,"skill":0.0836,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0799,"private_eq":0.164},"32":{"raw":0.6383,"skill":0.425,"coverage":1,"random":0.371,"tracks":[],"combined":0.2119,"private_eq":0.5093},"33":{"raw":0.38,"skill":0.3718,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3794,"private_eq":0.3935},"36":{"raw":0.4706,"skill":0.4011,"coverage":1,"random":0.116,"tracks":[],"combined":0.3444,"private_eq":0.4262},"37":{"raw":0.5514,"skill":0.4374,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4741,"private_eq":0.5847},"38":{"raw":0.239,"skill":0.2147,"coverage":1,"random":0.031,"tracks":[],"combined":0.3599,"private_eq":0.3858},"39":{"raw":0.8247,"skill":0.7422,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5156,"private_eq":0.6737},"40":{"raw":0.7154,"skill":0.6338,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3083,"headline":false},{"track":"A · English","score":0.7154,"headline":true},{"track":"C · Arabic pairs","score":0.815,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.5597,"private_eq":0.6609},"41":{"raw":0.7968,"skill":0.6953,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3289,"private_eq":0.557},"42":{"raw":0.7961,"skill":0.6033,"coverage":1,"random":0.486,"tracks":[],"combined":0.8235,"private_eq":0.9098},"43":{"raw":0.5825,"skill":0.3375,"coverage":1,"random":0.3697,"tracks":[],"combined":0.516,"private_eq":0.6978},"44":{"raw":0.7044,"skill":0.4088,"coverage":1,"random":0.5,"tracks":[],"combined":0.4603,"private_eq":0.7327},"45":{"raw":0.1078,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4457,"skill":0.1955,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.778,"skill":0.556,"coverage":1,"random":0.5,"tracks":[],"combined":0.3261,"private_eq":0.6664},"57":{"raw":0.5615,"skill":0.5068,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5205,"combined":0.4555},"58":{"raw":0.7481,"skill":0.6349,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4763,"combined":0.2332},"59":{"raw":0.8043,"skill":0.5942,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3162,"private_eq":0.6735},"61":{"raw":0.826,"skill":0.652,"coverage":1,"random":0.5,"tracks":[],"combined":0.6039,"private_eq":0.8037},"62":{"raw":0.8639,"skill":0.8185,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.7689,"skill":0.7111,"coverage":1,"random":0.2,"tracks":[],"combined":0.7296,"private_eq":0.7853}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #68).","Replaces the earlier board entry ajev-lora5 (newer version of the same model).","Public 0.3 suite run by the lab with the author's engine (55.17; 0.2.1 claim 55.06). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.23% on 727 rows (bar 98.5%).","Training includes train splits of seven board benchmarks (author-declared).","Continued from the lora5 adapter."],"scores":{"balanced_raw":65.63,"balanced_skill":48.91,"breadth_skill":53.9,"public_skill":55.17},"frozen_scores":{"balanced_raw":65.63,"balanced_skill":55.17,"breadth_skill":53.9},"categories":[{"id":"knowledge","raw":0.5216,"skill":0.3827,"combined":0.3518,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7332,"skill":0.6259,"combined":0.5369,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7203,"skill":0.6298,"combined":0.5572,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7072,"skill":0.6844,"combined":0.6134,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5842,"skill":0.3979,"combined":0.43,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"ajev-lora7","pub":55.17,"same_skills":50.51,"new_domains":36.08,"v":48.91,"lab":false,"jev":false,"rank_v":95,"rank_pub":99,"name":"AJev lora7 (Gemma 4 12B)","eq":{"S":50.04,"O":47.2},"rank_tie":22,"tied":true,"parts":{"public":0,"same_skills":30.03,"new_domains":18.88}}},{"engine":"bobcat-1.3","name":"Bobcat 1.3 (31B)","long_name":"Bobcat 1.3 (31B)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sanghwa-na/bobcat-1.3","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7264,"conf":0.8274,"ece":0.1021,"brier":0.3857,"over95":0.0229,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0012,"conf":0.0933,"acc":0.2421},{"lo":0.1,"w":0.0112,"conf":0.1536,"acc":0.0999},{"lo":0.2,"w":0.0154,"conf":0.2522,"acc":0.1055},{"lo":0.3,"w":0.0229,"conf":0.3536,"acc":0.3694},{"lo":0.4,"w":0.042,"conf":0.4543,"acc":0.2746},{"lo":0.5,"w":0.0749,"conf":0.552,"acc":0.4721},{"lo":0.6,"w":0.0668,"conf":0.6519,"acc":0.496},{"lo":0.7,"w":0.0918,"conf":0.7528,"acc":0.5539},{"lo":0.8,"w":0.1389,"conf":0.8553,"acc":0.6666},{"lo":0.9,"w":0.535,"conf":0.9752,"acc":0.9186}],"areas":{"knowledge":{"ece":0.118,"acc":0.6848,"conf":0.7981},"language":{"ece":0.0304,"acc":0.83,"conf":0.8599},"retrieval":{"ece":0.2216,"acc":0.626,"conf":0.8471},"tools":{"ece":0.0405,"acc":0.9377,"conf":0.9736},"arts":{"ece":0.1497,"acc":0.5764,"conf":0.7237}}},"gaps":null,"latency":{"median":63.8,"p95":767.1,"mean":182.7},"results":{"1":{"score":0.9734,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6608,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8583,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.7884,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9001,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7993,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.625,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0738,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7262,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7354,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.8742,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6343,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.262,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6402,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8652,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.992,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9735,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8414,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9273,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.7381,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2286,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6582,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3497,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4773,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.6053,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.2396,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9437,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.5217,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7072,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8497,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8053,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7928,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1218,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.463,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.8675,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7084,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7777,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.778,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.786,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.77,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7557,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9734,"skill":0.9641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6608,"skill":0.6083,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6981,"private_eq":0.7407},"3":{"raw":0.8583,"skill":0.8556,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9118,"private_eq":0.9135},"4":{"raw":0.7884,"skill":0.7857,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8696,"private_eq":0.8717},"5":{"raw":0.9001,"skill":0.8995,"coverage":1,"random":0.006,"tracks":[],"combined":0.783,"private_eq":0.7858},"6":{"raw":0.7993,"skill":0.5294,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.625,"skill":0.625,"coverage":1,"random":0,"tracks":[],"combined":0.6394,"private_eq":0.6425},"10":{"raw":0.0738,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7262,"skill":0.604,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7354,"skill":0.6037,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4777,"private_eq":0.6546},"20":{"raw":0.8765,"skill":0.7531,"coverage":1,"random":0.5,"tracks":[],"combined":0.7698,"private_eq":0.8857},"21":{"raw":0.6343,"skill":0.2686,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.2533,"skill":0.2474,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3264,"private_eq":0.3383},"23":{"raw":0.635,"skill":0.2699,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8652,"skill":0.8203,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5,"skill":0.3333,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5117,"combined":0.3424},"26":{"raw":0.992,"skill":0.9893,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9735,"skill":0.9647,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8414,"skill":0.6828,"coverage":1,"random":0.5,"tracks":[],"combined":0.6278,"private_eq":0.8155},"29":{"raw":0.9273,"skill":0.9031,"coverage":1,"random":0.25,"tracks":[],"combined":0.9236,"private_eq":0.9426},"30":{"raw":0.7381,"skill":0.6861,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7945,"random":0.25},{"track":"10-choice","raw":0.6816,"random":0.1}],"combined":0.7594,"private_eq":0.8029},"31":{"raw":0.2286,"skill":0.1598,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0918,"private_eq":0.1748},"32":{"raw":0.6582,"skill":0.4566,"coverage":1,"random":0.371,"tracks":[],"combined":0.3975,"private_eq":0.6247},"33":{"raw":0.3497,"skill":0.3411,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4616,"private_eq":0.4737},"36":{"raw":0.4773,"skill":0.4087,"coverage":1,"random":0.116,"tracks":[],"combined":0.4062,"private_eq":0.4802},"37":{"raw":0.6053,"skill":0.5049,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5697,"private_eq":0.66},"38":{"raw":0.2396,"skill":0.2153,"coverage":1,"random":0.031,"tracks":[],"combined":0.4083,"private_eq":0.4322},"39":{"raw":0.9437,"skill":0.9172,"coverage":1,"random":0.3201,"tracks":[],"combined":0.598,"private_eq":0.7291},"40":{"raw":0.5217,"skill":0.3847,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5116,"headline":false},{"track":"A · English","score":0.5217,"headline":true},{"track":"C · Arabic pairs","score":0.915,"headline":false},{"track":"C · English pairs","score":0.99,"headline":false}],"combined":0.436,"private_eq":0.5658},"41":{"raw":0.7072,"skill":0.5608,"coverage":1,"random":0.3333,"tracks":[],"combined":0.516,"private_eq":0.6803},"42":{"raw":0.8497,"skill":0.7076,"coverage":1,"random":0.486,"tracks":[],"combined":0.8826,"private_eq":0.9398},"43":{"raw":0.8053,"skill":0.691,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6122,"private_eq":0.7577},"44":{"raw":0.7928,"skill":0.5856,"coverage":1,"random":0.5,"tracks":[],"combined":0.5566,"private_eq":0.7803},"45":{"raw":0.1218,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.463,"skill":0.2206,"coverage":1,"random":0.311,"tracks":[],"combined":0.283,"private_eq":0.5109},"56":{"raw":0.8675,"skill":0.735,"coverage":1,"random":0.5,"tracks":[],"combined":0.6636,"private_eq":0.8332},"57":{"raw":0.7084,"skill":0.672,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5361,"combined":0.4732},"58":{"raw":0.7777,"skill":0.6778,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5478,"combined":0.338},"59":{"raw":0.778,"skill":0.5397,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3381,"private_eq":0.6839},"61":{"raw":0.786,"skill":0.572,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.77,"skill":0.6933,"coverage":1,"random":0.25,"tracks":[],"combined":0.5698,"private_eq":0.6803},"64":{"raw":0.7557,"skill":0.6946,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #146).","Submitted public 0.3 run adopted (kit rescore 59.31, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (vLLM online dynamic FP8 of the BF16 weights at load); decision agreement with the submitted rows 98.44% on 900 rows (bar 96%).","Served in FP8 (vLLM online quantization of the bf16 weights at load), as in the author's run, so the quantized agreement bar applies: 98.44% vs 96%. It would not pass the 98.5% bf16 bar.","Model selection used half of the public 0.3 suite's items, so the public score includes items used for selection (author-declared).","Uniform average of three merged LoRA fine-tunes of gemma-4-31B-it; the 1.2 lineage's ARC and BANKING77 train splits were removed for 1.3 (author-declared).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":69.08,"balanced_skill":64.85,"breadth_skill":58.38,"public_skill":59.31},"frozen_scores":{"balanced_raw":69.08,"balanced_skill":59.31,"breadth_skill":58.38},"categories":[{"id":"knowledge","raw":0.5927,"skill":0.4799,"combined":0.4645,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7193,"skill":0.6104,"combined":0.5786,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.736,"skill":0.6552,"combined":0.6561,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7877,"skill":0.7612,"combined":0.7646,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.603,"skill":0.409,"combined":0.4906,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"bobcat-1.3","pub":59.31,"same_skills":59.9,"new_domains":56.81,"v":64.85,"lab":false,"jev":false,"rank_v":30,"rank_pub":65,"name":"Bobcat 1.3 (31B)","eq":{"S":59.54,"O":72.8},"rank_tie":6,"tied":true,"parts":{"public":0,"same_skills":35.73,"new_domains":29.12}}},{"engine":"standard-one-8b-sh-v4","name":"Standard One 8B SH (v4)","long_name":"Standard One 8B SH (v4)","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":8918026240,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/StandardThinking/StandardOne-8B-SH","code_url":null,"base_model":"StandardThinking/StandardOne-8B","base_url":"https://huggingface.co/StandardThinking/StandardOne-8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"StandardThinking","name":"Standard Thinking","logo":"data/orgs/StandardThinking.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7365,"conf":0.8406,"ece":0.1065,"brier":0.3856,"over95":0.0392,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0083,"conf":0.0813,"acc":0.09},{"lo":0.1,"w":0.0154,"conf":0.1421,"acc":0.1765},{"lo":0.2,"w":0.0068,"conf":0.2455,"acc":0.2875},{"lo":0.3,"w":0.0209,"conf":0.3535,"acc":0.3682},{"lo":0.4,"w":0.0304,"conf":0.454,"acc":0.4252},{"lo":0.5,"w":0.0665,"conf":0.5516,"acc":0.4706},{"lo":0.6,"w":0.0709,"conf":0.6507,"acc":0.4943},{"lo":0.7,"w":0.0697,"conf":0.7524,"acc":0.5554},{"lo":0.8,"w":0.1018,"conf":0.8555,"acc":0.624},{"lo":0.9,"w":0.6093,"conf":0.9725,"acc":0.8893}],"areas":{"knowledge":{"ece":0.136,"acc":0.6258,"conf":0.7538},"language":{"ece":0.0934,"acc":0.827,"conf":0.9203},"retrieval":{"ece":0.144,"acc":0.743,"conf":0.8854},"tools":{"ece":0.0188,"acc":0.9447,"conf":0.9591},"arts":{"ece":0.1001,"acc":0.6647,"conf":0.7639}}},"gaps":null,"latency":{"median":89.9,"p95":219.9,"mean":132.6},"results":{"1":{"score":0.951,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":991.9},"2":{"score":0.6546,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":61911.4},"3":{"score":0.8681,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3959.3},"4":{"score":0.9116,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":418.6},"5":{"score":0.9462,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":518.1},"6":{"score":0.7767,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":855.6},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":926.8},"10":{"score":0.4591,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":637.8},"11":{"score":0.8536,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":901.8},"12":{"score":0.6434,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":330.9},"20":{"score":0.7772,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":289.8},"21":{"score":0.5373,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":284.6},"22":{"score":0.7865,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3025},"23":{"score":0.625,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":472.9},"24":{"score":0.74,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":326.9},"25":{"score":0.3571,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":309.4},"26":{"score":0.9701,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":271.4},"27":{"score":0.9155,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":254.9},"28":{"score":0.8461,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":235},"29":{"score":0.9264,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":257.8},"30":{"score":0.4219,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1958,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":435.9},"32":{"score":0.6449,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":565.6},"33":{"score":0.2376,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":554.8},"36":{"score":0.4179,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":15595.5},"37":{"score":0.5938,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":321.7},"38":{"score":0.2749,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3393.5},"39":{"score":0.8314,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":337.9},"40":{"score":0.5205,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":194.3},"41":{"score":0.7174,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":226.8},"42":{"score":0.7791,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":326.4},"43":{"score":0.5211,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":269.9},"44":{"score":0.894,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":226.9},"45":{"score":0.1078,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":288.9},"50":{"score":0.5865,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":342.8},"56":{"score":0.7625,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":327.9},"57":{"score":0.501,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":207.9},"58":{"score":0.7556,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":183.9},"59":{"score":0.751,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":336.8},"61":{"score":0.864,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":237.9},"62":{"score":0.8321,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":377.9},"64":{"score":0.6117,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":256.4}},"benchmarks":{"1":{"raw":0.951,"skill":0.9339,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8306,"private_eq":0.8752},"2":{"raw":0.6546,"skill":0.6011,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6912,"private_eq":0.7348},"3":{"raw":0.8681,"skill":0.8656,"coverage":1,"random":0.0189,"tracks":[],"combined":0.812,"private_eq":0.8167},"4":{"raw":0.9116,"skill":0.9105,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8009,"private_eq":0.8047},"5":{"raw":0.9462,"skill":0.9459,"coverage":1,"random":0.006,"tracks":[],"combined":0.6864,"private_eq":0.6908},"6":{"raw":0.7767,"skill":0.4764,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8468,"private_eq":0.8475},"10":{"raw":0.4591,"skill":0.1,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8536,"skill":0.7883,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6434,"skill":0.4658,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4418,"private_eq":0.6309},"20":{"raw":0.78,"skill":0.56,"coverage":1,"random":0.5,"tracks":[],"combined":0.8347,"private_eq":0.9178},"21":{"raw":0.5373,"skill":0.0746,"coverage":1,"random":0.5,"tracks":[],"combined":0.0051,"private_eq":0.5077},"22":{"raw":0.7823,"skill":0.7806,"coverage":1,"random":0.0078,"tracks":[],"combined":0.483,"private_eq":0.4919},"23":{"raw":0.6112,"skill":0.2224,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.74,"skill":0.6533,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3571,"skill":0.1429,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3579,"combined":0.135},"26":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9155,"skill":0.8873,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8461,"skill":0.6922,"coverage":1,"random":0.5,"tracks":[],"combined":0.4714,"private_eq":0.7382},"29":{"raw":0.9264,"skill":0.9019,"coverage":1,"random":0.25,"tracks":[],"combined":0.9038,"private_eq":0.9279},"30":{"raw":0.4219,"skill":0.3036,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5133,"random":0.25},{"track":"10-choice","raw":0.3306,"random":0.1}],"combined":0.4631,"private_eq":0.5613},"31":{"raw":0.1958,"skill":0.1241,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0453,"private_eq":0.1326},"32":{"raw":0.6449,"skill":0.4355,"coverage":1,"random":0.371,"tracks":[],"combined":0.2892,"private_eq":0.5574},"33":{"raw":0.2376,"skill":0.2275,"coverage":1,"random":0.0131,"tracks":[],"combined":0.143,"private_eq":0.1629},"36":{"raw":0.4179,"skill":0.3415,"coverage":1,"random":0.116,"tracks":[],"combined":0.3254,"private_eq":0.4096},"37":{"raw":0.5938,"skill":0.4905,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4854,"private_eq":0.5936},"38":{"raw":0.2749,"skill":0.2517,"coverage":1,"random":0.031,"tracks":[],"combined":0.383,"private_eq":0.408},"39":{"raw":0.8314,"skill":0.752,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4349,"private_eq":0.6195},"40":{"raw":0.5205,"skill":0.3831,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2572,"headline":false},{"track":"A · English","score":0.5205,"headline":true},{"track":"C · Arabic pairs","score":0.425,"headline":false},{"track":"C · English pairs","score":0.65,"headline":false}],"combined":0.35,"private_eq":0.4997},"41":{"raw":0.7174,"skill":0.576,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4488,"private_eq":0.636},"42":{"raw":0.7791,"skill":0.5702,"coverage":1,"random":0.486,"tracks":[],"combined":0.7814,"private_eq":0.8884},"43":{"raw":0.5211,"skill":0.2401,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4283,"private_eq":0.6431},"44":{"raw":0.894,"skill":0.788,"coverage":1,"random":0.5,"tracks":[],"combined":0.7763,"private_eq":0.8889},"45":{"raw":0.1078,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5865,"skill":0.3999,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.7625,"skill":0.525,"coverage":1,"random":0.5,"tracks":[],"combined":0.5337,"private_eq":0.769},"57":{"raw":0.501,"skill":0.4388,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3662,"combined":0.2799},"58":{"raw":0.7556,"skill":0.6457,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.376,"combined":0.0861},"59":{"raw":0.751,"skill":0.4837,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.864,"skill":0.728,"coverage":1,"random":0.5,"tracks":[],"combined":0.5414,"private_eq":0.7728},"62":{"raw":0.8321,"skill":0.7761,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"64":{"raw":0.6117,"skill":0.5146,"coverage":1,"random":0.2,"tracks":[],"combined":0.4482,"private_eq":0.5628}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #139).","Replaces the earlier board entry standard-one-8b-sh-v3 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 57.40, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.70% on 830 rows (bar 98.5%).","Training data includes train splits of index benchmark source datasets in request format; no item-level overlap check (author-disclosed).","The v4 rounds add decision data built from public sources the index also draws on, plus LLM-written synthetic items; evaluated items held out per author, no overlap check.","Submitted on AMD MI350X/ROCm with the author's SGLang fork; the card ran the same fork on CUDA."],"scores":{"balanced_raw":68.44,"balanced_skill":42.93,"breadth_skill":55.38,"public_skill":57.4},"frozen_scores":{"balanced_raw":68.44,"balanced_skill":57.4,"breadth_skill":55.38},"categories":[{"id":"knowledge","raw":0.4997,"skill":0.3685,"combined":0.3067,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7034,"skill":0.5794,"combined":0.4764,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7502,"skill":0.6638,"combined":0.566,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8647,"skill":0.8401,"combined":0.7503,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6515,"skill":0.4253,"combined":0.3934,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"standard-one-8b-sh-v4","pub":57.4,"same_skills":50.43,"new_domains":24.07,"v":42.93,"lab":false,"jev":false,"rank_v":122,"rank_pub":87,"name":"Standard One 8B SH (v4)","eq":{"S":49.96,"O":32.37},"rank_tie":27,"tied":true,"parts":{"public":0,"same_skills":29.98,"new_domains":12.95}}},{"engine":"jev-35b","name":"JEV-35B","long_name":"JEV-35B","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":35951822704,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/autotrust/JEV-35B","code_url":null,"base_model":"Qwen/Qwen3.5-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","weights_repo":null,"served_checkpoint":null,"org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7412,"conf":0.8257,"ece":0.0846,"brier":0.3587,"over95":0.0282,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0004,"conf":0.0838,"acc":0.25},{"lo":0.1,"w":0.0077,"conf":0.1566,"acc":0.1338},{"lo":0.2,"w":0.0161,"conf":0.2545,"acc":0.125},{"lo":0.3,"w":0.0296,"conf":0.3554,"acc":0.2792},{"lo":0.4,"w":0.04,"conf":0.4536,"acc":0.3607},{"lo":0.5,"w":0.0926,"conf":0.5478,"acc":0.4317},{"lo":0.6,"w":0.081,"conf":0.651,"acc":0.5437},{"lo":0.7,"w":0.0843,"conf":0.7511,"acc":0.6291},{"lo":0.8,"w":0.0953,"conf":0.8519,"acc":0.7301},{"lo":0.9,"w":0.5529,"conf":0.9832,"acc":0.92}],"areas":{"knowledge":{"ece":0.1438,"acc":0.6511,"conf":0.7949},"language":{"ece":0.0351,"acc":0.8536,"conf":0.8884},"retrieval":{"ece":0.1181,"acc":0.7735,"conf":0.8914},"tools":{"ece":0.0497,"acc":0.938,"conf":0.9827},"arts":{"ece":0.0579,"acc":0.5752,"conf":0.6274}}},"gaps":null,"latency":{"median":50.5,"p95":258.3,"mean":90.2},"results":{"1":{"score":0.9628,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":50.1},"2":{"score":0.6533,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":269.4},"3":{"score":0.8583,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":117.1},"4":{"score":0.9224,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":10.9},"5":{"score":0.9428,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":17.7},"6":{"score":0.7984,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":83.1},"9":{"score":0.5,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":80.2},"10":{"score":0.607,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":18.8},"11":{"score":0.8145,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":84.3},"12":{"score":0.696,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":2.6},"20":{"score":0.8714,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":2.6},"21":{"score":0.6164,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":1.9},"22":{"score":0.1415,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":62.3},"23":{"score":0.6216,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.6},"24":{"score":0.8645,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":2},"25":{"score":0.5204,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":2.5},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":2.1},"27":{"score":0.977,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":2},"28":{"score":0.9203,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":1.9},"29":{"score":0.9717,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":3.7},"30":{"score":0.4621,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.198,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.8},"32":{"score":0.5984,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":15.3},"33":{"score":0.2933,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":55.3},"36":{"score":0.4882,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":217.4},"37":{"score":0.6172,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5},"38":{"score":0.2985,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":106.7},"39":{"score":0.9017,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":4.7},"40":{"score":0.5536,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":3.1},"41":{"score":0.6992,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":2.7},"42":{"score":0.8077,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":6.5},"43":{"score":0.714,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":2.5},"44":{"score":0.7204,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":2.5},"45":{"score":0.1078,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":6.8},"50":{"score":0.5979,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":10.5},"56":{"score":0.83,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":25.2},"57":{"score":0.6818,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":3.2},"58":{"score":0.7567,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":2.9},"59":{"score":0.8386,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":11.7},"61":{"score":0.8808,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":5.9},"62":{"score":0.8968,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":8.8},"64":{"score":0.7955,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":4.1}},"benchmarks":{"1":{"raw":0.9628,"skill":0.9498,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9329,"private_eq":0.9501},"2":{"raw":0.6533,"skill":0.5996,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6876,"private_eq":0.7317},"3":{"raw":0.8583,"skill":0.8556,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8487,"private_eq":0.8523},"4":{"raw":0.9224,"skill":0.9214,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8849,"private_eq":0.8867},"5":{"raw":0.9428,"skill":0.9425,"coverage":1,"random":0.006,"tracks":[],"combined":0.7528,"private_eq":0.7561},"6":{"raw":0.7984,"skill":0.5273,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5,"skill":0.5,"coverage":1,"random":0,"tracks":[],"combined":0.4194,"private_eq":0.425},"10":{"raw":0.607,"skill":0.3461,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8145,"skill":0.7317,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.696,"skill":0.5446,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5633,"private_eq":0.7111},"20":{"raw":0.8748,"skill":0.7497,"coverage":1,"random":0.5,"tracks":[],"combined":0.8276,"private_eq":0.9143},"21":{"raw":0.6164,"skill":0.2329,"coverage":1,"random":0.5,"tracks":[],"combined":0.1459,"private_eq":0.5773},"22":{"raw":0.1385,"skill":0.1318,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1623,"private_eq":0.1773},"23":{"raw":0.6281,"skill":0.2562,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8645,"skill":0.8193,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5204,"skill":0.3605,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4587,"combined":0.271},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.977,"skill":0.9693,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9203,"skill":0.8406,"coverage":1,"random":0.5,"tracks":[],"combined":0.5319,"private_eq":0.7681},"29":{"raw":0.9717,"skill":0.9623,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.4621,"skill":0.36,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6194,"random":0.25},{"track":"10-choice","raw":0.3048,"random":0.1}],"combined":0.5492,"private_eq":0.6315},"31":{"raw":0.198,"skill":0.1265,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1167,"private_eq":0.1974},"32":{"raw":0.5984,"skill":0.3615,"coverage":1,"random":0.371,"tracks":[],"combined":0.328,"private_eq":0.5815},"33":{"raw":0.2933,"skill":0.2839,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2406,"private_eq":0.2581},"36":{"raw":0.4882,"skill":0.421,"coverage":1,"random":0.116,"tracks":[],"combined":0.3684,"private_eq":0.4471},"37":{"raw":0.6172,"skill":0.5198,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5725,"private_eq":0.6622},"38":{"raw":0.2985,"skill":0.2761,"coverage":1,"random":0.031,"tracks":[],"combined":0.4647,"private_eq":0.4862},"39":{"raw":0.9017,"skill":0.8554,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5461,"private_eq":0.6942},"40":{"raw":0.5536,"skill":0.4258,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4796,"headline":false},{"track":"A · English","score":0.5536,"headline":true},{"track":"C · Arabic pairs","score":0.66,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}],"combined":0.4021,"private_eq":0.5398},"41":{"raw":0.6992,"skill":0.5488,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5293,"private_eq":0.6891},"42":{"raw":0.8077,"skill":0.6259,"coverage":1,"random":0.486,"tracks":[],"combined":0.8987,"private_eq":0.948},"43":{"raw":0.714,"skill":0.5463,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5119,"private_eq":0.6952},"44":{"raw":0.7204,"skill":0.4408,"coverage":1,"random":0.5,"tracks":[],"combined":0.4388,"private_eq":0.7221},"45":{"raw":0.1078,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5979,"skill":0.4163,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.83,"skill":0.66,"coverage":1,"random":0.5,"tracks":[],"combined":0.5424,"private_eq":0.7733},"57":{"raw":0.6818,"skill":0.6421,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.542,"combined":0.48},"58":{"raw":0.7567,"skill":0.6473,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4642,"combined":0.2154},"59":{"raw":0.8386,"skill":0.6654,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3968,"private_eq":0.7119},"61":{"raw":0.8808,"skill":0.7616,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.8968,"skill":0.8624,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.7955,"skill":0.7444,"coverage":1,"random":0.2,"tracks":[],"combined":0.7489,"private_eq":0.8006}},"coverage":1,"pending":0,"notes":["New in the 2026-10-12 PR batch (PR #141).","Submitted public 0.3 run adopted (kit rescore 59.73, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.42% on 831 rows (bar 98.5%).","Listed under a maintainer exception to the agreement bar: 98.42% vs 98.5% (31 of 2,022 decisions flipped, 30 of them near-ties).","Training: LoRA on Qwen3.5-35B-A3B plus a decision head, using public training splits of suite benchmark datasets and distillation from Jev 1.13 answer distributions (author-declared).","The author's run used an unreleased vLLM build; the card ran vLLM 0.29.0."],"scores":{"balanced_raw":69.22,"balanced_skill":57.61,"breadth_skill":58.34,"public_skill":59.73},"frozen_scores":{"balanced_raw":69.22,"balanced_skill":59.73,"breadth_skill":58.34},"categories":[{"id":"knowledge","raw":0.5475,"skill":0.4256,"combined":0.3688,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7356,"skill":0.6316,"combined":0.6089,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7806,"skill":0.7096,"combined":0.6312,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7843,"skill":0.7645,"combined":0.7153,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6085,"skill":0.4219,"combined":0.4219,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-35b","pub":59.73,"same_skills":56.18,"new_domains":46.73,"v":57.61,"lab":false,"jev":false,"rank_v":66,"rank_pub":62,"name":"JEV-35B","eq":{"S":55.78,"O":60.36},"rank_tie":14,"tied":true,"parts":{"public":0,"same_skills":33.47,"new_domains":24.15}}},{"engine":"jev-1.13.0","name":"Jev 1.13","long_name":"Jev 1.13","variant":null,"meta":{"kind":"hosted API","technique":null,"technique_name":null,"served_params":null,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":null,"base_model":null,"base_url":null,"weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7377,"conf":0.8146,"ece":0.078,"brier":0.3577,"over95":0.0213,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0002,"conf":0.082,"acc":0.4},{"lo":0.1,"w":0.0188,"conf":0.1646,"acc":0.0982},{"lo":0.2,"w":0.0303,"conf":0.2417,"acc":0.1844},{"lo":0.3,"w":0.0255,"conf":0.3499,"acc":0.3687},{"lo":0.4,"w":0.0289,"conf":0.4517,"acc":0.3659},{"lo":0.5,"w":0.0827,"conf":0.5466,"acc":0.476},{"lo":0.6,"w":0.0819,"conf":0.6437,"acc":0.5148},{"lo":0.7,"w":0.0849,"conf":0.7465,"acc":0.5798},{"lo":0.8,"w":0.103,"conf":0.8487,"acc":0.7003},{"lo":0.9,"w":0.5438,"conf":0.9811,"acc":0.9329}],"areas":{"knowledge":{"ece":0.0773,"acc":0.708,"conf":0.783},"language":{"ece":0.0615,"acc":0.8248,"conf":0.8861},"retrieval":{"ece":0.1536,"acc":0.6685,"conf":0.8206},"tools":{"ece":0.0219,"acc":0.9471,"conf":0.9543},"arts":{"ece":0.0934,"acc":0.5701,"conf":0.6605}}},"gaps":null,"latency":{"median":524,"p95":536.1,"mean":524},"results":{"1":{"score":0.9575,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":876.6},"2":{"score":0.6528,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1020.1},"3":{"score":0.8819,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":805.2},"4":{"score":0.7974,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":920.4},"5":{"score":0.8927,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":913.6},"6":{"score":0.7992,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":971.2},"9":{"score":0.5227,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":758.8},"10":{"score":0.4295,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":951},"11":{"score":0.7169,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":751.7},"12":{"score":0.7479,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":948.9},"20":{"score":0.906,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":971.5},"21":{"score":0.6187,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":932},"22":{"score":0.181,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1017.3},"23":{"score":0.6474,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":956.1},"24":{"score":0.9173,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":946.9},"25":{"score":0.7857,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":765.9},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":908.3},"27":{"score":0.9778,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":872.1},"28":{"score":0.9195,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":909},"29":{"score":0.9452,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":955.1},"30":{"score":0.7176,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1722,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":936.6},"32":{"score":0.6609,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":831.3},"33":{"score":0.2642,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":868.3},"36":{"score":0.4752,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1095.1},"37":{"score":0.5521,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":939},"38":{"score":0.2952,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1020.3},"39":{"score":0.8698,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":806.7},"40":{"score":0.5051,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":970.9},"41":{"score":0.6463,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":894.1},"42":{"score":0.8406,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":936.7},"43":{"score":0.7298,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":802.7},"44":{"score":0.7264,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":944.9},"45":{"score":0.2036,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":718.6},"50":{"score":0.4594,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":911.4},"56":{"score":0.6255,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1372.4},"57":{"score":0.827,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":1053.5},"58":{"score":0.9292,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":1032.4},"59":{"score":0.7653,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":1039.7},"61":{"score":0.7285,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":945},"62":{"score":0.8097,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":1047.2},"64":{"score":0.7008,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":1043.8}},"benchmarks":{"1":{"raw":0.9575,"skill":0.9426,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8512,"private_eq":0.8903},"2":{"raw":0.6528,"skill":0.5991,"coverage":1,"random":0.1341,"tracks":[],"combined":0.652,"private_eq":0.7012},"3":{"raw":0.8819,"skill":0.8796,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8645,"private_eq":0.8676},"4":{"raw":0.7974,"skill":0.7948,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8681,"private_eq":0.8703},"5":{"raw":0.8927,"skill":0.8921,"coverage":1,"random":0.006,"tracks":[],"combined":0.7577,"private_eq":0.7609},"6":{"raw":0.7992,"skill":0.5292,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5227,"skill":0.5227,"coverage":1,"random":0,"tracks":[],"combined":0.5812,"private_eq":0.585},"10":{"raw":0.4295,"skill":0.0507,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7169,"skill":0.5906,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7479,"skill":0.6224,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4837,"private_eq":0.6586},"20":{"raw":0.9092,"skill":0.8184,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6187,"skill":0.2374,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.1658,"skill":0.1593,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2937,"private_eq":0.3062},"23":{"raw":0.6441,"skill":0.2883,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.9173,"skill":0.8897,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.7857,"skill":0.7143,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5553,"combined":0.4012},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9778,"skill":0.9704,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9195,"skill":0.839,"coverage":1,"random":0.5,"tracks":[],"combined":0.6278,"private_eq":0.8155},"29":{"raw":0.9452,"skill":0.9269,"coverage":1,"random":0.25,"tracks":[],"combined":0.9731,"private_eq":0.9793},"30":{"raw":0.7176,"skill":0.662,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7824,"random":0.25},{"track":"10-choice","raw":0.6528,"random":0.1}],"combined":0.7459,"private_eq":0.7919},"31":{"raw":0.1722,"skill":0.0984,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.6609,"skill":0.4609,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.2642,"skill":0.2544,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0967,"private_eq":0.1177},"36":{"raw":0.4752,"skill":0.4063,"coverage":1,"random":0.116,"tracks":[],"combined":0.382,"private_eq":0.459},"37":{"raw":0.5521,"skill":0.4382,"coverage":1,"random":0.2027,"tracks":[],"combined":0.626,"private_eq":0.7044},"38":{"raw":0.2952,"skill":0.2727,"coverage":1,"random":0.031,"tracks":[],"combined":0.445,"private_eq":0.4674},"39":{"raw":0.8698,"skill":0.8085,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6062,"private_eq":0.7346},"40":{"raw":0.5051,"skill":0.3633,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4449,"headline":false},{"track":"A · English","score":0.5051,"headline":true},{"track":"C · Arabic pairs","score":0.7,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0.3657,"private_eq":0.5118},"41":{"raw":0.6463,"skill":0.4695,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4824,"private_eq":0.6582},"42":{"raw":0.8406,"skill":0.6899,"coverage":1,"random":0.486,"tracks":[],"combined":0.8389,"private_eq":0.9176},"43":{"raw":0.7298,"skill":0.5713,"coverage":1,"random":0.3697,"tracks":[],"combined":0.516,"private_eq":0.6978},"44":{"raw":0.7264,"skill":0.4528,"coverage":1,"random":0.5,"tracks":[],"combined":0.4924,"private_eq":0.7486},"45":{"raw":0.2036,"skill":0.0473,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4594,"skill":0.2154,"coverage":1,"random":0.311,"tracks":[],"combined":0.2539,"private_eq":0.4911},"56":{"raw":0.6255,"skill":0.251,"coverage":1,"random":0.5,"tracks":[],"combined":0.5596,"private_eq":0.7818},"57":{"raw":0.827,"skill":0.8054,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6592,"combined":0.6133},"58":{"raw":0.9292,"skill":0.8974,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5256,"combined":0.3055},"59":{"raw":0.7653,"skill":0.5134,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3741,"private_eq":0.7011},"61":{"raw":0.7285,"skill":0.457,"coverage":1,"random":0.5,"tracks":[],"combined":0.6152,"private_eq":0.8093},"62":{"raw":0.8097,"skill":0.7463,"coverage":1,"random":0.25,"tracks":[],"combined":0.6589,"private_eq":0.7463},"64":{"raw":0.7008,"skill":0.626,"coverage":1,"random":0.2,"tracks":[],"combined":0.7296,"private_eq":0.7853}},"coverage":1,"pending":0,"notes":["TypeSafe hosted API, model id jev-1.13.0: the stored rows of the earlier reference run, scored with the same chain as every row.","Jev 1.14 is the reference model for the board's comparisons.","Latency is a hosted HTTP round trip, not a local GPU run."],"scores":{"balanced_raw":68.55,"balanced_skill":59.22,"breadth_skill":57.27,"public_skill":57.96},"frozen_scores":{"balanced_raw":68.55,"balanced_skill":57.96,"breadth_skill":57.27},"categories":[{"id":"knowledge","raw":0.6391,"skill":0.5387,"combined":0.4309,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7097,"skill":0.5922,"combined":0.5786,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6825,"skill":0.5542,"combined":0.6379,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7764,"skill":0.7509,"combined":0.7317,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.583,"skill":0.3908,"combined":0.4898,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-1.13.0","pub":57.96,"same_skills":58.04,"new_domains":47.72,"v":59.22,"lab":false,"jev":false,"rank_v":56,"rank_pub":80,"name":"Jev 1.13","eq":{"S":57.66,"O":61.58},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":34.59,"new_domains":24.63}}},{"engine":"startlux-decision-v1.1-27b-instant","name":"StartLux-Decision v1.1 27B","long_name":"StartLux-Decision v1.1 27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/startlux-models/StartLux-Decision-v1.1-27B-Instant","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"startlux-models","name":"StartLux","logo":"data/orgs/startlux-models.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8283,"conf":0.8534,"ece":0.0261,"brier":0.2388,"over95":0.0128,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0002,"conf":0.0903,"acc":0.2},{"lo":0.1,"w":0.0053,"conf":0.1462,"acc":0.1345},{"lo":0.2,"w":0.0135,"conf":0.2574,"acc":0.1628},{"lo":0.3,"w":0.0243,"conf":0.3557,"acc":0.3756},{"lo":0.4,"w":0.0377,"conf":0.4553,"acc":0.4212},{"lo":0.5,"w":0.0805,"conf":0.5514,"acc":0.505},{"lo":0.6,"w":0.0627,"conf":0.6487,"acc":0.6198},{"lo":0.7,"w":0.0659,"conf":0.7519,"acc":0.7055},{"lo":0.8,"w":0.0878,"conf":0.8546,"acc":0.8048},{"lo":0.9,"w":0.6221,"conf":0.9864,"acc":0.9703}],"areas":{"knowledge":{"ece":0.0803,"acc":0.7416,"conf":0.8146},"language":{"ece":0.008,"acc":0.9158,"conf":0.9236},"retrieval":{"ece":0.0145,"acc":0.8237,"conf":0.8278},"tools":{"ece":0.0431,"acc":0.9632,"conf":0.9273},"arts":{"ece":0.0387,"acc":0.7666,"conf":0.7805}}},"gaps":null,"latency":{"median":79.8,"p95":853.5,"mean":265.9},"results":{"1":{"score":0.9793,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":557},"2":{"score":0.6955,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":5492.8},"3":{"score":0.8465,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":4129.1},"4":{"score":0.9255,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":231.1},"5":{"score":0.9375,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":307.5},"6":{"score":0.7962,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":752.6},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1274.7},"10":{"score":0.7306,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":414.6},"11":{"score":0.8672,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1176.3},"12":{"score":0.8027,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":104.3},"20":{"score":0.9814,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":97.2},"21":{"score":0.6884,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":90.2},"22":{"score":0.8555,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3191.2},"23":{"score":0.7206,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":104.1},"24":{"score":0.8752,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":95.3},"25":{"score":0.5204,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":97.8},"26":{"score":0.9912,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":95.9},"27":{"score":0.9744,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":96.4},"28":{"score":0.9511,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":95.4},"29":{"score":0.9807,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":97},"30":{"score":0.8332,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2536,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":313.9},"32":{"score":0.7527,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":302.6},"33":{"score":0.5976,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":872.9},"36":{"score":0.4883,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1165.7},"37":{"score":0.6499,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":422.1},"38":{"score":0.6571,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":688.4},"39":{"score":0.9583,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":311.2},"40":{"score":0.6224,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":167.6},"41":{"score":0.811,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":112.5},"42":{"score":0.8726,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":140.4},"43":{"score":0.8667,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":73.8},"44":{"score":0.943,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.4},"45":{"score":0.1756,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":99.4},"50":{"score":0.6504,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":133},"56":{"score":0.9175,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":244.7},"57":{"score":0.7289,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":97.9},"58":{"score":0.9165,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":95.8},"59":{"score":0.8613,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":157.1},"61":{"score":0.9008,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":135.8},"62":{"score":0.9381,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":151.5},"64":{"score":0.8447,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":98}},"benchmarks":{"1":{"raw":0.9793,"skill":0.9721,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9805,"private_eq":0.985},"2":{"raw":0.6955,"skill":0.6483,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7035,"private_eq":0.7453},"3":{"raw":0.8465,"skill":0.8435,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9276,"private_eq":0.9288},"4":{"raw":0.9255,"skill":0.9245,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8947,"private_eq":0.8962},"5":{"raw":0.9375,"skill":0.9371,"coverage":1,"random":0.006,"tracks":[],"combined":0.8499,"private_eq":0.8515},"6":{"raw":0.7962,"skill":0.5222,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9504,"private_eq":0.95},"10":{"raw":0.7306,"skill":0.5517,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8672,"skill":0.808,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.8027,"skill":0.7045,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5286,"private_eq":0.6882},"20":{"raw":0.9821,"skill":0.9642,"coverage":1,"random":0.5,"tracks":[],"combined":0.9794,"private_eq":0.9893},"21":{"raw":0.6884,"skill":0.3767,"coverage":1,"random":0.5,"tracks":[],"combined":0.3472,"private_eq":0.6768},"22":{"raw":0.855,"skill":0.8539,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6343,"private_eq":0.6403},"23":{"raw":0.7395,"skill":0.479,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8752,"skill":0.8336,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5204,"skill":0.3605,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5417,"combined":0.3829},"26":{"raw":0.9912,"skill":0.9883,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9744,"skill":0.9659,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9511,"skill":0.9022,"coverage":1,"random":0.5,"tracks":[],"combined":0.6126,"private_eq":0.808},"29":{"raw":0.9807,"skill":0.9743,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8332,"skill":0.8009,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8764,"random":0.25},{"track":"10-choice","raw":0.79,"random":0.1}],"combined":0.7476,"private_eq":0.7933},"31":{"raw":0.2536,"skill":0.1871,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0962,"private_eq":0.1788},"32":{"raw":0.7527,"skill":0.6068,"coverage":1,"random":0.371,"tracks":[],"combined":0.4285,"private_eq":0.644},"33":{"raw":0.5976,"skill":0.5923,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4616,"private_eq":0.4737},"36":{"raw":0.4883,"skill":0.4211,"coverage":1,"random":0.116,"tracks":[],"combined":0.4229,"private_eq":0.4948},"37":{"raw":0.6499,"skill":0.5608,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5902,"private_eq":0.6762},"38":{"raw":0.6571,"skill":0.6461,"coverage":1,"random":0.031,"tracks":[],"combined":0.7535,"private_eq":0.7629},"39":{"raw":0.9583,"skill":0.9387,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5593,"private_eq":0.7031},"40":{"raw":0.6224,"skill":0.5142,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5475,"headline":false},{"track":"A · English","score":0.6224,"headline":true},{"track":"C · Arabic pairs","score":0.96,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.5711,"private_eq":0.6696},"41":{"raw":0.811,"skill":0.7166,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2373,"private_eq":0.4966},"42":{"raw":0.8726,"skill":0.7521,"coverage":1,"random":0.486,"tracks":[],"combined":0.9457,"private_eq":0.9719},"43":{"raw":0.8667,"skill":0.7885,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6372,"private_eq":0.7733},"44":{"raw":0.943,"skill":0.886,"coverage":1,"random":0.5,"tracks":[],"combined":0.7815,"private_eq":0.8915},"45":{"raw":0.1756,"skill":0.0138,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6504,"skill":0.4926,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.9175,"skill":0.835,"coverage":1,"random":0.5,"tracks":[],"combined":0.7241,"private_eq":0.8631},"57":{"raw":0.7289,"skill":0.6951,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5742,"combined":0.5166},"58":{"raw":0.9165,"skill":0.879,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5412,"combined":0.3283},"59":{"raw":0.8613,"skill":0.7124,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2525,"private_eq":0.6431},"61":{"raw":0.9008,"skill":0.8016,"coverage":1,"random":0.5,"tracks":[],"combined":0.6605,"private_eq":0.8317},"62":{"raw":0.9381,"skill":0.9175,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.8447,"skill":0.8059,"coverage":1,"random":0.2,"tracks":[],"combined":0.8264,"private_eq":0.8619}},"coverage":1,"pending":0,"notes":["New in the 2026-10-14 PR batch (PR #172).","Submitted public 0.3 run adopted (kit rescore 72.88, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.85% on 831 rows (bar 98.5%).","Public 0.3 scores guided the recipe and the seed-average choice; training data targets 33 of 38 index benchmarks with official train splits or same-layout generators (author-disclosed).","License CC BY-NC 4.0."],"scores":{"balanced_raw":79.39,"balanced_skill":67.92,"breadth_skill":72.36,"public_skill":72.88},"frozen_scores":{"balanced_raw":79.39,"balanced_skill":72.88,"breadth_skill":72.36},"categories":[{"id":"knowledge","raw":0.6831,"skill":0.6069,"combined":0.4964,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8287,"skill":0.7556,"combined":0.6137,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.8015,"skill":0.748,"combined":0.6933,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8934,"skill":0.8786,"combined":0.8409,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7933,"skill":0.662,"combined":0.6024,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"startlux-decision-v1.1-27b-instant","pub":72.88,"same_skills":64.82,"new_domains":56.98,"v":67.92,"lab":false,"jev":false,"rank_v":13,"rank_pub":3,"name":"StartLux-Decision v1.1 27B","eq":{"S":64.51,"O":73.02},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.71,"new_domains":29.21}}},{"engine":"pplx-decider-v1.2","name":"Perplexity Decider v1.2 (27B)","long_name":"Perplexity Decider v1.2 (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/perplexity-ai/pplx-decider-v1.2","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"perplexity-ai","name":"Perplexity","logo":"data/orgs/perplexity-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7877,"conf":0.8206,"ece":0.0551,"brier":0.2996,"over95":0.018,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0143,"conf":0.0766,"acc":0.1721},{"lo":0.1,"w":0.0227,"conf":0.1452,"acc":0.2894},{"lo":0.2,"w":0.0166,"conf":0.2485,"acc":0.4783},{"lo":0.3,"w":0.0248,"conf":0.3512,"acc":0.4578},{"lo":0.4,"w":0.0409,"conf":0.4537,"acc":0.4116},{"lo":0.5,"w":0.0679,"conf":0.547,"acc":0.5273},{"lo":0.6,"w":0.07,"conf":0.6484,"acc":0.5916},{"lo":0.7,"w":0.0702,"conf":0.7494,"acc":0.6632},{"lo":0.8,"w":0.0893,"conf":0.8554,"acc":0.7226},{"lo":0.9,"w":0.5833,"conf":0.9828,"acc":0.9502}],"areas":{"knowledge":{"ece":0.0833,"acc":0.7161,"conf":0.7897},"language":{"ece":0.0145,"acc":0.8985,"conf":0.9073},"retrieval":{"ece":0.1383,"acc":0.7122,"conf":0.8474},"tools":{"ece":0.0372,"acc":0.9557,"conf":0.9858},"arts":{"ece":0.0922,"acc":0.6911,"conf":0.6056}}},"gaps":null,"latency":{"median":144.4,"p95":1552.2,"mean":415.1},"results":{"1":{"score":0.9776,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":180.1},"2":{"score":0.6745,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1808.9},"3":{"score":0.874,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":603.3},"4":{"score":0.9103,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":122.2},"5":{"score":0.944,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":133.2},"6":{"score":0.7977,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":339.8},"9":{"score":0.9091,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2007.5},"10":{"score":0.3632,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":126},"11":{"score":0.8399,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3047.7},"12":{"score":0.7549,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":120.6},"20":{"score":0.9512,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.5},"21":{"score":0.6579,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":119.5},"22":{"score":0.6155,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":515.3},"23":{"score":0.6824,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.7},"24":{"score":0.8797,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":118.9},"25":{"score":0.551,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":120.2},"26":{"score":0.9937,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":119},"27":{"score":0.971,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":118.5},"28":{"score":0.94,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":118.1},"29":{"score":0.9659,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":119.6},"30":{"score":0.837,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2464,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.3},"32":{"score":0.7074,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":125.4},"33":{"score":0.2315,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":277.8},"36":{"score":0.5013,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2126.9},"37":{"score":0.6242,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":122.1},"38":{"score":0.4094,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":806},"39":{"score":0.9432,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":122.2},"40":{"score":0.7465,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":120.6},"41":{"score":0.802,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":119.1},"42":{"score":0.8574,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":121.7},"43":{"score":0.8596,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":119},"44":{"score":0.9082,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":118.9},"45":{"score":0.1597,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":120.5},"50":{"score":0.5889,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":123.6},"56":{"score":0.7465,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":196.6},"57":{"score":0.7187,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":119.6},"58":{"score":0.8522,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":118.5},"59":{"score":0.8441,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":122.9},"61":{"score":0.8625,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":120.6},"62":{"score":0.9033,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":122.8},"64":{"score":0.7898,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":118.6}},"benchmarks":{"1":{"raw":0.9776,"skill":0.9697,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9669,"private_eq":0.975},"2":{"raw":0.6745,"skill":0.6241,"coverage":1,"random":0.1341,"tracks":[],"combined":0.716,"private_eq":0.756},"3":{"raw":0.874,"skill":0.8716,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.9103,"skill":0.9091,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8825,"private_eq":0.8843},"5":{"raw":0.944,"skill":0.9437,"coverage":1,"random":0.006,"tracks":[],"combined":0.8567,"private_eq":0.8582},"6":{"raw":0.7977,"skill":0.5257,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9091,"skill":0.9091,"coverage":1,"random":0,"tracks":[],"combined":0.8442,"private_eq":0.845},"10":{"raw":0.3632,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8399,"skill":0.7685,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7549,"skill":0.6329,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5139,"private_eq":0.6785},"20":{"raw":0.9526,"skill":0.9051,"coverage":1,"random":0.5,"tracks":[],"combined":0.907,"private_eq":0.9535},"21":{"raw":0.6579,"skill":0.3158,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.6046,"skill":0.6015,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6023,"private_eq":0.6089},"23":{"raw":0.7056,"skill":0.4111,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8797,"skill":0.8396,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.551,"skill":0.4014,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5407,"combined":0.3816},"26":{"raw":0.9937,"skill":0.9916,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.971,"skill":0.9613,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.94,"skill":0.88,"coverage":1,"random":0.5,"tracks":[],"combined":0.6278,"private_eq":0.8155},"29":{"raw":0.9659,"skill":0.9545,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.837,"skill":0.8053,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8779,"random":0.25},{"track":"10-choice","raw":0.7961,"random":0.1}],"combined":0.7701,"private_eq":0.8117},"31":{"raw":0.2464,"skill":0.1792,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0853,"private_eq":0.1689},"32":{"raw":0.7074,"skill":0.5348,"coverage":1,"random":0.371,"tracks":[],"combined":0.4131,"private_eq":0.6344},"33":{"raw":0.2315,"skill":0.2213,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1943,"private_eq":0.2129},"36":{"raw":0.5013,"skill":0.4359,"coverage":1,"random":0.116,"tracks":[],"combined":0.4287,"private_eq":0.4998},"37":{"raw":0.6242,"skill":0.5286,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6146,"private_eq":0.6954},"38":{"raw":0.4094,"skill":0.3905,"coverage":1,"random":0.031,"tracks":[],"combined":0.6078,"private_eq":0.6233},"39":{"raw":0.9432,"skill":0.9165,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5513,"private_eq":0.6977},"40":{"raw":0.7465,"skill":0.6738,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5501,"headline":false},{"track":"A · English","score":0.7465,"headline":true},{"track":"C · Arabic pairs","score":0.86,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.6615,"private_eq":0.7391},"41":{"raw":0.802,"skill":0.703,"coverage":1,"random":0.3333,"tracks":[],"combined":0.605,"private_eq":0.739},"42":{"raw":0.8574,"skill":0.7226,"coverage":1,"random":0.486,"tracks":[],"combined":0.9109,"private_eq":0.9542},"43":{"raw":0.8596,"skill":0.7773,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6205,"private_eq":0.7629},"44":{"raw":0.9082,"skill":0.8164,"coverage":1,"random":0.5,"tracks":[],"combined":0.7655,"private_eq":0.8836},"45":{"raw":0.1597,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5889,"skill":0.4034,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.7465,"skill":0.493,"coverage":1,"random":0.5,"tracks":[],"combined":0.7069,"private_eq":0.8546},"57":{"raw":0.7187,"skill":0.6836,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5215,"combined":0.4566},"58":{"raw":0.8522,"skill":0.7858,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5237,"combined":0.3027},"59":{"raw":0.8441,"skill":0.6768,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3509,"private_eq":0.69},"61":{"raw":0.8625,"skill":0.725,"coverage":1,"random":0.5,"tracks":[],"combined":0.6549,"private_eq":0.8289},"62":{"raw":0.9033,"skill":0.8711,"coverage":1,"random":0.25,"tracks":[],"combined":0.6246,"private_eq":0.7209},"64":{"raw":0.7898,"skill":0.7372,"coverage":1,"random":0.2,"tracks":[],"combined":0.7682,"private_eq":0.8159}},"coverage":1,"pending":0,"notes":["Submitted directly as a results dataset (perplexity-ai/decision-index-results), no PR.","Replaces the earlier board entry pplx-decider-v1.1-27b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 67.74, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.79% on 827 rows (bar 98.5%).","One question per request, the submitter's chosen mode.","Training includes official training splits of several Decision Index benchmarks, decontaminated against the test sets; the checkpoint is a weight average of several runs chosen partly on Decision Index (author-declared)."],"scores":{"balanced_raw":75.59,"balanced_skill":67.98,"breadth_skill":67.03,"public_skill":67.74},"frozen_scores":{"balanced_raw":75.59,"balanced_skill":67.74,"breadth_skill":67.03},"categories":[{"id":"knowledge","raw":0.638,"skill":0.5529,"combined":0.4603,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7987,"skill":0.7188,"combined":0.6554,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7667,"skill":0.6808,"combined":0.6936,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.872,"skill":0.8544,"combined":0.8244,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7166,"skill":0.5624,"combined":0.5642,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"pplx-decider-v1.2","pub":67.74,"same_skills":64.3,"new_domains":57.76,"v":67.98,"lab":false,"jev":false,"rank_v":11,"rank_pub":22,"name":"Perplexity Decider v1.2 (27B)","eq":{"S":63.99,"O":73.98},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.39,"new_domains":29.59}}},{"engine":"pplx-decider-flash-v1.2","name":"Perplexity Decider Flash v1.2 (4B)","long_name":"Perplexity Decider Flash v1.2 (4B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4539265536,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"perplexity-ai","name":"Perplexity","logo":"data/orgs/perplexity-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.742,"conf":0.7799,"ece":0.0523,"brier":0.3626,"over95":0.0305,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0327,"conf":0.0712,"acc":0.1476},{"lo":0.1,"w":0.0226,"conf":0.1407,"acc":0.2411},{"lo":0.2,"w":0.0103,"conf":0.2513,"acc":0.3572},{"lo":0.3,"w":0.0249,"conf":0.3527,"acc":0.2749},{"lo":0.4,"w":0.0403,"conf":0.452,"acc":0.4567},{"lo":0.5,"w":0.1132,"conf":0.5478,"acc":0.5268},{"lo":0.6,"w":0.0753,"conf":0.6507,"acc":0.6664},{"lo":0.7,"w":0.075,"conf":0.7538,"acc":0.7168},{"lo":0.8,"w":0.1036,"conf":0.8543,"acc":0.7893},{"lo":0.9,"w":0.5022,"conf":0.9733,"acc":0.9111}],"areas":{"knowledge":{"ece":0.1153,"acc":0.6506,"conf":0.7518},"language":{"ece":0.0171,"acc":0.8681,"conf":0.8664},"retrieval":{"ece":0.1389,"acc":0.7057,"conf":0.8417},"tools":{"ece":0.0342,"acc":0.9394,"conf":0.9701},"arts":{"ece":0.0918,"acc":0.6086,"conf":0.5168}}},"gaps":null,"latency":{"median":74.8,"p95":380.5,"mean":136.8},"results":{"1":{"score":0.9575,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":73.1},"2":{"score":0.6511,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":481.6},"3":{"score":0.8642,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":187.6},"4":{"score":0.9009,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":64.8},"5":{"score":0.9331,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":66.8},"6":{"score":0.7979,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":101.5},"9":{"score":0.75,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1052.7},"10":{"score":0.3567,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":66.6},"11":{"score":0.829,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1152.6},"12":{"score":0.6935,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":61.5},"20":{"score":0.9128,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.4},"21":{"score":0.6157,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":60.9},"22":{"score":0.272,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":155.9},"23":{"score":0.6624,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62},"24":{"score":0.8006,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":60.8},"25":{"score":0.4439,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":61},"26":{"score":0.9907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":59.8},"27":{"score":0.9565,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":59.8},"28":{"score":0.8863,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":59.6},"29":{"score":0.9382,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":60.8},"30":{"score":0.6759,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2116,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":64.5},"32":{"score":0.6024,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":66},"33":{"score":0.117,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":87},"36":{"score":0.4564,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":614},"37":{"score":0.5775,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.7},"38":{"score":0.3547,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":258.7},"39":{"score":0.9047,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":63.8},"40":{"score":0.5942,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":61.4},"41":{"score":0.7688,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":61.2},"42":{"score":0.8233,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":63.4},"43":{"score":0.6263,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":61},"44":{"score":0.8542,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":61.1},"45":{"score":0.1257,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":61.8},"50":{"score":0.6163,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":65.1},"56":{"score":0.7255,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":77.1},"57":{"score":0.5912,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":61.5},"58":{"score":0.7343,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":61},"59":{"score":0.8248,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":64.8},"61":{"score":0.862,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":62.9},"62":{"score":0.8965,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":64.2},"64":{"score":0.7405,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":61}},"benchmarks":{"1":{"raw":0.9575,"skill":0.9426,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9192,"private_eq":0.9401},"2":{"raw":0.6511,"skill":0.5971,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6885,"private_eq":0.7325},"3":{"raw":0.8642,"skill":0.8616,"coverage":1,"random":0.0189,"tracks":[],"combined":0.812,"private_eq":0.8167},"4":{"raw":0.9009,"skill":0.8996,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8208,"private_eq":0.8241},"5":{"raw":0.9331,"skill":0.9327,"coverage":1,"random":0.006,"tracks":[],"combined":0.8432,"private_eq":0.8449},"6":{"raw":0.7979,"skill":0.5261,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.75,"skill":0.75,"coverage":1,"random":0,"tracks":[],"combined":0.69,"private_eq":0.6925},"10":{"raw":0.3567,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.829,"skill":0.7527,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6935,"skill":0.5409,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5009,"private_eq":0.6699},"20":{"raw":0.9159,"skill":0.8318,"coverage":1,"random":0.5,"tracks":[],"combined":0.8709,"private_eq":0.9357},"21":{"raw":0.6157,"skill":0.2314,"coverage":1,"random":0.5,"tracks":[],"combined":0.2466,"private_eq":0.6271},"22":{"raw":0.2608,"skill":0.255,"coverage":1,"random":0.0078,"tracks":[],"combined":0.275,"private_eq":0.2879},"23":{"raw":0.68,"skill":0.36,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8006,"skill":0.7341,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4181,"combined":0.2162},"26":{"raw":0.9907,"skill":0.9876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9565,"skill":0.942,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8863,"skill":0.7726,"coverage":1,"random":0.5,"tracks":[],"combined":0.547,"private_eq":0.7756},"29":{"raw":0.9382,"skill":0.9176,"coverage":1,"random":0.25,"tracks":[],"combined":0.9929,"private_eq":0.994},"30":{"raw":0.6759,"skill":0.6122,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7513,"random":0.25},{"track":"10-choice","raw":0.6005,"random":0.1}],"combined":0.6234,"private_eq":0.692},"31":{"raw":0.2116,"skill":0.1413,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0691,"private_eq":0.1542},"32":{"raw":0.6024,"skill":0.3679,"coverage":1,"random":0.371,"tracks":[],"combined":0.181,"private_eq":0.4901},"33":{"raw":0.117,"skill":0.1053,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.4564,"skill":0.385,"coverage":1,"random":0.116,"tracks":[],"combined":0.3907,"private_eq":0.4666},"37":{"raw":0.5775,"skill":0.4701,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5876,"private_eq":0.6741},"38":{"raw":0.3547,"skill":0.3341,"coverage":1,"random":0.031,"tracks":[],"combined":0.4899,"private_eq":0.5104},"39":{"raw":0.9047,"skill":0.8598,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6202,"private_eq":0.744},"40":{"raw":0.5942,"skill":0.478,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5011,"headline":false},{"track":"A · English","score":0.5942,"headline":true},{"track":"C · Arabic pairs","score":0.825,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"combined":0.4122,"private_eq":0.5475},"41":{"raw":0.7688,"skill":0.6532,"coverage":1,"random":0.3333,"tracks":[],"combined":0.7105,"private_eq":0.8085},"42":{"raw":0.8233,"skill":0.6562,"coverage":1,"random":0.486,"tracks":[],"combined":0.8599,"private_eq":0.9283},"43":{"raw":0.6263,"skill":0.4071,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4534,"private_eq":0.6588},"44":{"raw":0.8542,"skill":0.7084,"coverage":1,"random":0.5,"tracks":[],"combined":0.6102,"private_eq":0.8068},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6163,"skill":0.4432,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.7255,"skill":0.451,"coverage":1,"random":0.5,"tracks":[],"combined":0.3607,"private_eq":0.6835},"57":{"raw":0.5912,"skill":0.5402,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4483,"combined":0.3733},"58":{"raw":0.7343,"skill":0.6149,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4137,"combined":0.1414},"59":{"raw":0.8248,"skill":0.6367,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3399,"private_eq":0.6848},"61":{"raw":0.862,"skill":0.724,"coverage":1,"random":0.5,"tracks":[],"combined":0.6891,"private_eq":0.8458},"62":{"raw":0.8965,"skill":0.862,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.7405,"skill":0.6756,"coverage":1,"random":0.2,"tracks":[],"combined":0.6615,"private_eq":0.7315}},"coverage":1,"pending":0,"notes":["Submitted directly as a results dataset (perplexity-ai/decision-index-results), no PR.","Submitted public 0.3 run adopted (kit rescore 60.06, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.58% on 827 rows (bar 98.5%).","Distilled from the Perplexity Decider v1.2 27B models; the checkpoint is a weight average of several runs (author-declared).","One question per request, the submitter's chosen mode."],"scores":{"balanced_raw":69.71,"balanced_skill":54.29,"breadth_skill":58.6,"public_skill":60.06},"frozen_scores":{"balanced_raw":69.71,"balanced_skill":60.06,"breadth_skill":58.6},"categories":[{"id":"knowledge","raw":0.5354,"skill":0.4138,"combined":0.3204,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7508,"skill":0.6512,"combined":0.622,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7445,"skill":0.6524,"combined":0.6216,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8303,"skill":0.81,"combined":0.757,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6382,"skill":0.4662,"combined":0.45,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"pplx-decider-flash-v1.2","pub":60.06,"same_skills":56.16,"new_domains":40.02,"v":54.29,"lab":false,"jev":false,"rank_v":80,"rank_pub":59,"name":"Perplexity Decider Flash v1.2 (4B)","eq":{"S":55.76,"O":52.07},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":33.46,"new_domains":20.83}}},{"engine":"amos-1-large","name":"Amos 1 Large (27B)","long_name":"Amos 1 Large (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/mukaddamzaid/amos-1-large","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7664,"conf":0.7593,"ece":0.0317,"brier":0.3126,"over95":0.0007,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.002,"conf":0.0847,"acc":0.1354},{"lo":0.1,"w":0.0201,"conf":0.1517,"acc":0.1372},{"lo":0.2,"w":0.0246,"conf":0.2518,"acc":0.3033},{"lo":0.3,"w":0.0359,"conf":0.3549,"acc":0.2876},{"lo":0.4,"w":0.0512,"conf":0.451,"acc":0.3745},{"lo":0.5,"w":0.0876,"conf":0.5489,"acc":0.5191},{"lo":0.6,"w":0.0905,"conf":0.6513,"acc":0.6171},{"lo":0.7,"w":0.107,"conf":0.7544,"acc":0.7604},{"lo":0.8,"w":0.2301,"conf":0.8558,"acc":0.8699},{"lo":0.9,"w":0.351,"conf":0.9386,"acc":0.979}],"areas":{"knowledge":{"ece":0.0726,"acc":0.6915,"conf":0.73},"language":{"ece":0.0827,"acc":0.8997,"conf":0.817},"retrieval":{"ece":0.1376,"acc":0.6656,"conf":0.7691},"tools":{"ece":0.0417,"acc":0.9508,"conf":0.9192},"arts":{"ece":0.0395,"acc":0.6465,"conf":0.6106}}},"gaps":null,"latency":{"median":147.9,"p95":1365.8,"mean":460.7},"results":{"1":{"score":0.9793,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":254.6},"2":{"score":0.6894,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2379.3},"3":{"score":0.8602,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3127.8},"4":{"score":0.8891,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":253},"5":{"score":0.9484,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":417.2},"6":{"score":0.7953,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":723},"9":{"score":0.8182,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":737.8},"10":{"score":0.1183,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":432.5},"11":{"score":0.8471,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":887.8},"12":{"score":0.7777,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":119.6},"20":{"score":0.967,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.9},"21":{"score":0.6651,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":119.4},"22":{"score":0.57,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2640.1},"23":{"score":0.6498,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":197.9},"24":{"score":0.8771,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":119.5},"25":{"score":0.5612,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":121.6},"26":{"score":0.9907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":118.9},"27":{"score":0.9744,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":118.9},"28":{"score":0.9353,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":118.9},"29":{"score":0.9725,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":120.3},"30":{"score":0.7502,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1896,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":311.2},"32":{"score":0.6676,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":260.3},"33":{"score":0.2982,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":236.1},"36":{"score":0.5011,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3762.9},"37":{"score":0.6209,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.1},"38":{"score":0.4512,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1808.6},"39":{"score":0.9422,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":120.5},"40":{"score":0.733,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":119.2},"41":{"score":0.7817,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":119.3},"42":{"score":0.8682,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":156},"43":{"score":0.8684,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":118.2},"44":{"score":0.8054,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":118.3},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":120.5},"50":{"score":0.4004,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":236.5},"56":{"score":0.784,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":181.8},"57":{"score":0.6804,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":120.6},"58":{"score":0.8791,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":120.6},"59":{"score":0.8326,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":124.1},"61":{"score":0.774,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":135.3},"62":{"score":0.7177,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":264.3},"64":{"score":0.7538,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":120.2}},"benchmarks":{"1":{"raw":0.9793,"skill":0.9721,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9874,"private_eq":0.99},"2":{"raw":0.6894,"skill":0.6413,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7104,"private_eq":0.7512},"3":{"raw":0.8602,"skill":0.8575,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9013,"private_eq":0.9033},"4":{"raw":0.8891,"skill":0.8877,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8601,"private_eq":0.8625},"5":{"raw":0.9484,"skill":0.9481,"coverage":1,"random":0.006,"tracks":[],"combined":0.7697,"private_eq":0.7727},"6":{"raw":0.7953,"skill":0.52,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8182,"skill":0.8182,"coverage":1,"random":0,"tracks":[],"combined":0.7431,"private_eq":0.745},"10":{"raw":0.1183,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8471,"skill":0.7789,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7777,"skill":0.667,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5871,"private_eq":0.7268},"20":{"raw":0.9686,"skill":0.9372,"coverage":1,"random":0.5,"tracks":[],"combined":0.9432,"private_eq":0.9714},"21":{"raw":0.6651,"skill":0.3303,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.5704,"skill":0.567,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5221,"private_eq":0.5302},"23":{"raw":0.6567,"skill":0.3134,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8771,"skill":0.8361,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5612,"skill":0.415,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4721,"combined":0.289},"26":{"raw":0.9907,"skill":0.9876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9744,"skill":0.9659,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9353,"skill":0.8706,"coverage":1,"random":0.5,"tracks":[],"combined":0.6883,"private_eq":0.8454},"29":{"raw":0.9725,"skill":0.9633,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7502,"skill":0.7021,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8173,"random":0.25},{"track":"10-choice","raw":0.6831,"random":0.1}],"combined":0.7228,"private_eq":0.7731},"31":{"raw":0.1896,"skill":0.1174,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0529,"private_eq":0.1395},"32":{"raw":0.6676,"skill":0.4715,"coverage":1,"random":0.371,"tracks":[],"combined":0.3589,"private_eq":0.6007},"33":{"raw":0.2982,"skill":0.2889,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3022,"private_eq":0.3182},"36":{"raw":0.5011,"skill":0.4356,"coverage":1,"random":0.116,"tracks":[],"combined":0.4087,"private_eq":0.4824},"37":{"raw":0.6209,"skill":0.5245,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6219,"private_eq":0.7012},"38":{"raw":0.4512,"skill":0.4336,"coverage":1,"random":0.031,"tracks":[],"combined":0.6277,"private_eq":0.6424},"39":{"raw":0.9422,"skill":0.915,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5736,"private_eq":0.7127},"40":{"raw":0.733,"skill":0.6565,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.589,"headline":false},{"track":"A · English","score":0.733,"headline":true},{"track":"C · Arabic pairs","score":0.955,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.6637,"private_eq":0.7408},"41":{"raw":0.7817,"skill":0.6726,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4788,"private_eq":0.6558},"42":{"raw":0.8682,"skill":0.7436,"coverage":1,"random":0.486,"tracks":[],"combined":0.9335,"private_eq":0.9657},"43":{"raw":0.8684,"skill":0.7912,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6165,"private_eq":0.7604},"44":{"raw":0.8054,"skill":0.6108,"coverage":1,"random":0.5,"tracks":[],"combined":0.5511,"private_eq":0.7776},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4004,"skill":0.1297,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.784,"skill":0.568,"coverage":1,"random":0.5,"tracks":[],"combined":0.7241,"private_eq":0.8631},"57":{"raw":0.6804,"skill":0.6405,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6221,"combined":0.5711},"58":{"raw":0.8791,"skill":0.8248,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5199,"combined":0.2971},"59":{"raw":0.8326,"skill":0.6529,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3542,"private_eq":0.6916},"61":{"raw":0.774,"skill":0.548,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.7177,"skill":0.6236,"coverage":1,"random":0.25,"tracks":[],"combined":0.5767,"private_eq":0.6854},"64":{"raw":0.7538,"skill":0.6923,"coverage":1,"random":0.2,"tracks":[],"combined":0.7003,"private_eq":0.7622}},"coverage":1,"pending":0,"notes":["New in the 2026-10-14 PR batch (PR #170).","Submitted public 0.3 run adopted (kit rescore 64.94, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.80% on 831 rows (bar 98.5%).","Served with --rotations 2 (answers averaged over two option orders), the author's submitted setting.","Training data unpublished."],"scores":{"balanced_raw":73.54,"balanced_skill":64.81,"breadth_skill":64.15,"public_skill":64.94},"frozen_scores":{"balanced_raw":73.54,"balanced_skill":64.94,"breadth_skill":64.15},"categories":[{"id":"knowledge","raw":0.6167,"skill":0.5236,"combined":0.4462,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8039,"skill":0.7244,"combined":0.6592,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7553,"skill":0.6615,"combined":0.6695,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8209,"skill":0.7923,"combined":0.7957,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6692,"skill":0.495,"combined":0.5331,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"amos-1-large","pub":64.94,"same_skills":62.69,"new_domains":53.31,"v":64.81,"lab":false,"jev":false,"rank_v":31,"rank_pub":40,"name":"Amos 1 Large (27B)","eq":{"S":62.37,"O":68.48},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":37.42,"new_domains":27.39}}},{"engine":"lychee-27b","name":"Lychee-27B","long_name":"Lychee-27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/LycheeAI/Lychee-27B","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7582,"conf":0.8629,"ece":0.108,"brier":0.3524,"over95":0.0428,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.002,"conf":0.0801,"acc":0.2527},{"lo":0.1,"w":0.0137,"conf":0.1575,"acc":0.2376},{"lo":0.2,"w":0.0185,"conf":0.2528,"acc":0.2626},{"lo":0.3,"w":0.0204,"conf":0.3503,"acc":0.3361},{"lo":0.4,"w":0.0318,"conf":0.4569,"acc":0.314},{"lo":0.5,"w":0.0608,"conf":0.5509,"acc":0.4335},{"lo":0.6,"w":0.0506,"conf":0.6485,"acc":0.4565},{"lo":0.7,"w":0.0596,"conf":0.752,"acc":0.536},{"lo":0.8,"w":0.0774,"conf":0.8549,"acc":0.61},{"lo":0.9,"w":0.6652,"conf":0.9876,"acc":0.9081}],"areas":{"knowledge":{"ece":0.1292,"acc":0.694,"conf":0.8232},"language":{"ece":0.0603,"acc":0.8695,"conf":0.9298},"retrieval":{"ece":0.2013,"acc":0.6773,"conf":0.8775},"tools":{"ece":0.0455,"acc":0.9466,"conf":0.992},"arts":{"ece":0.1581,"acc":0.6377,"conf":0.7326}}},"gaps":null,"latency":{"median":97.2,"p95":1158.4,"mean":320},"results":{"1":{"score":0.9693,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":222.2},"2":{"score":0.6652,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2173},"3":{"score":0.8642,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1037.5},"4":{"score":0.8069,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":136.1},"5":{"score":0.9207,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":219.5},"6":{"score":0.7983,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":597.7},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2585.3},"10":{"score":0.4087,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":172.1},"11":{"score":0.7617,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":4236.9},"12":{"score":0.7297,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":120.2},"20":{"score":0.955,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":117.8},"21":{"score":0.6096,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":118.7},"22":{"score":0.558,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":883.2},"23":{"score":0.6612,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":120.3},"24":{"score":0.8739,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":118.1},"25":{"score":0.5255,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":118.8},"26":{"score":0.9916,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":117},"27":{"score":0.9667,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":117.3},"28":{"score":0.9195,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":116.7},"29":{"score":0.9464,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":118.4},"30":{"score":0.7502,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1744,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":145.5},"32":{"score":0.7061,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":149},"33":{"score":0.3255,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":465.3},"36":{"score":0.4974,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3830.9},"37":{"score":0.5698,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":120.6},"38":{"score":0.2662,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1537},"39":{"score":0.9487,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":120.7},"40":{"score":0.7054,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":118.8},"41":{"score":0.7515,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":118.2},"42":{"score":0.8645,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":121.8},"43":{"score":0.8632,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":118.2},"44":{"score":0.8062,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":117.9},"45":{"score":0.1717,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":119.3},"50":{"score":0.4004,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":132.1},"56":{"score":0.764,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":421.1},"57":{"score":0.705,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":118.6},"58":{"score":0.8346,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":118.1},"59":{"score":0.7998,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":126.9},"61":{"score":0.821,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":120.8},"62":{"score":0.8187,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":126.6},"64":{"score":0.75,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":117.9}},"benchmarks":{"1":{"raw":0.9693,"skill":0.9586,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9669,"private_eq":0.975},"2":{"raw":0.6652,"skill":0.6133,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7001,"private_eq":0.7424},"3":{"raw":0.8642,"skill":0.8616,"coverage":1,"random":0.0189,"tracks":[],"combined":0.938,"private_eq":0.9389},"4":{"raw":0.8069,"skill":0.8044,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8633,"private_eq":0.8656},"5":{"raw":0.9207,"skill":0.9202,"coverage":1,"random":0.006,"tracks":[],"combined":0.8216,"private_eq":0.8237},"6":{"raw":0.7983,"skill":0.5271,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.776,"private_eq":0.7775},"10":{"raw":0.4087,"skill":0.0161,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7617,"skill":0.6554,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7297,"skill":0.5951,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5259,"private_eq":0.6864},"20":{"raw":0.956,"skill":0.912,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.6096,"skill":0.2192,"coverage":1,"random":0.5,"tracks":[],"combined":0.2064,"private_eq":0.6072},"22":{"raw":0.5485,"skill":0.5449,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5629,"private_eq":0.5703},"23":{"raw":0.6628,"skill":0.3255,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8739,"skill":0.8319,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5255,"skill":0.3673,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5074,"combined":0.3366},"26":{"raw":0.9916,"skill":0.9888,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9667,"skill":0.9556,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9195,"skill":0.839,"coverage":1,"random":0.5,"tracks":[],"combined":0.6429,"private_eq":0.823},"29":{"raw":0.9464,"skill":0.9285,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7502,"skill":0.7016,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8127,"random":0.25},{"track":"10-choice","raw":0.6876,"random":0.1}],"combined":0.7369,"private_eq":0.7846},"31":{"raw":0.1744,"skill":0.1008,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0853,"private_eq":0.1689},"32":{"raw":0.7061,"skill":0.5328,"coverage":1,"random":0.371,"tracks":[],"combined":0.4131,"private_eq":0.6344},"33":{"raw":0.3255,"skill":0.3165,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3485,"private_eq":0.3634},"36":{"raw":0.4974,"skill":0.4314,"coverage":1,"random":0.116,"tracks":[],"combined":0.4184,"private_eq":0.4908},"37":{"raw":0.5698,"skill":0.4604,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5755,"private_eq":0.6646},"38":{"raw":0.2662,"skill":0.2427,"coverage":1,"random":0.031,"tracks":[],"combined":0.4123,"private_eq":0.436},"39":{"raw":0.9487,"skill":0.9245,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5328,"private_eq":0.6853},"40":{"raw":0.7054,"skill":0.621,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3813,"headline":false},{"track":"A · English","score":0.7054,"headline":true},{"track":"C · Arabic pairs","score":0.79,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.6167,"private_eq":0.7047},"41":{"raw":0.7515,"skill":0.6272,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5574,"private_eq":0.7076},"42":{"raw":0.8645,"skill":0.7364,"coverage":1,"random":0.486,"tracks":[],"combined":0.8792,"private_eq":0.9381},"43":{"raw":0.8632,"skill":0.7829,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6247,"private_eq":0.7655},"44":{"raw":0.8062,"skill":0.6124,"coverage":1,"random":0.5,"tracks":[],"combined":0.5139,"private_eq":0.7592},"45":{"raw":0.1717,"skill":0.0091,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4004,"skill":0.1297,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.764,"skill":0.528,"coverage":1,"random":0.5,"tracks":[],"combined":0.8539,"private_eq":0.9273},"57":{"raw":0.705,"skill":0.6682,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5215,"combined":0.4566},"58":{"raw":0.8346,"skill":0.7603,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5172,"combined":0.2931},"59":{"raw":0.7998,"skill":0.5849,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4157,"private_eq":0.7209},"61":{"raw":0.821,"skill":0.642,"coverage":1,"random":0.5,"tracks":[],"combined":0.6605,"private_eq":0.8317},"62":{"raw":0.8187,"skill":0.7583,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.75,"skill":0.6875,"coverage":1,"random":0.2,"tracks":[],"combined":0.6228,"private_eq":0.7009}},"coverage":1,"pending":0,"notes":["New in the 2026-10-14 PR batch (PR #162).","Submitted public 0.3 run adopted (kit rescore 62.76, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.60% on 831 rows (bar 98.5%).","Uses the published autojev recipe of Perplexity Decider v1.1 (same prompts, answer codes, readout and attention); the weights are an independent fine-tune of Qwen3.8-27B (lab check).","Training data undisclosed; license CC BY-NC 4.0."],"scores":{"balanced_raw":71.9,"balanced_skill":67.35,"breadth_skill":61.97,"public_skill":62.76},"frozen_scores":{"balanced_raw":71.9,"balanced_skill":62.76,"breadth_skill":61.97},"categories":[{"id":"knowledge","raw":0.6178,"skill":0.5183,"combined":0.4442,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7563,"skill":0.6617,"combined":0.6246,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.731,"skill":0.639,"combined":0.6991,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8316,"skill":0.8079,"combined":0.8136,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6545,"skill":0.4698,"combined":0.5048,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"lychee-27b","pub":62.76,"same_skills":62.52,"new_domains":58.66,"v":67.35,"lab":false,"jev":false,"rank_v":17,"rank_pub":46,"name":"Lychee-27B","eq":{"S":62.19,"O":75.09},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.31,"new_domains":30.03}}},{"engine":"span-02","name":"span-02 (Gemma 4 26B-A4B)","long_name":"span-02 (Gemma 4 26B-A4B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/lihe07/span-02","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7621,"conf":0.8761,"ece":0.114,"brier":0.3553,"over95":0.0277,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0001,"conf":0.0816,"acc":0},{"lo":0.1,"w":0.0062,"conf":0.1631,"acc":0.0945},{"lo":0.2,"w":0.0086,"conf":0.2504,"acc":0.1027},{"lo":0.3,"w":0.0099,"conf":0.3589,"acc":0.145},{"lo":0.4,"w":0.0202,"conf":0.452,"acc":0.1936},{"lo":0.5,"w":0.044,"conf":0.5524,"acc":0.3904},{"lo":0.6,"w":0.0533,"conf":0.6521,"acc":0.4683},{"lo":0.7,"w":0.0603,"conf":0.7498,"acc":0.526},{"lo":0.8,"w":0.1888,"conf":0.8531,"acc":0.6667},{"lo":0.9,"w":0.6085,"conf":0.9776,"acc":0.913}],"areas":{"knowledge":{"ece":0.1508,"acc":0.6744,"conf":0.8252},"language":{"ece":0.0511,"acc":0.8635,"conf":0.9146},"retrieval":{"ece":0.1091,"acc":0.7622,"conf":0.8713},"tools":{"ece":0.0394,"acc":0.9282,"conf":0.9617},"arts":{"ece":0.2056,"acc":0.6556,"conf":0.8612}}},"gaps":null,"latency":{"median":33,"p95":822.4,"mean":234.1},"results":{"1":{"score":0.9622,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":39.5},"2":{"score":0.6612,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":624.5},"3":{"score":0.8091,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":200.3},"4":{"score":0.8562,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":35.8},"5":{"score":0.9262,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":38.9},"6":{"score":0.7988,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":105.5},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":85.1},"10":{"score":0.6666,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":50.6},"11":{"score":0.7808,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":804.8},"12":{"score":0.7807,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":26.2},"20":{"score":0.9504,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.1},"21":{"score":0.621,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":24.1},"22":{"score":0.639,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":189.4},"23":{"score":0.6224,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.3},"24":{"score":0.8612,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":24.7},"25":{"score":0.4694,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":702.8},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":23.9},"27":{"score":0.9735,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":24.3},"28":{"score":0.8863,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":23.9},"29":{"score":0.943,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":27.4},"30":{"score":0.9121,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1814,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.1},"32":{"score":0.6649,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":46.1},"33":{"score":0.3497,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":609},"36":{"score":0.4623,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":642.4},"37":{"score":0.587,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":798.2},"38":{"score":0.342,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":634.7},"39":{"score":0.9251,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":27.4},"40":{"score":0.5392,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":27.9},"41":{"score":0.7735,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":27.6},"42":{"score":0.8225,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32.7},"43":{"score":0.8228,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":25.7},"44":{"score":0.8052,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.5},"45":{"score":0.1238,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":739.2},"50":{"score":0.4749,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":47.7},"56":{"score":0.8115,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":712.9},"57":{"score":0.7303,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":425.9},"58":{"score":0.8763,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":26.4},"59":{"score":0.8085,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":41.3},"61":{"score":0.8073,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":30.8},"62":{"score":0.8628,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":41.3},"64":{"score":0.7367,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":27.2}},"benchmarks":{"1":{"raw":0.9622,"skill":0.949,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9056,"private_eq":0.9301},"2":{"raw":0.6612,"skill":0.6088,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6922,"private_eq":0.7356},"3":{"raw":0.8091,"skill":0.8054,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7963,"private_eq":0.8014},"4":{"raw":0.8562,"skill":0.8544,"coverage":1,"random":0.0127,"tracks":[],"combined":0.885,"private_eq":0.8868},"5":{"raw":0.9262,"skill":0.9258,"coverage":1,"random":0.006,"tracks":[],"combined":0.7711,"private_eq":0.7741},"6":{"raw":0.7988,"skill":0.5283,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.7709,"private_eq":0.7725},"10":{"raw":0.6666,"skill":0.4453,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7808,"skill":0.683,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7807,"skill":0.6715,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5389,"private_eq":0.695},"20":{"raw":0.9519,"skill":0.9038,"coverage":1,"random":0.5,"tracks":[],"combined":0.9432,"private_eq":0.9714},"21":{"raw":0.621,"skill":0.242,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.635,"skill":0.6322,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4911,"private_eq":0.4998},"23":{"raw":0.6155,"skill":0.231,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8612,"skill":0.8149,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4694,"skill":0.2925,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4763,"combined":0.2947},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9735,"skill":0.9647,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8863,"skill":0.7726,"coverage":1,"random":0.5,"tracks":[],"combined":0.6126,"private_eq":0.808},"29":{"raw":0.943,"skill":0.924,"coverage":1,"random":0.25,"tracks":[],"combined":0.9731,"private_eq":0.9793},"30":{"raw":0.9121,"skill":0.8914,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9022,"random":0.25},{"track":"10-choice","raw":0.9219,"random":0.1}],"combined":0.8729,"private_eq":0.8955},"31":{"raw":0.1814,"skill":0.1084,"coverage":1,"random":0.0819,"tracks":[],"combined":0.079,"private_eq":0.1631},"32":{"raw":0.6649,"skill":0.4672,"coverage":1,"random":0.371,"tracks":[],"combined":0.1963,"private_eq":0.4996},"33":{"raw":0.3497,"skill":0.3411,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4307,"private_eq":0.4436},"36":{"raw":0.4623,"skill":0.3917,"coverage":1,"random":0.116,"tracks":[],"combined":0.3676,"private_eq":0.4464},"37":{"raw":0.587,"skill":0.482,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5836,"private_eq":0.671},"38":{"raw":0.342,"skill":0.3209,"coverage":1,"random":0.031,"tracks":[],"combined":0.536,"private_eq":0.5545},"39":{"raw":0.9251,"skill":0.8898,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5929,"private_eq":0.7257},"40":{"raw":0.5392,"skill":0.4072,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5195,"headline":false},{"track":"A · English","score":0.5392,"headline":true},{"track":"C · Arabic pairs","score":0.89,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.3813,"private_eq":0.5238},"41":{"raw":0.7735,"skill":0.6603,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5867,"private_eq":0.7269},"42":{"raw":0.8225,"skill":0.6547,"coverage":1,"random":0.486,"tracks":[],"combined":0.7287,"private_eq":0.8616},"43":{"raw":0.8228,"skill":0.7189,"coverage":1,"random":0.3697,"tracks":[],"combined":0.629,"private_eq":0.7682},"44":{"raw":0.8052,"skill":0.6104,"coverage":1,"random":0.5,"tracks":[],"combined":0.6154,"private_eq":0.8094},"45":{"raw":0.1238,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4749,"skill":0.238,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.8115,"skill":0.623,"coverage":1,"random":0.5,"tracks":[],"combined":0.5424,"private_eq":0.7733},"57":{"raw":0.7303,"skill":0.6967,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5801,"combined":0.5233},"58":{"raw":0.8763,"skill":0.8207,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4984,"combined":0.2656},"59":{"raw":0.8085,"skill":0.6029,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1806,"private_eq":0.6088},"61":{"raw":0.8073,"skill":0.6146,"coverage":1,"random":0.5,"tracks":[],"combined":0.6152,"private_eq":0.8093},"62":{"raw":0.8628,"skill":0.8171,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.7367,"skill":0.6709,"coverage":1,"random":0.2,"tracks":[],"combined":0.7489,"private_eq":0.8006}},"coverage":1,"pending":0,"notes":["New in the 2026-10-14 PR batch (PR #171).","Submitted public 0.3 run adopted (kit rescore 63.62, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (NVFP4 experts, FP8 dense); decision agreement with the submitted rows 100.00% on 831 rows (bar 96%).","Served quantized (NVFP4 experts, FP8 dense) by the author's own engine, so the quantized agreement bar applies.","Gated short thinking on about 19% of questions with a deterministic think budget; serving settings were compared on Decision Index public samples (author-disclosed)."],"scores":{"balanced_raw":72.77,"balanced_skill":59.07,"breadth_skill":62.76,"public_skill":63.62},"frozen_scores":{"balanced_raw":72.77,"balanced_skill":63.62,"breadth_skill":62.76},"categories":[{"id":"knowledge","raw":0.62,"skill":0.5197,"combined":0.4499,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7511,"skill":0.6525,"combined":0.5738,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7423,"skill":0.6554,"combined":0.6317,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.861,"skill":0.8391,"combined":0.7635,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6725,"skill":0.4863,"combined":0.5395,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"span-02","pub":63.62,"same_skills":59.32,"new_domains":45.84,"v":59.07,"lab":false,"jev":false,"rank_v":57,"rank_pub":44,"name":"span-02 (Gemma 4 26B-A4B)","eq":{"S":58.95,"O":59.26},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":35.37,"new_domains":23.7}}},{"engine":"drex-max-v1","name":"Nace.AI Drex Max v1 (27B)","long_name":"Nace.AI Drex Max v1 (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"weights_note":"Open weights coming soon","org":{"id":"Nace-DeepInfra","name":"Nace-Deepinfra","logo":"data/orgs/Nace-DeepInfra.png"},"pr":238,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7947,"conf":0.8429,"ece":0.0573,"brier":0.2886,"over95":0.0184,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0154,"conf":0.0691,"acc":0.1327},{"lo":0.1,"w":0.0118,"conf":0.1478,"acc":0.2359},{"lo":0.2,"w":0.0117,"conf":0.254,"acc":0.3528},{"lo":0.3,"w":0.0287,"conf":0.351,"acc":0.4003},{"lo":0.4,"w":0.0316,"conf":0.4554,"acc":0.4084},{"lo":0.5,"w":0.0653,"conf":0.5512,"acc":0.5197},{"lo":0.6,"w":0.0568,"conf":0.6503,"acc":0.5416},{"lo":0.7,"w":0.0618,"conf":0.7519,"acc":0.6315},{"lo":0.8,"w":0.0938,"conf":0.8535,"acc":0.7151},{"lo":0.9,"w":0.6231,"conf":0.9841,"acc":0.9478}],"areas":{"knowledge":{"ece":0.1127,"acc":0.7166,"conf":0.8103},"language":{"ece":0.0178,"acc":0.9049,"conf":0.9093},"retrieval":{"ece":0.0806,"acc":0.7694,"conf":0.8491},"tools":{"ece":0.0309,"acc":0.952,"conf":0.9829},"arts":{"ece":0.0582,"acc":0.6784,"conf":0.6993}}},"gaps":null,"latency":{"median":103.6,"p95":1158.6,"mean":325.3},"results":{"1":{"score":0.9793,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6971,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8839,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.9032,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9561,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7967,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.7273,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5764,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8392,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7709,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9532,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6454,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.6305,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6624,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8786,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5612,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9924,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9727,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.9519,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9794,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.8336,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2454,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6968,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3127,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4988,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.6273,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.4115,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9458,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.7333,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7834,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.864,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8684,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8542,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1996,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.5084,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.752,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7283,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.9368,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8483,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8788,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8639,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8049,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9793,"skill":0.9721,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9602,"private_eq":0.9701},"2":{"raw":0.6971,"skill":0.6503,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7038,"private_eq":0.7456},"3":{"raw":0.8839,"skill":0.8817,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.9032,"skill":0.902,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8887,"private_eq":0.8904},"5":{"raw":0.9561,"skill":0.9558,"coverage":1,"random":0.006,"tracks":[],"combined":0.8404,"private_eq":0.8422},"6":{"raw":0.7967,"skill":0.5233,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7273,"skill":0.7273,"coverage":1,"random":0,"tracks":[],"combined":0.7051,"private_eq":0.7075},"10":{"raw":0.5764,"skill":0.2952,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8392,"skill":0.7674,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7709,"skill":0.6568,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5089,"private_eq":0.6752},"20":{"raw":0.9546,"skill":0.9092,"coverage":1,"random":0.5,"tracks":[],"combined":0.9577,"private_eq":0.9786},"21":{"raw":0.6454,"skill":0.2907,"coverage":1,"random":0.5,"tracks":[],"combined":0.2264,"private_eq":0.6171},"22":{"raw":0.6282,"skill":0.6253,"coverage":1,"random":0.0078,"tracks":[],"combined":0.545,"private_eq":0.5527},"23":{"raw":0.6681,"skill":0.3363,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8786,"skill":0.8381,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5612,"skill":0.415,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5179,"combined":0.3508},"26":{"raw":0.9924,"skill":0.9899,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9727,"skill":0.9636,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9519,"skill":0.9038,"coverage":1,"random":0.5,"tracks":[],"combined":0.6429,"private_eq":0.823},"29":{"raw":0.9794,"skill":0.9725,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8336,"skill":0.8003,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8666,"random":0.25},{"track":"10-choice","raw":0.8006,"random":0.1}],"combined":0.7746,"private_eq":0.8153},"31":{"raw":0.2454,"skill":0.1781,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0723,"private_eq":0.1571},"32":{"raw":0.6968,"skill":0.518,"coverage":1,"random":0.371,"tracks":[],"combined":0.3589,"private_eq":0.6007},"33":{"raw":0.3127,"skill":0.3036,"coverage":1,"random":0.0131,"tracks":[],"combined":0.405,"private_eq":0.4185},"36":{"raw":0.4988,"skill":0.433,"coverage":1,"random":0.116,"tracks":[],"combined":0.4147,"private_eq":0.4876},"37":{"raw":0.6273,"skill":0.5325,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5912,"private_eq":0.677},"38":{"raw":0.4115,"skill":0.3927,"coverage":1,"random":0.031,"tracks":[],"combined":0.5778,"private_eq":0.5946},"39":{"raw":0.9458,"skill":0.9203,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4394,"private_eq":0.6225},"40":{"raw":0.7333,"skill":0.6569,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5603,"headline":false},{"track":"A · English","score":0.7333,"headline":true},{"track":"C · Arabic pairs","score":0.95,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.6325,"private_eq":0.7168},"41":{"raw":0.7834,"skill":0.6752,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5275,"private_eq":0.6879},"42":{"raw":0.864,"skill":0.7354,"coverage":1,"random":0.486,"tracks":[],"combined":0.9339,"private_eq":0.9659},"43":{"raw":0.8684,"skill":0.7912,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6205,"private_eq":0.7629},"44":{"raw":0.8542,"skill":0.7084,"coverage":1,"random":0.5,"tracks":[],"combined":0.6638,"private_eq":0.8333},"45":{"raw":0.1996,"skill":0.0425,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5084,"skill":0.2865,"coverage":1,"random":0.311,"tracks":[],"combined":0.312,"private_eq":0.5307},"56":{"raw":0.752,"skill":0.504,"coverage":1,"random":0.5,"tracks":[],"combined":0.6203,"private_eq":0.8118},"57":{"raw":0.7283,"skill":0.6944,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6328,"combined":0.5833},"58":{"raw":0.9368,"skill":0.9084,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5295,"combined":0.3112},"59":{"raw":0.8483,"skill":0.6855,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2959,"private_eq":0.6638},"61":{"raw":0.8788,"skill":0.7576,"coverage":1,"random":0.5,"tracks":[],"combined":0.6777,"private_eq":0.8402},"62":{"raw":0.8639,"skill":0.8185,"coverage":1,"random":0.25,"tracks":[],"combined":0.6725,"private_eq":0.7564},"64":{"raw":0.8049,"skill":0.7561,"coverage":1,"random":0.2,"tracks":[],"combined":0.7585,"private_eq":0.8082}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #238).","Submitted public 0.3 run adopted (kit rescore 67.52, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (submitted endpoint undeclared, lab run bf16); decision agreement with the submitted rows 98.52% on 831 rows (bar 96%).","A separate model from Drex v1.5 and Drex v2.0, which stay on the board.","The submitted run used a private endpoint; the lab ran the published bf16 loader. Listed under a maintainer decision: one smoke row ran out of memory in the first check and matched the submitted answers once re-run.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":75.4,"balanced_skill":66.6,"breadth_skill":66.94,"public_skill":67.52},"frozen_scores":{"balanced_raw":75.4,"balanced_skill":67.52,"breadth_skill":66.94},"categories":[{"id":"knowledge","raw":0.6554,"skill":0.5714,"combined":0.4779,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8006,"skill":0.7222,"combined":0.6213,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7709,"skill":0.6883,"combined":0.676,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8378,"skill":0.8186,"combined":0.8038,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7016,"skill":0.534,"combined":0.5599,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"drex-max-v1","pub":67.52,"same_skills":63.08,"new_domains":56.45,"v":66.6,"lab":false,"jev":false,"rank_v":22,"rank_pub":23,"name":"Nace.AI Drex Max v1 (27B)","eq":{"S":62.76,"O":72.36},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.65,"new_domains":28.94}},"new_in_edition":true},{"engine":"ajev-lm6q","name":"AJev lm6q (Gemma 4 31B)","long_name":"AJev lm6q (Gemma 4 31B)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/andyzhang232/ajev-gemma4-31b-lm6q","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":215,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7674,"conf":0.8622,"ece":0.0956,"brier":0.3479,"over95":0.0372,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0004,"conf":0.0898,"acc":0.1111},{"lo":0.1,"w":0.0063,"conf":0.1577,"acc":0.1743},{"lo":0.2,"w":0.0066,"conf":0.2486,"acc":0.2894},{"lo":0.3,"w":0.0157,"conf":0.3537,"acc":0.2222},{"lo":0.4,"w":0.0231,"conf":0.4575,"acc":0.2741},{"lo":0.5,"w":0.0545,"conf":0.5481,"acc":0.4431},{"lo":0.6,"w":0.0568,"conf":0.6517,"acc":0.4915},{"lo":0.7,"w":0.0864,"conf":0.7545,"acc":0.6175},{"lo":0.8,"w":0.1168,"conf":0.8564,"acc":0.7026},{"lo":0.9,"w":0.6335,"conf":0.9651,"acc":0.8952}],"areas":{"knowledge":{"ece":0.1012,"acc":0.7124,"conf":0.8124},"language":{"ece":0.0192,"acc":0.8713,"conf":0.8864},"retrieval":{"ece":0.294,"acc":0.625,"conf":0.919},"tools":{"ece":0.0339,"acc":0.9449,"conf":0.9757},"arts":{"ece":0.1109,"acc":0.6977,"conf":0.8048}}},"gaps":null,"latency":{"median":144.4,"p95":2245.1,"mean":489},"results":{"1":{"score":0.9782,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6508,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8524,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8065,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9243,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7993,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0043,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7749,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7917,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9524,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6811,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.7085,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6564,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8809,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5357,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.992,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9727,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.88,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9298,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.8817,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.277,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6862,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.343,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.467,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5898,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.324,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9021,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.7166,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.8217,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8608,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.893,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8054,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1238,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4403,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.8655,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7252,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.8353,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8095,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8367,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8606,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8295,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9782,"skill":0.9705,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.6508,"skill":0.5967,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6909,"private_eq":0.7345},"3":{"raw":0.8524,"skill":0.8496,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9171,"private_eq":0.9186},"4":{"raw":0.8065,"skill":0.804,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8968,"private_eq":0.8983},"5":{"raw":0.9243,"skill":0.9238,"coverage":1,"random":0.006,"tracks":[],"combined":0.7961,"private_eq":0.7986},"6":{"raw":0.7993,"skill":0.5294,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.7709,"private_eq":0.7725},"10":{"raw":0.0043,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7749,"skill":0.6744,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7917,"skill":0.688,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5206,"private_eq":0.6829},"20":{"raw":0.9534,"skill":0.9068,"coverage":1,"random":0.5,"tracks":[],"combined":0.965,"private_eq":0.9822},"21":{"raw":0.6811,"skill":0.3623,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.7081,"skill":0.7059,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6195,"private_eq":0.6258},"23":{"raw":0.6579,"skill":0.3158,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8809,"skill":0.8412,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5357,"skill":0.381,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5251,"combined":0.3605},"26":{"raw":0.992,"skill":0.9893,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9727,"skill":0.9636,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.88,"skill":0.76,"coverage":1,"random":0.5,"tracks":[],"combined":0.6984,"private_eq":0.8504},"29":{"raw":0.9298,"skill":0.9064,"coverage":1,"random":0.25,"tracks":[],"combined":0.9681,"private_eq":0.9756},"30":{"raw":0.8817,"skill":0.8588,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9121,"random":0.25},{"track":"10-choice","raw":0.8514,"random":0.1}],"combined":0.8071,"private_eq":0.8418},"31":{"raw":0.277,"skill":0.2125,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0669,"private_eq":0.1522},"32":{"raw":0.6862,"skill":0.5011,"coverage":1,"random":0.371,"tracks":[],"combined":0.4362,"private_eq":0.6488},"33":{"raw":0.343,"skill":0.3343,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3794,"private_eq":0.3935},"36":{"raw":0.467,"skill":0.397,"coverage":1,"random":0.116,"tracks":[],"combined":0.4082,"private_eq":0.4819},"37":{"raw":0.5898,"skill":0.4855,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6001,"private_eq":0.684},"38":{"raw":0.324,"skill":0.3024,"coverage":1,"random":0.031,"tracks":[],"combined":0.5619,"private_eq":0.5793},"39":{"raw":0.9021,"skill":0.856,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5221,"private_eq":0.6781},"40":{"raw":0.7166,"skill":0.6354,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4912,"headline":false},{"track":"A · English","score":0.7166,"headline":true},{"track":"C · Arabic pairs","score":0.875,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.7369,"private_eq":0.797},"41":{"raw":0.8217,"skill":0.7326,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4227,"private_eq":0.6188},"42":{"raw":0.8608,"skill":0.7292,"coverage":1,"random":0.486,"tracks":[],"combined":0.8215,"private_eq":0.9088},"43":{"raw":0.893,"skill":0.8302,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6624,"private_eq":0.789},"44":{"raw":0.8054,"skill":0.6108,"coverage":1,"random":0.5,"tracks":[],"combined":0.6316,"private_eq":0.8174},"45":{"raw":0.1238,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4403,"skill":0.1877,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.8655,"skill":0.731,"coverage":1,"random":0.5,"tracks":[],"combined":0.7069,"private_eq":0.8546},"57":{"raw":0.7252,"skill":0.6909,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5508,"combined":0.49},"58":{"raw":0.8353,"skill":0.7613,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.535,"combined":0.3192},"59":{"raw":0.8095,"skill":0.605,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4257,"private_eq":0.7257},"61":{"raw":0.8367,"skill":0.6734,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.8606,"skill":0.8141,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.8295,"skill":0.7869,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #215).","Submitted public 0.3 run adopted (kit rescore 65.46). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.80% on 831 rows (bar 98.5%).","Public score is the kit rescore of the published compact results (65.46); the submission states 65.08 and full results were not published.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":73.75,"balanced_skill":67.57,"breadth_skill":64.83,"public_skill":65.46},"frozen_scores":{"balanced_raw":73.75,"balanced_skill":65.46,"breadth_skill":64.83},"categories":[{"id":"knowledge","raw":0.6339,"skill":0.5383,"combined":0.4838,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.774,"skill":0.686,"combined":0.6282,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7469,"skill":0.6727,"combined":0.675,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8361,"skill":0.8151,"combined":0.7891,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7117,"skill":0.5442,"combined":0.5914,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"ajev-lm6q","pub":65.46,"same_skills":63.37,"new_domains":58.05,"v":67.57,"lab":false,"jev":false,"rank_v":15,"rank_pub":33,"name":"AJev lm6q (Gemma 4 31B)","eq":{"S":63.05,"O":74.34},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.83,"new_domains":29.74}},"new_in_edition":true},{"engine":"mammoth-1","name":"baselabs mammoth-1 (27B)","long_name":"baselabs mammoth-1 (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27356728560,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/baselabs/mammoth-1","code_url":null,"base_model":"Qwen/Qwen3.5-27B","base_url":"https://huggingface.co/Qwen/Qwen3.5-27B","weights_repo":"baselabs/mammoth-1","served_checkpoint":"9d1962f193e48a68cf0f3598dc7fb4462fd23ad9","org":{"id":"baselabs","name":"baselabs","logo":"data/orgs/baselabs.png"},"pr":260,"new":true,"show_notes":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8266,"conf":0.8653,"ece":0.0408,"brier":0.2377,"over95":0.0137,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0026,"conf":0.0772,"acc":0.2456},{"lo":0.1,"w":0.0116,"conf":0.1531,"acc":0.1653},{"lo":0.2,"w":0.0139,"conf":0.2517,"acc":0.2851},{"lo":0.3,"w":0.0192,"conf":0.3503,"acc":0.3059},{"lo":0.4,"w":0.0256,"conf":0.4546,"acc":0.3383},{"lo":0.5,"w":0.0698,"conf":0.5484,"acc":0.5254},{"lo":0.6,"w":0.0627,"conf":0.6487,"acc":0.5826},{"lo":0.7,"w":0.0572,"conf":0.7506,"acc":0.6205},{"lo":0.8,"w":0.0668,"conf":0.8539,"acc":0.7225},{"lo":0.9,"w":0.6705,"conf":0.9879,"acc":0.9671}],"areas":{"knowledge":{"ece":0.0563,"acc":0.779,"conf":0.8285},"language":{"ece":0.0153,"acc":0.9135,"conf":0.9271},"retrieval":{"ece":0.1139,"acc":0.7402,"conf":0.8541},"tools":{"ece":0.032,"acc":0.9557,"conf":0.9853},"arts":{"ece":0.0531,"acc":0.7567,"conf":0.7645}}},"gaps":null,"latency":{"median":220.1,"p95":712.5,"mean":435.1},"results":{"1":{"score":0.9823,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":331.3},"2":{"score":0.7042,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2517.5},"3":{"score":0.8898,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1519.8},"4":{"score":0.9391,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":306.5},"5":{"score":0.976,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":391},"6":{"score":0.7932,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":453.6},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":752.9},"10":{"score":0.4272,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":358.7},"11":{"score":0.8562,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":749.6},"12":{"score":0.8293,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":302.1},"20":{"score":0.991,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":298.6},"21":{"score":0.6944,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":298.8},"22":{"score":0.777,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1300.9},"23":{"score":0.6768,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":304.3},"24":{"score":0.9002,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":298.6},"25":{"score":0.5918,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":300.4},"26":{"score":0.9937,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":296.1},"27":{"score":0.9761,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":298.8},"28":{"score":0.9613,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":298.3},"29":{"score":0.9848,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":299},"30":{"score":0.8851,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.331,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":314.1},"32":{"score":0.9189,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":310.4},"33":{"score":0.6218,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":345.2},"36":{"score":0.4808,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4054.5},"37":{"score":0.6463,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":303},"38":{"score":0.68,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1143.9},"39":{"score":0.9536,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":302.7},"40":{"score":0.6808,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":301.6},"41":{"score":0.7545,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":300.5},"42":{"score":0.8793,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":305.3},"43":{"score":0.9544,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":298.9},"44":{"score":0.957,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":299.3},"45":{"score":0.1956,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":301.3},"50":{"score":0.6342,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":309.7},"56":{"score":0.971,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":328.7},"57":{"score":0.7495,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":299.4},"58":{"score":0.9624,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":298.3},"59":{"score":0.8582,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":310.2},"61":{"score":0.908,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":304.2},"62":{"score":0.9168,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":311.4},"64":{"score":0.8731,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":298.5}},"benchmarks":{"1":{"raw":0.9823,"skill":0.9761,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9669,"private_eq":0.975},"2":{"raw":0.7042,"skill":0.6585,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7045,"private_eq":0.7462},"3":{"raw":0.8898,"skill":0.8877,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9276,"private_eq":0.9288},"4":{"raw":0.9391,"skill":0.9383,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8784,"private_eq":0.8803},"5":{"raw":0.976,"skill":0.9759,"coverage":1,"random":0.006,"tracks":[],"combined":0.8186,"private_eq":0.8207},"6":{"raw":0.7932,"skill":0.5151,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9302,"private_eq":0.93},"10":{"raw":0.4272,"skill":0.0469,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8562,"skill":0.7921,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.8293,"skill":0.7443,"coverage":1,"random":0.3324,"tracks":[],"combined":0.473,"private_eq":0.6515},"20":{"raw":0.9913,"skill":0.9826,"coverage":1,"random":0.5,"tracks":[],"combined":0.9867,"private_eq":0.9929},"21":{"raw":0.6944,"skill":0.3889,"coverage":1,"random":0.5,"tracks":[],"combined":0.4378,"private_eq":0.7216},"22":{"raw":0.7746,"skill":0.7729,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5409,"private_eq":0.5487},"23":{"raw":0.695,"skill":0.39,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.9002,"skill":0.8669,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5918,"skill":0.4558,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6737,"combined":0.5609},"26":{"raw":0.9937,"skill":0.9916,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9761,"skill":0.9681,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9613,"skill":0.9226,"coverage":1,"random":0.5,"tracks":[],"combined":0.5774,"private_eq":0.7906},"29":{"raw":0.9848,"skill":0.9797,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8851,"skill":0.8619,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.906,"random":0.25},{"track":"10-choice","raw":0.8643,"random":0.1}],"combined":0.8475,"private_eq":0.8748},"31":{"raw":0.331,"skill":0.2714,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1026,"private_eq":0.1846},"32":{"raw":0.9189,"skill":0.8711,"coverage":1,"random":0.371,"tracks":[],"combined":0.4981,"private_eq":0.6873},"33":{"raw":0.6218,"skill":0.6168,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4512,"private_eq":0.4636},"36":{"raw":0.4808,"skill":0.4126,"coverage":1,"random":0.116,"tracks":[],"combined":0.4186,"private_eq":0.491},"37":{"raw":0.6463,"skill":0.5563,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5747,"private_eq":0.664},"38":{"raw":0.68,"skill":0.6698,"coverage":1,"random":0.031,"tracks":[],"combined":0.7744,"private_eq":0.7829},"39":{"raw":0.9536,"skill":0.9318,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5778,"private_eq":0.7155},"40":{"raw":0.6808,"skill":0.5893,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5419,"headline":false},{"track":"A · English","score":0.6808,"headline":true},{"track":"C · Arabic pairs","score":0.965,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.6435,"private_eq":0.7253},"41":{"raw":0.7545,"skill":0.6318,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5924,"private_eq":0.7307},"42":{"raw":0.8793,"skill":0.7652,"coverage":1,"random":0.486,"tracks":[],"combined":1.001,"private_eq":1},"43":{"raw":0.9544,"skill":0.9276,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5369,"private_eq":0.7108},"44":{"raw":0.957,"skill":0.914,"coverage":1,"random":0.5,"tracks":[],"combined":0.7975,"private_eq":0.8994},"45":{"raw":0.1956,"skill":0.0377,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6342,"skill":0.4692,"coverage":1,"random":0.311,"tracks":[],"combined":0.3338,"private_eq":0.5455},"56":{"raw":0.971,"skill":0.942,"coverage":1,"random":0.5,"tracks":[],"combined":0.6377,"private_eq":0.8204},"57":{"raw":0.7495,"skill":0.7183,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5596,"combined":0.5},"58":{"raw":0.9624,"skill":0.9455,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5598,"combined":0.3556},"59":{"raw":0.8582,"skill":0.706,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2777,"private_eq":0.6551},"61":{"raw":0.908,"skill":0.816,"coverage":1,"random":0.5,"tracks":[],"combined":0.6209,"private_eq":0.8121},"62":{"raw":0.9168,"skill":0.8891,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.8731,"skill":0.8414,"coverage":1,"random":0.2,"tracks":[],"combined":0.8361,"private_eq":0.8696}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #244); updated to revision 9d1962f in PR #260.","Public 0.3 run from PR #260 adopted (kit rescore 75.34, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving at revision 8f9f8619, whose weights are identical to 9d1962f; decision agreement with the submitted rows 99.85% on 831 rows (bar 98.5%).","Weights are public on the Hugging Face Hub under a proprietary license.","Training data not disclosed.","One private new-domain test is not counted for this model: our checks found signs its training data overlapped it. Its new-domain score uses the remaining tests, put on the same scale with the same stock models."],"scores":{"balanced_raw":81.19,"balanced_skill":68.44,"breadth_skill":75,"public_skill":75.34},"frozen_scores":{"balanced_raw":81.19,"balanced_skill":75.34,"breadth_skill":75},"categories":[{"id":"knowledge","raw":0.7351,"skill":0.6799,"combined":0.521,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.834,"skill":0.7611,"combined":0.6709,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.8182,"skill":0.7737,"combined":0.6624,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.9014,"skill":0.886,"combined":0.8343,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7771,"skill":0.6408,"combined":0.627,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"mammoth-1","pub":75.34,"same_skills":66.26,"new_domains":56.27,"v":68.44,"lab":false,"jev":false,"rank_v":9,"rank_pub":1,"name":"baselabs mammoth-1 (27B)","eq":{"S":65.98,"O":72.14},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":39.59,"new_domains":28.86}},"new_in_edition":true},{"engine":"elephant-1-baselabs","name":"baselabs elephant-1 (9B)","long_name":"baselabs elephant-1 (9B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":9409813744,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/baselabs/elephant-1","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":"baselabs/elephant-1","served_checkpoint":"14537f08b0374edf9867ed0a4d2bd8df59e0400c","org":{"id":"baselabs","name":"baselabs","logo":"data/orgs/baselabs.png"},"pr":243,"new":true,"show_notes":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8013,"conf":0.8571,"ece":0.0578,"brier":0.2711,"over95":0.0154,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0052,"conf":0.0795,"acc":0.1676},{"lo":0.1,"w":0.0138,"conf":0.148,"acc":0.1881},{"lo":0.2,"w":0.0108,"conf":0.2555,"acc":0.2489},{"lo":0.3,"w":0.0203,"conf":0.3552,"acc":0.2792},{"lo":0.4,"w":0.0242,"conf":0.4495,"acc":0.3536},{"lo":0.5,"w":0.0752,"conf":0.5483,"acc":0.4976},{"lo":0.6,"w":0.0584,"conf":0.6478,"acc":0.5074},{"lo":0.7,"w":0.06,"conf":0.7476,"acc":0.6155},{"lo":0.8,"w":0.086,"conf":0.8527,"acc":0.6906},{"lo":0.9,"w":0.6461,"conf":0.9852,"acc":0.9558}],"areas":{"knowledge":{"ece":0.0634,"acc":0.742,"conf":0.7992},"language":{"ece":0.0205,"acc":0.9025,"conf":0.9229},"retrieval":{"ece":0.1966,"acc":0.681,"conf":0.8774},"tools":{"ece":0.0281,"acc":0.9603,"conf":0.9831},"arts":{"ece":0.0353,"acc":0.7386,"conf":0.7627}}},"gaps":null,"latency":{"median":113.7,"p95":253.3,"mean":221.5},"results":{"1":{"score":0.9829,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":113.4},"2":{"score":0.6765,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":889.7},"3":{"score":0.8898,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":541.3},"4":{"score":0.9292,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":104.6},"5":{"score":0.9734,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":137.2},"6":{"score":0.7979,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":167.5},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":349.3},"10":{"score":0.0893,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":125.6},"11":{"score":0.8668,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":408.2},"12":{"score":0.7925,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":97.8},"20":{"score":0.9882,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":97.4},"21":{"score":0.6689,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":97},"22":{"score":0.7805,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":463.7},"23":{"score":0.6654,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":100.9},"24":{"score":0.8443,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":97.5},"25":{"score":0.4439,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":97.3},"26":{"score":0.9886,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":97.1},"27":{"score":0.9659,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":97.3},"28":{"score":0.9242,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":97.3},"29":{"score":0.9724,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":97.7},"30":{"score":0.7604,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2944,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":105.6},"32":{"score":0.8777,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":105.7},"33":{"score":0.5885,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":128.7},"36":{"score":0.4719,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1469.6},"37":{"score":0.6251,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":98.9},"38":{"score":0.6601,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":453},"39":{"score":0.9514,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":98.7},"40":{"score":0.6206,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":97.2},"41":{"score":0.7542,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":97.5},"42":{"score":0.8603,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":100.1},"43":{"score":0.9123,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":97.6},"44":{"score":0.9532,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":97.3},"45":{"score":0.1697,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":98.4},"50":{"score":0.6068,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":102.6},"56":{"score":0.969,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":120.3},"57":{"score":0.6462,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":97.1},"58":{"score":0.9515,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":97},"59":{"score":0.8526,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":103.4},"61":{"score":0.8992,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":99.8},"62":{"score":0.9146,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":103.6},"64":{"score":0.822,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":98.1}},"benchmarks":{"1":{"raw":0.9829,"skill":0.9769,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6765,"skill":0.6264,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7062,"private_eq":0.7476},"3":{"raw":0.8898,"skill":0.8877,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9118,"private_eq":0.9135},"4":{"raw":0.9292,"skill":0.9283,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8587,"private_eq":0.8611},"5":{"raw":0.9734,"skill":0.9732,"coverage":1,"random":0.006,"tracks":[],"combined":0.7673,"private_eq":0.7703},"6":{"raw":0.7979,"skill":0.5261,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8695,"private_eq":0.87},"10":{"raw":0.0893,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8668,"skill":0.8073,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7925,"skill":0.6892,"coverage":1,"random":0.3324,"tracks":[],"combined":0.509,"private_eq":0.6753},"20":{"raw":0.9882,"skill":0.9764,"coverage":1,"random":0.5,"tracks":[],"combined":0.9721,"private_eq":0.9857},"21":{"raw":0.6689,"skill":0.3379,"coverage":1,"random":0.5,"tracks":[],"combined":0.3672,"private_eq":0.6867},"22":{"raw":0.7829,"skill":0.7812,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5424,"private_eq":0.5501},"23":{"raw":0.6979,"skill":0.3958,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8443,"skill":0.7924,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5844,"combined":0.4405},"26":{"raw":0.9886,"skill":0.9848,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9659,"skill":0.9545,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9242,"skill":0.8484,"coverage":1,"random":0.5,"tracks":[],"combined":0.5873,"private_eq":0.7955},"29":{"raw":0.9724,"skill":0.9632,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7604,"skill":0.7134,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8165,"random":0.25},{"track":"10-choice","raw":0.7043,"random":0.1}],"combined":0.6835,"private_eq":0.741},"31":{"raw":0.2944,"skill":0.2315,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0962,"private_eq":0.1788},"32":{"raw":0.8777,"skill":0.8056,"coverage":1,"random":0.371,"tracks":[],"combined":0.4208,"private_eq":0.6392},"33":{"raw":0.5885,"skill":0.583,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4512,"private_eq":0.4636},"36":{"raw":0.4719,"skill":0.4025,"coverage":1,"random":0.116,"tracks":[],"combined":0.3946,"private_eq":0.47},"37":{"raw":0.6251,"skill":0.5298,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5617,"private_eq":0.6537},"38":{"raw":0.6601,"skill":0.6492,"coverage":1,"random":0.031,"tracks":[],"combined":0.7618,"private_eq":0.7708},"39":{"raw":0.9514,"skill":0.9285,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5767,"private_eq":0.7148},"40":{"raw":0.6206,"skill":0.5118,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4955,"headline":false},{"track":"A · English","score":0.6206,"headline":true},{"track":"C · Arabic pairs","score":0.97,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.532,"private_eq":0.6396},"41":{"raw":0.7542,"skill":0.6313,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5739,"private_eq":0.7185},"42":{"raw":0.8603,"skill":0.7282,"coverage":1,"random":0.486,"tracks":[],"combined":0.7416,"private_eq":0.8682},"43":{"raw":0.9123,"skill":0.8608,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2735,"private_eq":0.5467},"44":{"raw":0.9532,"skill":0.9064,"coverage":1,"random":0.5,"tracks":[],"combined":0.7922,"private_eq":0.8968},"45":{"raw":0.1697,"skill":0.0067,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6068,"skill":0.4293,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.969,"skill":0.938,"coverage":1,"random":0.5,"tracks":[],"combined":0.6288,"private_eq":0.816},"57":{"raw":0.6462,"skill":0.6021,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5733,"combined":0.5156},"58":{"raw":0.9515,"skill":0.9297,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5299,"combined":0.3118},"59":{"raw":0.8526,"skill":0.6944,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2206,"private_eq":0.6279},"61":{"raw":0.8992,"skill":0.7984,"coverage":1,"random":0.5,"tracks":[],"combined":0.6834,"private_eq":0.843},"62":{"raw":0.9146,"skill":0.8861,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.822,"skill":0.7775,"coverage":1,"random":0.2,"tracks":[],"combined":0.7489,"private_eq":0.8006}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #243).","Submitted public 0.3 run adopted (kit rescore 72.24, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.84% on 819 rows (bar 98.5%).","Weights are public on the Hugging Face Hub under a proprietary license.","Training includes train splits of Decision Index benchmarks (author-declared).","One private new-domain test is not counted for this model: our checks found signs its training data overlapped it. Its new-domain score uses the remaining tests, put on the same scale with the same stock models."],"scores":{"balanced_raw":78.88,"balanced_skill":60.26,"breadth_skill":71.68,"public_skill":72.24},"frozen_scores":{"balanced_raw":78.88,"balanced_skill":72.24,"breadth_skill":71.68},"categories":[{"id":"knowledge","raw":0.6791,"skill":0.6063,"combined":0.4577,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8174,"skill":0.7376,"combined":0.6213,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.8095,"skill":0.7623,"combined":0.6513,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.896,"skill":0.8796,"combined":0.8207,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7611,"skill":0.6163,"combined":0.5667,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"elephant-1-baselabs","pub":72.24,"same_skills":62.44,"new_domains":44.41,"v":60.26,"lab":false,"jev":false,"rank_v":47,"rank_pub":4,"name":"baselabs elephant-1 (9B)","eq":{"S":62.11,"O":57.49},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":37.27,"new_domains":22.99}},"new_in_edition":true},{"engine":"matilda-decision-v1","name":"Matilda Decision v1","long_name":"Matilda Decision v1","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Maincode/matilda-decision-v1","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"Maincode","name":"Maincode","logo":"data/orgs/Maincode.png"},"pr":189,"replaces":"matilda-jev-v1.5","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7738,"conf":0.8584,"ece":0.0858,"brier":0.3273,"over95":0.0293,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0003,"conf":0.0826,"acc":0.2341},{"lo":0.1,"w":0.008,"conf":0.1597,"acc":0.129},{"lo":0.2,"w":0.0137,"conf":0.2546,"acc":0.2941},{"lo":0.3,"w":0.0164,"conf":0.3507,"acc":0.3135},{"lo":0.4,"w":0.0313,"conf":0.4595,"acc":0.339},{"lo":0.5,"w":0.0705,"conf":0.5481,"acc":0.4487},{"lo":0.6,"w":0.0668,"conf":0.647,"acc":0.5313},{"lo":0.7,"w":0.0729,"conf":0.7514,"acc":0.5549},{"lo":0.8,"w":0.0931,"conf":0.8543,"acc":0.6751},{"lo":0.9,"w":0.6271,"conf":0.9845,"acc":0.929}],"areas":{"knowledge":{"ece":0.1495,"acc":0.6885,"conf":0.8376},"language":{"ece":0.0276,"acc":0.8976,"conf":0.9162},"retrieval":{"ece":0.118,"acc":0.7423,"conf":0.8598},"tools":{"ece":0.0381,"acc":0.9487,"conf":0.9846},"arts":{"ece":0.0998,"acc":0.6417,"conf":0.7189}}},"gaps":null,"latency":{"median":91.9,"p95":1150.1,"mean":308.2},"results":{"1":{"score":0.9823,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":128.1},"2":{"score":0.6837,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1420.3},"3":{"score":0.8622,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":536.5},"4":{"score":0.9048,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":89.9},"5":{"score":0.9275,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":131.7},"6":{"score":0.7958,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":320.6},"9":{"score":0.875,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1304.4},"10":{"score":0.5234,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":107.7},"11":{"score":0.8546,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2071.5},"12":{"score":0.7629,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":51.7},"20":{"score":0.9516,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.8},"21":{"score":0.6533,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":51.1},"22":{"score":0.5775,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":456.6},"23":{"score":0.6484,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.7},"24":{"score":0.8979,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":51.7},"25":{"score":0.5357,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":51.9},"26":{"score":0.9907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":51.2},"27":{"score":0.9693,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.2},"28":{"score":0.9006,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":51.1},"29":{"score":0.9592,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":53.3},"30":{"score":0.7627,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2266,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":92.4},"32":{"score":0.6662,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":97.1},"33":{"score":0.2297,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":255.5},"36":{"score":0.4842,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2360.5},"37":{"score":0.615,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":57.9},"38":{"score":0.3805,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":762.3},"39":{"score":0.9422,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":59},"40":{"score":0.6963,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":53},"41":{"score":0.8097,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":52.6},"42":{"score":0.8503,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":65},"43":{"score":0.8053,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":51.5},"44":{"score":0.8064,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52},"45":{"score":0.1677,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":53.9},"50":{"score":0.3801,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":90.4},"56":{"score":0.742,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":182.9},"57":{"score":0.8069,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":52.3},"58":{"score":0.8313,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":52},"59":{"score":0.8541,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":84.3},"61":{"score":0.8812,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":61.4},"62":{"score":0.8918,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":82.7},"64":{"score":0.7235,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":51.7}},"benchmarks":{"1":{"raw":0.9823,"skill":0.9761,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9805,"private_eq":0.985},"2":{"raw":0.6837,"skill":0.6347,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7084,"private_eq":0.7495},"3":{"raw":0.8622,"skill":0.8595,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.9048,"skill":0.9036,"coverage":1,"random":0.0127,"tracks":[],"combined":0.875,"private_eq":0.877},"5":{"raw":0.9275,"skill":0.9271,"coverage":1,"random":0.006,"tracks":[],"combined":0.7848,"private_eq":0.7875},"6":{"raw":0.7958,"skill":0.5212,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.875,"skill":0.875,"coverage":1,"random":0,"tracks":[],"combined":0.7583,"private_eq":0.76},"10":{"raw":0.5234,"skill":0.207,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8546,"skill":0.7897,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7629,"skill":0.6448,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5481,"private_eq":0.7011},"20":{"raw":0.9526,"skill":0.9052,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.6533,"skill":0.3067,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.5734,"skill":0.5701,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4533,"private_eq":0.4627},"23":{"raw":0.6526,"skill":0.3052,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8979,"skill":0.8639,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5357,"skill":0.381,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5127,"combined":0.3438},"26":{"raw":0.9907,"skill":0.9876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9693,"skill":0.9591,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9006,"skill":0.8012,"coverage":1,"random":0.5,"tracks":[],"combined":0.5873,"private_eq":0.7955},"29":{"raw":0.9592,"skill":0.9456,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7627,"skill":0.7169,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8249,"random":0.25},{"track":"10-choice","raw":0.7005,"random":0.1}],"combined":0.7431,"private_eq":0.7896},"31":{"raw":0.2266,"skill":0.1576,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0918,"private_eq":0.1748},"32":{"raw":0.6662,"skill":0.4693,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.2297,"skill":0.2195,"coverage":1,"random":0.0131,"tracks":[],"combined":0.22,"private_eq":0.238},"36":{"raw":0.4842,"skill":0.4165,"coverage":1,"random":0.116,"tracks":[],"combined":0.4142,"private_eq":0.4872},"37":{"raw":0.615,"skill":0.5171,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5898,"private_eq":0.6759},"38":{"raw":0.3805,"skill":0.3607,"coverage":1,"random":0.031,"tracks":[],"combined":0.4989,"private_eq":0.519},"39":{"raw":0.9422,"skill":0.915,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5157,"private_eq":0.6738},"40":{"raw":0.6963,"skill":0.6093,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5523,"headline":false},{"track":"A · English","score":0.6963,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.6018,"private_eq":0.6932},"41":{"raw":0.8097,"skill":0.7145,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1623,"private_eq":0.4472},"42":{"raw":0.8503,"skill":0.7088,"coverage":1,"random":0.486,"tracks":[],"combined":0.904,"private_eq":0.9507},"43":{"raw":0.8053,"skill":0.691,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5663,"private_eq":0.7291},"44":{"raw":0.8064,"skill":0.6128,"coverage":1,"random":0.5,"tracks":[],"combined":0.5727,"private_eq":0.7883},"45":{"raw":0.1677,"skill":0.0043,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3801,"skill":0.1003,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.742,"skill":0.484,"coverage":1,"random":0.5,"tracks":[],"combined":0.4819,"private_eq":0.7434},"57":{"raw":0.8069,"skill":0.7828,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6934,"combined":0.6522},"58":{"raw":0.8313,"skill":0.7555,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5365,"combined":0.3214},"59":{"raw":0.8541,"skill":0.6975,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3699,"private_eq":0.6991},"61":{"raw":0.8812,"skill":0.7624,"coverage":1,"random":0.5,"tracks":[],"combined":0.6605,"private_eq":0.8317},"62":{"raw":0.8918,"skill":0.8557,"coverage":1,"random":0.25,"tracks":[],"combined":0.6246,"private_eq":0.7209},"64":{"raw":0.7235,"skill":0.6544,"coverage":1,"random":0.2,"tracks":[],"combined":0.6422,"private_eq":0.7162}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #189).","Replaces the earlier board entry matilda-jev-v1.5 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 65.38, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.79% on 827 rows (bar 98.5%)."],"scores":{"balanced_raw":73.98,"balanced_skill":64.75,"breadth_skill":64.31,"public_skill":65.39},"frozen_scores":{"balanced_raw":73.98,"balanced_skill":65.39,"breadth_skill":64.31},"categories":[{"id":"knowledge","raw":0.6156,"skill":0.5114,"combined":0.4501,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.793,"skill":0.7132,"combined":0.5847,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7603,"skill":0.6758,"combined":0.6395,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8637,"skill":0.8459,"combined":0.8101,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6559,"skill":0.4736,"combined":0.5031,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"matilda-decision-v1","pub":65.39,"same_skills":60.34,"new_domains":56.07,"v":64.75,"lab":false,"jev":false,"rank_v":32,"rank_pub":36,"name":"Matilda Decision v1","eq":{"S":59.98,"O":71.9},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":35.99,"new_domains":28.76}},"new_in_edition":true},{"engine":"sieve-27b-v2","name":"Sieve 27B v2","long_name":"Sieve 27B v2","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/Sieve-27B-v2","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"},"pr":219,"replaces":"sieve-27b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7961,"conf":0.8485,"ece":0.0531,"brier":0.2832,"over95":0.027,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0074,"conf":0.0808,"acc":0.0902},{"lo":0.1,"w":0.0151,"conf":0.1432,"acc":0.1643},{"lo":0.2,"w":0.0104,"conf":0.2494,"acc":0.2101},{"lo":0.3,"w":0.0296,"conf":0.3571,"acc":0.2711},{"lo":0.4,"w":0.0356,"conf":0.451,"acc":0.4243},{"lo":0.5,"w":0.064,"conf":0.5516,"acc":0.4672},{"lo":0.6,"w":0.0578,"conf":0.6474,"acc":0.5745},{"lo":0.7,"w":0.0642,"conf":0.7506,"acc":0.6945},{"lo":0.8,"w":0.0754,"conf":0.8546,"acc":0.7547},{"lo":0.9,"w":0.6407,"conf":0.9852,"acc":0.9413}],"areas":{"knowledge":{"ece":0.0656,"acc":0.7119,"conf":0.7752},"language":{"ece":0.0235,"acc":0.8974,"conf":0.9202},"retrieval":{"ece":0.1353,"acc":0.7327,"conf":0.8564},"tools":{"ece":0.0275,"acc":0.9527,"conf":0.9798},"arts":{"ece":0.0553,"acc":0.739,"conf":0.7807}}},"gaps":null,"latency":{"median":106.1,"p95":602.3,"mean":268.1},"results":{"1":{"score":0.9604,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":460.5},"2":{"score":0.6854,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":4079.7},"3":{"score":0.8898,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1609.9},"4":{"score":0.8995,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":237.4},"5":{"score":0.9088,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":362.7},"6":{"score":0.7977,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":751.2},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1213.9},"10":{"score":0.4645,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":265.6},"11":{"score":0.8341,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1258.1},"12":{"score":0.752,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":118.8},"20":{"score":0.98,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.2},"21":{"score":0.6012,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":107.6},"22":{"score":0.833,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1558.1},"23":{"score":0.648,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":196.3},"24":{"score":0.8228,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":110.8},"25":{"score":0.4949,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":126.9},"26":{"score":0.9853,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":107.6},"27":{"score":0.9616,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":108.8},"28":{"score":0.9519,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":105.9},"29":{"score":0.97,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":126.8},"30":{"score":0.7199,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2474,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":212.5},"32":{"score":0.883,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":224.4},"33":{"score":0.4291,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":660.6},"36":{"score":0.4605,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":7149.6},"37":{"score":0.6132,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":139.9},"38":{"score":0.4834,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1680.6},"39":{"score":0.9322,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":324.3},"40":{"score":0.4729,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":107.9},"41":{"score":0.7809,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":122.6},"42":{"score":0.8542,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":160.1},"43":{"score":0.8175,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":116.4},"44":{"score":0.9326,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.9},"45":{"score":0.1038,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":130.2},"50":{"score":0.6569,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":214.1},"56":{"score":0.9665,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":458.5},"57":{"score":0.6242,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":124.8},"58":{"score":0.8823,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":119.5},"59":{"score":0.7399,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":194.3},"61":{"score":0.8798,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":153.7},"62":{"score":0.8475,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":190.5},"64":{"score":0.7803,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":125}},"benchmarks":{"1":{"raw":0.9604,"skill":0.9466,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.6854,"skill":0.6366,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7076,"private_eq":0.7488},"3":{"raw":0.8898,"skill":0.8877,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9118,"private_eq":0.9135},"4":{"raw":0.8995,"skill":0.8982,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8798,"private_eq":0.8817},"5":{"raw":0.9088,"skill":0.9082,"coverage":1,"random":0.006,"tracks":[],"combined":0.7309,"private_eq":0.7346},"6":{"raw":0.7977,"skill":0.5257,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9201,"private_eq":0.92},"10":{"raw":0.4645,"skill":0.109,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8341,"skill":0.7601,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.752,"skill":0.6285,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5957,"private_eq":0.7325},"20":{"raw":0.979,"skill":0.9581,"coverage":1,"random":0.5,"tracks":[],"combined":0.9721,"private_eq":0.9857},"21":{"raw":0.6012,"skill":0.2024,"coverage":1,"random":0.5,"tracks":[],"combined":0.2565,"private_eq":0.632},"22":{"raw":0.8311,"skill":0.8298,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6875,"private_eq":0.6925},"23":{"raw":0.659,"skill":0.3179,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8228,"skill":0.7637,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4949,"skill":0.3265,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4483,"combined":0.2569},"26":{"raw":0.9853,"skill":0.9804,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9616,"skill":0.9488,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9519,"skill":0.9038,"coverage":1,"random":0.5,"tracks":[],"combined":0.542,"private_eq":0.7731},"29":{"raw":0.97,"skill":0.96,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7199,"skill":0.6651,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.787,"random":0.25},{"track":"10-choice","raw":0.6528,"random":0.1}],"combined":0.6817,"private_eq":0.7396},"31":{"raw":0.2474,"skill":0.1803,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0875,"private_eq":0.1709},"32":{"raw":0.883,"skill":0.814,"coverage":1,"random":0.371,"tracks":[],"combined":0.2736,"private_eq":0.5477},"33":{"raw":0.4291,"skill":0.4215,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4411,"private_eq":0.4537},"36":{"raw":0.4605,"skill":0.3897,"coverage":1,"random":0.116,"tracks":[],"combined":0.414,"private_eq":0.487},"37":{"raw":0.6132,"skill":0.5148,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5013,"private_eq":0.6061},"38":{"raw":0.4834,"skill":0.4669,"coverage":1,"random":0.031,"tracks":[],"combined":0.5809,"private_eq":0.5975},"39":{"raw":0.9322,"skill":0.9003,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5947,"private_eq":0.7269},"40":{"raw":0.4729,"skill":0.3219,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5014,"headline":false},{"track":"A · English","score":0.4729,"headline":true},{"track":"C · Arabic pairs","score":0.755,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0.3165,"private_eq":0.474},"41":{"raw":0.7809,"skill":0.6713,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4302,"private_eq":0.6238},"42":{"raw":0.8542,"skill":0.7163,"coverage":1,"random":0.486,"tracks":[],"combined":0.9583,"private_eq":0.9783},"43":{"raw":0.8175,"skill":0.7105,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5286,"private_eq":0.7056},"44":{"raw":0.9326,"skill":0.8652,"coverage":1,"random":0.5,"tracks":[],"combined":0.803,"private_eq":0.9021},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6569,"skill":0.5021,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.9665,"skill":0.933,"coverage":1,"random":0.5,"tracks":[],"combined":0.3433,"private_eq":0.6749},"57":{"raw":0.6242,"skill":0.5773,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4248,"combined":0.3466},"58":{"raw":0.8823,"skill":0.8294,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4677,"combined":0.2206},"59":{"raw":0.7399,"skill":0.4607,"coverage":1,"random":0.5177,"tracks":[],"combined":0.323,"private_eq":0.6767},"61":{"raw":0.8798,"skill":0.7596,"coverage":1,"random":0.5,"tracks":[],"combined":0.6492,"private_eq":0.8261},"62":{"raw":0.8475,"skill":0.7967,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.7803,"skill":0.7254,"coverage":1,"random":0.2,"tracks":[],"combined":0.7198,"private_eq":0.7776}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #219).","Replaces the earlier board entry sieve-27b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 68.02, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.70% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":75.61,"balanced_skill":59.89,"breadth_skill":67.38,"public_skill":68.02},"frozen_scores":{"balanced_raw":75.61,"balanced_skill":68.02,"breadth_skill":67.38},"categories":[{"id":"knowledge","raw":0.6415,"skill":0.5684,"combined":0.4125,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7621,"skill":0.66,"combined":0.6095,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7852,"skill":0.7337,"combined":0.5945,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8802,"skill":0.8582,"combined":0.8288,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7512,"skill":0.5893,"combined":0.5722,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-27b-v2","pub":68.02,"same_skills":60.03,"new_domains":46.62,"v":59.89,"lab":false,"jev":false,"rank_v":51,"rank_pub":18,"name":"Sieve 27B v2","eq":{"S":59.67,"O":60.23},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.8,"new_domains":24.09}},"new_in_edition":true},{"engine":"cerno-27b","name":"Cerno 27B","long_name":"Cerno 27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26895998464,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/apertixai/cerno-27b","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"pr":210,"new":true},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.7612,"conf":0.8334,"ece":0.0723,"brier":0.3293,"over95":0.0159,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0006,"conf":0.0914,"acc":0.1795},{"lo":0.1,"w":0.0115,"conf":0.1593,"acc":0.1559},{"lo":0.2,"w":0.0185,"conf":0.2519,"acc":0.2311},{"lo":0.3,"w":0.027,"conf":0.356,"acc":0.3205},{"lo":0.4,"w":0.0366,"conf":0.4546,"acc":0.3158},{"lo":0.5,"w":0.076,"conf":0.5481,"acc":0.4612},{"lo":0.6,"w":0.0723,"conf":0.6496,"acc":0.5324},{"lo":0.7,"w":0.0824,"conf":0.7499,"acc":0.5953},{"lo":0.8,"w":0.0994,"conf":0.8557,"acc":0.6889},{"lo":0.9,"w":0.5758,"conf":0.9815,"acc":0.9443}],"areas":{"knowledge":{"ece":0.139,"acc":0.6637,"conf":0.8027},"language":{"ece":0.0144,"acc":0.8857,"conf":0.891},"retrieval":{"ece":0.1286,"acc":0.7213,"conf":0.8499},"tools":{"ece":0.0383,"acc":0.9466,"conf":0.9842},"arts":{"ece":0.0431,"acc":0.6525,"conf":0.6874}}},"gaps":null,"latency":{"median":53.9,"p95":418,"mean":135.7},"results":{"1":{"score":0.9787,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":179.4},"2":{"score":0.6944,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":694.2},"3":{"score":0.8583,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":492.9},"4":{"score":0.9228,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":107.1},"5":{"score":0.9568,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":136.4},"6":{"score":0.7982,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":261.8},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":578.4},"10":{"score":0.3871,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":133.8},"11":{"score":0.8419,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":792.8},"12":{"score":0.7639,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":48.3},"20":{"score":0.9442,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.4},"21":{"score":0.6507,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":47.6},"22":{"score":0.3952,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":425.6},"23":{"score":0.6616,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":64.5},"24":{"score":0.8597,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":48.8},"25":{"score":0.4796,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":49.3},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":48.2},"27":{"score":0.9701,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":48.2},"28":{"score":0.8982,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":47.9},"29":{"score":0.9614,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":49.1},"30":{"score":0.7707,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1846,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111},"32":{"score":0.6503,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":112.3},"33":{"score":0.1539,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":280.3},"36":{"score":0.5034,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1122.4},"37":{"score":0.6039,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.3},"38":{"score":0.3962,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":629.2},"39":{"score":0.9164,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":92.3},"40":{"score":0.6944,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":48.6},"41":{"score":0.8112,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":48.6},"42":{"score":0.848,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":57},"43":{"score":0.7702,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":48.5},"44":{"score":0.8382,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.4},"45":{"score":0.1257,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":49.6},"50":{"score":0.6187,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":110.4},"56":{"score":0.6435,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":206.5},"57":{"score":0.6979,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":48.8},"58":{"score":0.9036,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":48.3},"59":{"score":0.8617,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":108.1},"61":{"score":0.891,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":50.7},"62":{"score":0.8839,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":65.9},"64":{"score":0.7614,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":49.3}},"benchmarks":{"1":{"raw":0.9787,"skill":0.9713,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.6934,"skill":0.646,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.7048,"private_eq":0.7464},"3":{"raw":0.8583,"skill":0.8556,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9171,"private_eq":0.9186},"4":{"raw":0.9228,"skill":0.9218,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8762,"private_eq":0.8782},"5":{"raw":0.9568,"skill":0.9565,"coverage":1,"random":0.006,"tracks":[],"combined":0.7669,"private_eq":0.7699},"6":{"raw":0.7982,"skill":0.5268,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8594,"private_eq":0.86},"10":{"raw":0.3871,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8419,"skill":0.7714,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7639,"skill":0.6463,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4704,"private_eq":0.6498},"20":{"raw":0.9454,"skill":0.8908,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.6507,"skill":0.3014,"coverage":1,"random":0.5,"tracks":[],"combined":0.3573,"private_eq":0.6818},"22":{"raw":0.393,"skill":0.3883,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.3777,"private_eq":0.3886},"23":{"raw":0.6632,"skill":0.3264,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8597,"skill":0.8129,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4836,"combined":0.3045},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8982,"skill":0.7964,"coverage":1,"random":0.5,"tracks":[],"combined":0.5774,"private_eq":0.7906},"29":{"raw":0.9614,"skill":0.9485,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7707,"skill":0.7261,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8287,"random":0.25},{"track":"10-choice","raw":0.7127,"random":0.1}],"combined":0.7347,"private_eq":0.7828},"31":{"raw":0.1846,"skill":0.1119,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0875,"private_eq":0.1709},"32":{"raw":0.6503,"skill":0.444,"coverage":1,"random":0.371,"tracks":[],"combined":0.2736,"private_eq":0.5477},"33":{"raw":0.1539,"skill":0.1427,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.5034,"skill":0.4382,"coverage":1,"random":0.116,"tracks":[],"combined":0.4071,"private_eq":0.481},"37":{"raw":0.6039,"skill":0.5031,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5678,"private_eq":0.6585},"38":{"raw":0.3962,"skill":0.3769,"coverage":1,"random":0.031,"tracks":[],"combined":0.5261,"private_eq":0.545},"39":{"raw":0.9164,"skill":0.877,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5635,"private_eq":0.7059},"40":{"raw":0.6944,"skill":0.6069,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1532,"headline":false},{"track":"A · English","score":0.6944,"headline":true},{"track":"C · Arabic pairs","score":0.75,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.5864,"private_eq":0.6814},"41":{"raw":0.8112,"skill":0.7167,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1471,"private_eq":0.4372},"42":{"raw":0.848,"skill":0.7043,"coverage":1,"random":0.486,"tracks":[],"combined":0.9672,"private_eq":0.9828},"43":{"raw":0.7702,"skill":0.6354,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5788,"private_eq":0.7369},"44":{"raw":0.8382,"skill":0.6764,"coverage":1,"random":0.5,"tracks":[],"combined":0.6102,"private_eq":0.8068},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6187,"skill":0.4467,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.6435,"skill":0.287,"coverage":1,"random":0.5,"tracks":[],"combined":0.4299,"private_eq":0.7177},"57":{"raw":0.6979,"skill":0.6602,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5566,"combined":0.4966},"58":{"raw":0.9036,"skill":0.8603,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5089,"combined":0.281},"59":{"raw":0.8617,"skill":0.7132,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3081,"private_eq":0.6696},"61":{"raw":0.891,"skill":0.782,"coverage":1,"random":0.5,"tracks":[],"combined":0.6947,"private_eq":0.8486},"62":{"raw":0.8839,"skill":0.8452,"coverage":1,"random":0.25,"tracks":[],"combined":0.7067,"private_eq":0.7818},"64":{"raw":0.7614,"skill":0.7017,"coverage":1,"random":0.2,"tracks":[],"combined":0.7198,"private_eq":0.7776}},"coverage":0.9999,"pending":0.0001,"notes":["New in the 2026-10-18 PR batch (PR #210).","Submitted public 0.3 run adopted (kit rescore 65.19, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8); decision agreement with the submitted rows 98.70% on 824 rows (bar 96%).","Served in FP8 as published, so the quantized agreement bar applies.","The submitted RouterBench rows were built from a different suite payload; the lab's own rows are used for the private tests."],"scores":{"balanced_raw":73.81,"balanced_skill":61.68,"breadth_skill":64.01,"public_skill":65.19},"frozen_scores":{"balanced_raw":73.81,"balanced_skill":65.19,"breadth_skill":64.01},"categories":[{"id":"knowledge","raw":0.5912,"skill":0.4903,"combined":0.395,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7915,"skill":0.7107,"combined":0.579,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7573,"skill":0.6594,"combined":0.6292,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8855,"skill":0.8673,"combined":0.8301,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.6721,"skill":0.5092,"combined":0.513,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"cerno-27b","pub":65.19,"same_skills":59.09,"new_domains":51.39,"v":61.68,"lab":false,"jev":false,"rank_v":40,"rank_pub":38,"name":"Cerno 27B","eq":{"S":58.72,"O":66.11},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":35.23,"new_domains":26.44}},"new_in_edition":true},{"engine":"jev-brain-gemma-4-31b","name":"jev-brain · Gemma-4-31B","long_name":"jev-brain · Gemma-4-31B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/wcole3/jev-brain/tree/di-0.3-gemma-4-31b-r2","base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":193,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7399,"conf":0.872,"ece":0.133,"brier":0.4095,"over95":0.07,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0037,"conf":0.0708,"acc":0.027},{"lo":0.1,"w":0.0111,"conf":0.151,"acc":0.1681},{"lo":0.2,"w":0.0184,"conf":0.2521,"acc":0.2638},{"lo":0.3,"w":0.019,"conf":0.3502,"acc":0.295},{"lo":0.4,"w":0.0197,"conf":0.4531,"acc":0.4096},{"lo":0.5,"w":0.0304,"conf":0.5504,"acc":0.3292},{"lo":0.6,"w":0.04,"conf":0.6558,"acc":0.4746},{"lo":0.7,"w":0.0837,"conf":0.7549,"acc":0.6139},{"lo":0.8,"w":0.1101,"conf":0.8463,"acc":0.6694},{"lo":0.9,"w":0.6639,"conf":0.9798,"acc":0.8515}],"areas":{"knowledge":{"ece":0.1563,"acc":0.6889,"conf":0.8451},"language":{"ece":0.0996,"acc":0.8591,"conf":0.9577},"retrieval":{"ece":0.2213,"acc":0.6158,"conf":0.8165},"tools":{"ece":0.0233,"acc":0.9385,"conf":0.9189},"arts":{"ece":0.2255,"acc":0.6074,"conf":0.7879}}},"gaps":null,"latency":{"median":100.5,"p95":818.9,"mean":373.6},"results":{"1":{"score":0.974,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":741.9},"2":{"score":0.6691,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":17238.2},"3":{"score":0.8504,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":8000.9},"4":{"score":0.7894,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1043.6},"5":{"score":0.908,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1628.8},"6":{"score":0.7991,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":2186.2},"9":{"score":0.625,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2863.9},"10":{"score":0.067,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":1132.8},"11":{"score":0.7843,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3083.5},"12":{"score":0.7851,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":275.2},"20":{"score":0.906,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":149.5},"21":{"score":0.629,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":131.8},"22":{"score":0.3785,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":6779.3},"23":{"score":0.6584,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":158.2},"24":{"score":0.8704,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":185},"25":{"score":0.4847,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":325.9},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":159.8},"27":{"score":0.9787,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":173.4},"28":{"score":0.8753,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":147.2},"29":{"score":0.9186,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":331.3},"30":{"score":0.7324,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2324,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":869.4},"32":{"score":0.6848,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":1215.2},"33":{"score":0.3455,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":963.7},"36":{"score":0.4813,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":27431.6},"37":{"score":0.5712,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":377.3},"38":{"score":0.402,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":7858.5},"39":{"score":0.9338,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":321.7},"40":{"score":0.5962,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":144},"41":{"score":0.7861,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":322.8},"42":{"score":0.808,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":732.2},"43":{"score":0.8035,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":199.5},"44":{"score":0.8186,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":226.2},"45":{"score":0.1477,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":450.6},"50":{"score":0.4839,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":761.5},"56":{"score":0.969,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":767.7},"57":{"score":0.6912,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":237.8},"58":{"score":0.751,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":165.4},"59":{"score":0.7791,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":337.7},"61":{"score":0.7658,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":420.6},"62":{"score":0.7913,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":348},"64":{"score":0.7746,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":182.2}},"benchmarks":{"1":{"raw":0.974,"skill":0.9649,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9329,"private_eq":0.9501},"2":{"raw":0.6691,"skill":0.6178,"coverage":1,"random":0.1341,"tracks":[],"combined":0.708,"private_eq":0.7492},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.7894,"skill":0.7867,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8626,"private_eq":0.8649},"5":{"raw":0.908,"skill":0.9074,"coverage":1,"random":0.006,"tracks":[],"combined":0.7717,"private_eq":0.7747},"6":{"raw":0.7991,"skill":0.529,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.625,"skill":0.625,"coverage":1,"random":0,"tracks":[],"combined":0.5888,"private_eq":0.5925},"10":{"raw":0.067,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7843,"skill":0.688,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7851,"skill":0.6781,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5459,"private_eq":0.6996},"20":{"raw":0.9084,"skill":0.8168,"coverage":1,"random":0.5,"tracks":[],"combined":0.8566,"private_eq":0.9286},"21":{"raw":0.629,"skill":0.258,"coverage":1,"random":0.5,"tracks":[],"combined":0.1962,"private_eq":0.6022},"22":{"raw":0.3832,"skill":0.3784,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3814,"private_eq":0.3922},"23":{"raw":0.6596,"skill":0.3193,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8704,"skill":0.8272,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4847,"skill":0.3129,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5106,"combined":0.341},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9787,"skill":0.9716,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8753,"skill":0.7506,"coverage":1,"random":0.5,"tracks":[],"combined":0.7235,"private_eq":0.8628},"29":{"raw":0.9186,"skill":0.8915,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7324,"skill":0.6814,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8089,"random":0.25},{"track":"10-choice","raw":0.6558,"random":0.1}],"combined":0.76,"private_eq":0.8034},"31":{"raw":0.2324,"skill":0.164,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1015,"private_eq":0.1836},"32":{"raw":0.6848,"skill":0.4989,"coverage":1,"random":0.371,"tracks":[],"combined":0.4285,"private_eq":0.644},"33":{"raw":0.3455,"skill":0.3368,"coverage":1,"random":0.0131,"tracks":[],"combined":0.477,"private_eq":0.4887},"36":{"raw":0.4813,"skill":0.4133,"coverage":1,"random":0.116,"tracks":[],"combined":0.4134,"private_eq":0.4865},"37":{"raw":0.5712,"skill":0.4621,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5756,"private_eq":0.6647},"38":{"raw":0.402,"skill":0.3829,"coverage":1,"random":0.031,"tracks":[],"combined":0.6043,"private_eq":0.62},"39":{"raw":0.9338,"skill":0.9026,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5966,"private_eq":0.7282},"40":{"raw":0.5962,"skill":0.4805,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5261,"headline":false},{"track":"A · English","score":0.5962,"headline":true},{"track":"C · Arabic pairs","score":0.905,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.4607,"private_eq":0.5848},"41":{"raw":0.7861,"skill":0.6791,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5141,"private_eq":0.6791},"42":{"raw":0.808,"skill":0.6265,"coverage":1,"random":0.486,"tracks":[],"combined":0.717,"private_eq":0.8557},"43":{"raw":0.8035,"skill":0.6882,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5997,"private_eq":0.7499},"44":{"raw":0.8186,"skill":0.6372,"coverage":1,"random":0.5,"tracks":[],"combined":0.6102,"private_eq":0.8068},"45":{"raw":0.1477,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4839,"skill":0.251,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.969,"skill":0.938,"coverage":1,"random":0.5,"tracks":[],"combined":0.8452,"private_eq":0.923},"57":{"raw":0.6912,"skill":0.6527,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5587,"combined":0.499},"58":{"raw":0.751,"skill":0.6391,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5397,"combined":0.3261},"59":{"raw":0.7791,"skill":0.542,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3771,"private_eq":0.7025},"61":{"raw":0.7658,"skill":0.5316,"coverage":1,"random":0.5,"tracks":[],"combined":0.6209,"private_eq":0.8121},"62":{"raw":0.7913,"skill":0.7217,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.7746,"skill":0.7182,"coverage":1,"random":0.2,"tracks":[],"combined":0.7392,"private_eq":0.7929}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #193).","Submitted public 0.3 run adopted (kit rescore 61.69, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.35% on 891 rows (bar 98.5%).","Inference technique on frozen Gemma 4 31B weights.","The submitted results file has extra lines and duplicate questions with differing answers; the board uses the adopted rows that match the suite."],"scores":{"balanced_raw":70.83,"balanced_skill":64.61,"breadth_skill":60.82,"public_skill":61.69},"frozen_scores":{"balanced_raw":70.83,"balanced_skill":61.69,"breadth_skill":60.82},"categories":[{"id":"knowledge","raw":0.5964,"skill":0.4861,"combined":0.4856,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7589,"skill":0.658,"combined":0.6103,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7455,"skill":0.6758,"combined":0.6817,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7916,"skill":0.7666,"combined":0.7589,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6398,"skill":0.4569,"combined":0.4825,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-brain-gemma-4-31b","pub":61.69,"same_skills":61.56,"new_domains":54.29,"v":64.61,"lab":false,"jev":false,"rank_v":33,"rank_pub":51,"name":"jev-brain · Gemma-4-31B","eq":{"S":61.22,"O":69.7},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":36.73,"new_domains":27.88}},"new_in_edition":true},{"engine":"onejev1.6-27b","name":"OneJev1.6-27B","long_name":"OneJev1.6-27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27356728560,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/OmniJev/OneJev1.6-27B","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pr":217,"replaces":"onejev1.5-27b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8056,"conf":0.8298,"ece":0.0269,"brier":0.2543,"over95":0.0091,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0021,"conf":0.0876,"acc":0.1296},{"lo":0.1,"w":0.0181,"conf":0.1505,"acc":0.1374},{"lo":0.2,"w":0.0136,"conf":0.2555,"acc":0.1595},{"lo":0.3,"w":0.0254,"conf":0.3564,"acc":0.2831},{"lo":0.4,"w":0.0361,"conf":0.4588,"acc":0.3354},{"lo":0.5,"w":0.0863,"conf":0.5461,"acc":0.4935},{"lo":0.6,"w":0.0733,"conf":0.6499,"acc":0.6231},{"lo":0.7,"w":0.0717,"conf":0.75,"acc":0.7673},{"lo":0.8,"w":0.0949,"conf":0.8538,"acc":0.8096},{"lo":0.9,"w":0.5786,"conf":0.9823,"acc":0.9702}],"areas":{"knowledge":{"ece":0.0626,"acc":0.7181,"conf":0.7735},"language":{"ece":0.016,"acc":0.8945,"conf":0.8794},"retrieval":{"ece":0.0305,"acc":0.8079,"conf":0.8375},"tools":{"ece":0.0325,"acc":0.9641,"conf":0.9758},"arts":{"ece":0.0356,"acc":0.7239,"conf":0.7595}}},"gaps":null,"latency":{"median":111.6,"p95":893.8,"mean":296.7},"results":{"1":{"score":0.9817,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":271.7},"2":{"score":0.721,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2634.3},"3":{"score":0.8957,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1398.2},"4":{"score":0.909,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":174.6},"5":{"score":0.9606,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":278.8},"6":{"score":0.8002,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":422.3},"9":{"score":0.7955,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":760},"10":{"score":0.6716,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":252.6},"11":{"score":0.8483,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":784.6},"12":{"score":0.7768,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":43.5},"20":{"score":0.9422,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.7},"21":{"score":0.648,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":35.4},"22":{"score":0.823,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1225.6},"23":{"score":0.675,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.3},"24":{"score":0.8463,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":40.6},"25":{"score":0.5255,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":57.5},"26":{"score":0.987,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":38.3},"27":{"score":0.9659,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":38.8},"28":{"score":0.9234,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":35.8},"29":{"score":0.9683,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":59.1},"30":{"score":0.746,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2216,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":198.5},"32":{"score":0.9202,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":169.7},"33":{"score":0.4206,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":374.2},"36":{"score":0.4406,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3971.2},"37":{"score":0.5873,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.4},"38":{"score":0.36,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1127.3},"39":{"score":0.9487,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":86.3},"40":{"score":0.6369,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":39.2},"41":{"score":0.8187,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":44.7},"42":{"score":0.8736,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":93.1},"43":{"score":0.7789,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":44.7},"44":{"score":0.9408,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.8},"45":{"score":0.1158,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":60.9},"50":{"score":0.6348,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":137.2},"56":{"score":0.961,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":531.1},"57":{"score":0.6641,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":59.5},"58":{"score":0.9001,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":45.2},"59":{"score":0.8493,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":127.5},"61":{"score":0.8878,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":91.3},"62":{"score":0.9239,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":173},"64":{"score":0.7955,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":61.4}},"benchmarks":{"1":{"raw":0.9817,"skill":0.9753,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9602,"private_eq":0.9701},"2":{"raw":0.721,"skill":0.6778,"coverage":1,"random":0.1341,"tracks":[],"combined":0.71,"private_eq":0.7509},"3":{"raw":0.8957,"skill":0.8937,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.909,"skill":0.9078,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9003,"private_eq":0.9017},"5":{"raw":0.9606,"skill":0.9604,"coverage":1,"random":0.006,"tracks":[],"combined":0.8657,"private_eq":0.867},"6":{"raw":0.8002,"skill":0.5315,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7955,"skill":0.7955,"coverage":1,"random":0,"tracks":[],"combined":0.7734,"private_eq":0.775},"10":{"raw":0.6716,"skill":0.4536,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8483,"skill":0.7806,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7768,"skill":0.6657,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5556,"private_eq":0.706},"20":{"raw":0.9434,"skill":0.8868,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.648,"skill":0.296,"coverage":1,"random":0.5,"tracks":[],"combined":0.3773,"private_eq":0.6917},"22":{"raw":0.8191,"skill":0.8177,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6168,"private_eq":0.6231},"23":{"raw":0.667,"skill":0.3339,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8463,"skill":0.7951,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5255,"skill":0.3673,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4742,"combined":0.2919},"26":{"raw":0.987,"skill":0.9827,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9659,"skill":0.9545,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9234,"skill":0.8468,"coverage":1,"random":0.5,"tracks":[],"combined":0.6984,"private_eq":0.8504},"29":{"raw":0.9683,"skill":0.9577,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.746,"skill":0.696,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8036,"random":0.25},{"track":"10-choice","raw":0.6884,"random":0.1}],"combined":0.7397,"private_eq":0.7869},"31":{"raw":0.2216,"skill":0.1522,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0658,"private_eq":0.1512},"32":{"raw":0.9202,"skill":0.8731,"coverage":1,"random":0.371,"tracks":[],"combined":0.3201,"private_eq":0.5766},"33":{"raw":0.4206,"skill":0.4129,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3331,"private_eq":0.3483},"36":{"raw":0.4406,"skill":0.3672,"coverage":1,"random":0.116,"tracks":[],"combined":0.378,"private_eq":0.4555},"37":{"raw":0.5873,"skill":0.4824,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6108,"private_eq":0.6924},"38":{"raw":0.36,"skill":0.3395,"coverage":1,"random":0.031,"tracks":[],"combined":0.6,"private_eq":0.6158},"39":{"raw":0.9487,"skill":0.9245,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4712,"private_eq":0.6439},"40":{"raw":0.6369,"skill":0.5329,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5233,"headline":false},{"track":"A · English","score":0.6369,"headline":true},{"track":"C · Arabic pairs","score":0.97,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.3968,"private_eq":0.5357},"41":{"raw":0.8187,"skill":0.7281,"coverage":1,"random":0.3333,"tracks":[],"combined":0.7118,"private_eq":0.8094},"42":{"raw":0.8736,"skill":0.7541,"coverage":1,"random":0.486,"tracks":[],"combined":0.8426,"private_eq":0.9195},"43":{"raw":0.7789,"skill":0.6493,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5746,"private_eq":0.7343},"44":{"raw":0.9408,"skill":0.8816,"coverage":1,"random":0.5,"tracks":[],"combined":0.7763,"private_eq":0.8889},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6348,"skill":0.47,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.961,"skill":0.922,"coverage":1,"random":0.5,"tracks":[],"combined":0.5683,"private_eq":0.7861},"57":{"raw":0.6641,"skill":0.6222,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5557,"combined":0.4955},"58":{"raw":0.9001,"skill":0.8552,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4907,"combined":0.2543},"59":{"raw":0.8493,"skill":0.6875,"coverage":1,"random":0.5177,"tracks":[],"combined":0.284,"private_eq":0.6581},"61":{"raw":0.8878,"skill":0.7756,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"62":{"raw":0.9239,"skill":0.8985,"coverage":1,"random":0.25,"tracks":[],"combined":0.6656,"private_eq":0.7513},"64":{"raw":0.7955,"skill":0.7444,"coverage":1,"random":0.2,"tracks":[],"combined":0.7198,"private_eq":0.7776}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #217).","Replaces the earlier board entry onejev1.5-27b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 69.55, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.74% on 827 rows (bar 98.5%)."],"scores":{"balanced_raw":76.4,"balanced_skill":61.46,"breadth_skill":68.94,"public_skill":69.55},"frozen_scores":{"balanced_raw":76.4,"balanced_skill":69.55,"breadth_skill":68.94},"categories":[{"id":"knowledge","raw":0.6486,"skill":0.5746,"combined":0.4535,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7904,"skill":0.7123,"combined":0.616,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7891,"skill":0.7367,"combined":0.6651,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8691,"skill":0.8546,"combined":0.8163,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7513,"skill":0.5915,"combined":0.5735,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"onejev1.6-27b","pub":69.55,"same_skills":62.48,"new_domains":46.79,"v":61.46,"lab":false,"jev":false,"rank_v":43,"rank_pub":13,"name":"OneJev1.6-27B","eq":{"S":62.15,"O":60.43},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":37.29,"new_domains":24.17}},"new_in_edition":true},{"engine":"onejev1.6-9b","name":"OneJev1.6-9B","long_name":"OneJev1.6-9B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":9409813744,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/OmniJev/OneJev1.6-9B","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pr":217,"replaces":"onejev1.5-9b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7892,"conf":0.8098,"ece":0.0224,"brier":0.2836,"over95":0.01,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0028,"conf":0.0834,"acc":0.0975},{"lo":0.1,"w":0.0166,"conf":0.1484,"acc":0.1132},{"lo":0.2,"w":0.0148,"conf":0.2547,"acc":0.143},{"lo":0.3,"w":0.0249,"conf":0.3549,"acc":0.2302},{"lo":0.4,"w":0.0408,"conf":0.4564,"acc":0.477},{"lo":0.5,"w":0.0989,"conf":0.5485,"acc":0.5091},{"lo":0.6,"w":0.0897,"conf":0.6502,"acc":0.6428},{"lo":0.7,"w":0.0914,"conf":0.75,"acc":0.7272},{"lo":0.8,"w":0.1001,"conf":0.8531,"acc":0.8202},{"lo":0.9,"w":0.52,"conf":0.9795,"acc":0.9676}],"areas":{"knowledge":{"ece":0.0694,"acc":0.6861,"conf":0.7507},"language":{"ece":0.0132,"acc":0.8741,"conf":0.8632},"retrieval":{"ece":0.0223,"acc":0.8098,"conf":0.8312},"tools":{"ece":0.0259,"acc":0.9649,"conf":0.9725},"arts":{"ece":0.0402,"acc":0.7206,"conf":0.7169}}},"gaps":null,"latency":{"median":42.2,"p95":305.4,"mean":120},"results":{"1":{"score":0.9823,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":101.8},"2":{"score":0.7096,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":820},"3":{"score":0.8976,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":455.2},"4":{"score":0.902,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":53.2},"5":{"score":0.9606,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":85.3},"6":{"score":0.8004,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":160.1},"9":{"score":0.7045,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":301.4},"10":{"score":0.6936,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":87},"11":{"score":0.8433,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":367.8},"12":{"score":0.7275,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":16.6},"20":{"score":0.9098,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.8},"21":{"score":0.6465,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.7},"22":{"score":0.8075,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":392.3},"23":{"score":0.6462,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41},"24":{"score":0.7993,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.6},"25":{"score":0.4796,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":19.1},"26":{"score":0.9815,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":14.7},"27":{"score":0.9608,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":14.8},"28":{"score":0.8879,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":13.5},"29":{"score":0.9567,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":20.1},"30":{"score":0.6221,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2252,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":62.4},"32":{"score":0.9069,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":53.4},"33":{"score":0.4067,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":152.4},"36":{"score":0.4164,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1225.6},"37":{"score":0.5361,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":25.3},"38":{"score":0.3299,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":373.8},"39":{"score":0.9494,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":32.3},"40":{"score":0.5152,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":13.8},"41":{"score":0.8098,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":16.8},"42":{"score":0.8182,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32.3},"43":{"score":0.5667,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":16.9},"44":{"score":0.919,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.4},"45":{"score":0.1058,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":19.7},"50":{"score":0.6289,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":45.7},"56":{"score":0.952,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":168.1},"57":{"score":0.5936,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":19.5},"58":{"score":0.8642,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":17},"59":{"score":0.8415,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":44.6},"61":{"score":0.8858,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":31.4},"62":{"score":0.9001,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":55.1},"64":{"score":0.7424,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":19.8}},"benchmarks":{"1":{"raw":0.9823,"skill":0.9761,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.7096,"skill":0.6646,"coverage":1,"random":0.1341,"tracks":[],"combined":0.695,"private_eq":0.738},"3":{"raw":0.8976,"skill":0.8956,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8699,"private_eq":0.8728},"4":{"raw":0.902,"skill":0.9007,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8501,"private_eq":0.8527},"5":{"raw":0.9606,"skill":0.9604,"coverage":1,"random":0.006,"tracks":[],"combined":0.891,"private_eq":0.8919},"6":{"raw":0.8004,"skill":0.532,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7045,"skill":0.7045,"coverage":1,"random":0,"tracks":[],"combined":0.6697,"private_eq":0.6725},"10":{"raw":0.6936,"skill":0.4902,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8433,"skill":0.7735,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7275,"skill":0.5918,"coverage":1,"random":0.3324,"tracks":[],"combined":0.489,"private_eq":0.6621},"20":{"raw":0.9121,"skill":0.8241,"coverage":1,"random":0.5,"tracks":[],"combined":0.9432,"private_eq":0.9714},"21":{"raw":0.6465,"skill":0.293,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.8045,"skill":0.803,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5838,"private_eq":0.5908},"23":{"raw":0.6488,"skill":0.2975,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7993,"skill":0.7324,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.415,"combined":0.212},"26":{"raw":0.9815,"skill":0.9753,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9608,"skill":0.9477,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8879,"skill":0.7758,"coverage":1,"random":0.5,"tracks":[],"combined":0.5066,"private_eq":0.7556},"29":{"raw":0.9567,"skill":0.9423,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6221,"skill":0.546,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6929,"random":0.25},{"track":"10-choice","raw":0.5512,"random":0.1}],"combined":0.6094,"private_eq":0.6806},"31":{"raw":0.2252,"skill":0.1561,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0474,"private_eq":0.1345},"32":{"raw":0.9069,"skill":0.852,"coverage":1,"random":0.371,"tracks":[],"combined":0.2815,"private_eq":0.5526},"33":{"raw":0.4067,"skill":0.3988,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3742,"private_eq":0.3884},"36":{"raw":0.4164,"skill":0.3398,"coverage":1,"random":0.116,"tracks":[],"combined":0.3244,"private_eq":0.4087},"37":{"raw":0.5361,"skill":0.4181,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5297,"private_eq":0.6285},"38":{"raw":0.3299,"skill":0.3085,"coverage":1,"random":0.031,"tracks":[],"combined":0.5122,"private_eq":0.5317},"39":{"raw":0.9494,"skill":0.9256,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4572,"private_eq":0.6345},"40":{"raw":0.5152,"skill":0.3763,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4986,"headline":false},{"track":"A · English","score":0.5152,"headline":true},{"track":"C · Arabic pairs","score":0.945,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.3221,"private_eq":0.4783},"41":{"raw":0.8098,"skill":0.7146,"coverage":1,"random":0.3333,"tracks":[],"combined":0.8032,"private_eq":0.8696},"42":{"raw":0.8182,"skill":0.6463,"coverage":1,"random":0.486,"tracks":[],"combined":0.6104,"private_eq":0.8015},"43":{"raw":0.5667,"skill":0.3125,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3488,"private_eq":0.5936},"44":{"raw":0.919,"skill":0.838,"coverage":1,"random":0.5,"tracks":[],"combined":0.7548,"private_eq":0.8783},"45":{"raw":0.1058,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6289,"skill":0.4614,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.952,"skill":0.904,"coverage":1,"random":0.5,"tracks":[],"combined":0.3087,"private_eq":0.6578},"57":{"raw":0.5936,"skill":0.5429,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5117,"combined":0.4455},"58":{"raw":0.8642,"skill":0.8032,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4615,"combined":0.2115},"59":{"raw":0.8415,"skill":0.6714,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2653,"private_eq":0.6492},"61":{"raw":0.8858,"skill":0.7716,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.9001,"skill":0.8668,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.7424,"skill":0.678,"coverage":1,"random":0.2,"tracks":[],"combined":0.6907,"private_eq":0.7546}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #217).","Replaces the earlier board entry onejev1.5-9b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 65.30, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.84% on 827 rows (bar 98.5%)."],"scores":{"balanced_raw":73.31,"balanced_skill":53.05,"breadth_skill":64.41,"public_skill":65.3},"frozen_scores":{"balanced_raw":73.31,"balanced_skill":65.3,"breadth_skill":64.41},"categories":[{"id":"knowledge","raw":0.597,"skill":0.5014,"combined":0.3766,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7583,"skill":0.6657,"combined":0.5665,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.774,"skill":0.7174,"combined":0.6001,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8463,"skill":0.83,"combined":0.7692,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7305,"skill":0.5595,"combined":0.5502,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"onejev1.6-9b","pub":65.3,"same_skills":56.86,"new_domains":36.66,"v":53.05,"lab":false,"jev":false,"rank_v":85,"rank_pub":37,"name":"OneJev1.6-9B","eq":{"S":56.46,"O":47.93},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":33.88,"new_domains":19.17}},"new_in_edition":true},{"engine":"onejev1.6-4b","name":"OneJev1.6-4B","long_name":"OneJev1.6-4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4539265536,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/OmniJev/OneJev1.6-4B","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pr":217,"replaces":"onejev1.5-4b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7683,"conf":0.7931,"ece":0.025,"brier":0.3049,"over95":0.008,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0022,"conf":0.0871,"acc":0.1233},{"lo":0.1,"w":0.019,"conf":0.1545,"acc":0.1176},{"lo":0.2,"w":0.014,"conf":0.2504,"acc":0.1587},{"lo":0.3,"w":0.0313,"conf":0.3516,"acc":0.3142},{"lo":0.4,"w":0.0541,"conf":0.4529,"acc":0.4183},{"lo":0.5,"w":0.1058,"conf":0.5499,"acc":0.5167},{"lo":0.6,"w":0.0893,"conf":0.6492,"acc":0.6184},{"lo":0.7,"w":0.0907,"conf":0.7515,"acc":0.6992},{"lo":0.8,"w":0.1064,"conf":0.8529,"acc":0.8245},{"lo":0.9,"w":0.4872,"conf":0.977,"acc":0.965}],"areas":{"knowledge":{"ece":0.0568,"acc":0.664,"conf":0.7202},"language":{"ece":0.0098,"acc":0.8528,"conf":0.8487},"retrieval":{"ece":0.0268,"acc":0.7988,"conf":0.8256},"tools":{"ece":0.0233,"acc":0.9628,"conf":0.9651},"arts":{"ece":0.0401,"acc":0.6837,"conf":0.7097}}},"gaps":null,"latency":{"median":27.2,"p95":201.1,"mean":90.4},"results":{"1":{"score":0.9793,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":79.9},"2":{"score":0.7102,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":612.6},"3":{"score":0.8898,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":350.5},"4":{"score":0.9003,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":41.9},"5":{"score":0.9578,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":64.2},"6":{"score":0.8004,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":137.8},"9":{"score":0.6591,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":259.6},"10":{"score":0.6706,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":65.5},"11":{"score":0.8453,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":333.6},"12":{"score":0.6808,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":13},"20":{"score":0.8242,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.3},"21":{"score":0.6412,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":10.9},"22":{"score":0.794,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":293},"23":{"score":0.609,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.4},"24":{"score":0.7441,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":11.8},"25":{"score":0.4439,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":15.3},"26":{"score":0.9781,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":11.3},"27":{"score":0.9369,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":11.5},"28":{"score":0.8319,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":10.6},"29":{"score":0.9329,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":15.9},"30":{"score":0.5629,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2158,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47.1},"32":{"score":0.8883,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":42.1},"33":{"score":0.3885,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":141.5},"36":{"score":0.3917,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":932.5},"37":{"score":0.5187,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.8},"38":{"score":0.3624,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":315.5},"39":{"score":0.9326,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":24.6},"40":{"score":0.4826,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":10.8},"41":{"score":0.8068,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":13},"42":{"score":0.8024,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":25.2},"43":{"score":0.4947,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":13.2},"44":{"score":0.9084,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.6},"45":{"score":0.1118,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":15.9},"50":{"score":0.6038,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":35.4},"56":{"score":0.906,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":133.1},"57":{"score":0.5205,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":15.6},"58":{"score":0.8284,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.3},"59":{"score":0.8338,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":34.2},"61":{"score":0.869,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":24.5},"62":{"score":0.8836,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":43.8},"64":{"score":0.7083,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":16}},"benchmarks":{"1":{"raw":0.9793,"skill":0.9721,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.7102,"skill":0.6653,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6802,"private_eq":0.7254},"3":{"raw":0.8898,"skill":0.8877,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8699,"private_eq":0.8728},"4":{"raw":0.9003,"skill":0.899,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8509,"private_eq":0.8535},"5":{"raw":0.9578,"skill":0.9575,"coverage":1,"random":0.006,"tracks":[],"combined":0.879,"private_eq":0.8801},"6":{"raw":0.8004,"skill":0.532,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6591,"skill":0.6591,"coverage":1,"random":0,"tracks":[],"combined":0.6065,"private_eq":0.61},"10":{"raw":0.6706,"skill":0.4519,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8453,"skill":0.7763,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6808,"skill":0.5219,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5053,"private_eq":0.6728},"20":{"raw":0.8221,"skill":0.6442,"coverage":1,"random":0.5,"tracks":[],"combined":0.7914,"private_eq":0.8964},"21":{"raw":0.6412,"skill":0.2823,"coverage":1,"random":0.5,"tracks":[],"combined":0.2565,"private_eq":0.632},"22":{"raw":0.794,"skill":0.7924,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5612,"private_eq":0.5686},"23":{"raw":0.6028,"skill":0.2055,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7441,"skill":0.6588,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3735,"combined":0.1561},"26":{"raw":0.9781,"skill":0.9708,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9369,"skill":0.9158,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8319,"skill":0.6638,"coverage":1,"random":0.5,"tracks":[],"combined":0.4109,"private_eq":0.7083},"29":{"raw":0.9329,"skill":0.9105,"coverage":1,"random":0.25,"tracks":[],"combined":0.9088,"private_eq":0.9316},"30":{"raw":0.5629,"skill":0.475,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6459,"random":0.25},{"track":"10-choice","raw":0.4799,"random":0.1}],"combined":0.5881,"private_eq":0.6632},"31":{"raw":0.2158,"skill":0.1459,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0366,"private_eq":0.1247},"32":{"raw":0.8883,"skill":0.8224,"coverage":1,"random":0.371,"tracks":[],"combined":0.2196,"private_eq":0.5141},"33":{"raw":0.3885,"skill":0.3804,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3588,"private_eq":0.3734},"36":{"raw":0.3917,"skill":0.3119,"coverage":1,"random":0.116,"tracks":[],"combined":0.34,"private_eq":0.4223},"37":{"raw":0.5187,"skill":0.3964,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5627,"private_eq":0.6545},"38":{"raw":0.3624,"skill":0.342,"coverage":1,"random":0.031,"tracks":[],"combined":0.5958,"private_eq":0.6118},"39":{"raw":0.9326,"skill":0.9009,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4678,"private_eq":0.6416},"40":{"raw":0.4826,"skill":0.3343,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2985,"headline":false},{"track":"A · English","score":0.4826,"headline":true},{"track":"C · Arabic pairs","score":0.905,"headline":false},{"track":"C · English pairs","score":0.92,"headline":false}],"combined":0.234,"private_eq":0.4106},"41":{"raw":0.8068,"skill":0.7102,"coverage":1,"random":0.3333,"tracks":[],"combined":0.7666,"private_eq":0.8455},"42":{"raw":0.8024,"skill":0.6156,"coverage":1,"random":0.486,"tracks":[],"combined":0.7361,"private_eq":0.8654},"43":{"raw":0.4947,"skill":0.1984,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3405,"private_eq":0.5884},"44":{"raw":0.9084,"skill":0.8168,"coverage":1,"random":0.5,"tracks":[],"combined":0.7065,"private_eq":0.8544},"45":{"raw":0.1118,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6038,"skill":0.425,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.906,"skill":0.812,"coverage":1,"random":0.5,"tracks":[],"combined":0.5511,"private_eq":0.7776},"57":{"raw":0.5205,"skill":0.4607,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4337,"combined":0.3567},"58":{"raw":0.8284,"skill":0.7513,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4332,"combined":0.17},"59":{"raw":0.8338,"skill":0.6554,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2407,"private_eq":0.6375},"61":{"raw":0.869,"skill":0.738,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.8836,"skill":0.8448,"coverage":1,"random":0.25,"tracks":[],"combined":0.6108,"private_eq":0.7107},"64":{"raw":0.7083,"skill":0.6354,"coverage":1,"random":0.2,"tracks":[],"combined":0.681,"private_eq":0.7469}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #217).","Replaces the earlier board entry onejev1.5-4b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 61.95, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.84% on 827 rows (bar 98.5%)."],"scores":{"balanced_raw":71.1,"balanced_skill":48.88,"breadth_skill":60.76,"public_skill":61.95},"frozen_scores":{"balanced_raw":71.1,"balanced_skill":61.95,"breadth_skill":60.76},"categories":[{"id":"knowledge","raw":0.5619,"skill":0.45,"combined":0.3298,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7449,"skill":0.644,"combined":0.564,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7566,"skill":0.6892,"combined":0.64,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8326,"skill":0.815,"combined":0.755,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6954,"skill":0.4975,"combined":0.5015,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"onejev1.6-4b","pub":61.95,"same_skills":55.78,"new_domains":29.56,"v":48.88,"lab":false,"jev":false,"rank_v":96,"rank_pub":49,"name":"OneJev1.6-4B","eq":{"S":55.37,"O":39.15},"rank_tie":22,"tied":true,"parts":{"public":0,"same_skills":33.22,"new_domains":15.66}},"new_in_edition":true},{"engine":"onejev1.6-0.8b","name":"OneJev1.6-0.8B","long_name":"OneJev1.6-0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":852985920,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/OmniJev/OneJev1.6-0.8B","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pr":217,"replaces":"onejev1.5-0.8b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7039,"conf":0.7379,"ece":0.0351,"brier":0.3828,"over95":0.0082,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0025,"conf":0.0828,"acc":0.0947},{"lo":0.1,"w":0.0197,"conf":0.1573,"acc":0.1064},{"lo":0.2,"w":0.0252,"conf":0.2571,"acc":0.2756},{"lo":0.3,"w":0.0516,"conf":0.3518,"acc":0.2611},{"lo":0.4,"w":0.0558,"conf":0.4512,"acc":0.4365},{"lo":0.5,"w":0.145,"conf":0.5488,"acc":0.5124},{"lo":0.6,"w":0.1168,"conf":0.6485,"acc":0.6157},{"lo":0.7,"w":0.1157,"conf":0.7498,"acc":0.7055},{"lo":0.8,"w":0.1058,"conf":0.8516,"acc":0.7971},{"lo":0.9,"w":0.3619,"conf":0.9744,"acc":0.9521}],"areas":{"knowledge":{"ece":0.0464,"acc":0.5737,"conf":0.6166},"language":{"ece":0.0203,"acc":0.7841,"conf":0.7943},"retrieval":{"ece":0.0338,"acc":0.7916,"conf":0.8224},"tools":{"ece":0.0256,"acc":0.9336,"conf":0.9492},"arts":{"ece":0.074,"acc":0.6119,"conf":0.6847}}},"gaps":null,"latency":{"median":13.1,"p95":103.9,"mean":32.6},"results":{"1":{"score":0.9634,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":39.6},"2":{"score":0.6846,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":264.3},"3":{"score":0.8681,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":161.3},"4":{"score":0.8728,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":18.6},"5":{"score":0.9242,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":27.3},"6":{"score":0.8006,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":100.7},"9":{"score":0.2955,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":184.8},"10":{"score":0.7077,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":31.3},"11":{"score":0.8273,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":283.1},"12":{"score":0.5268,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":7.8},"20":{"score":0.503,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.9},"21":{"score":0.613,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":6.6},"22":{"score":0.785,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":123.4},"23":{"score":0.628,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.6},"24":{"score":0.5145,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":7.2},"25":{"score":0.2857,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":8.3},"26":{"score":0.8855,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6.9},"27":{"score":0.7457,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":7},"28":{"score":0.6472,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":6.6},"29":{"score":0.7674,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":8.9},"30":{"score":0.3484,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.198,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.7},"32":{"score":0.7181,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":19.4},"33":{"score":0.3273,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":78.6},"36":{"score":0.3595,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":401.4},"37":{"score":0.5042,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.3},"38":{"score":0.2428,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":170.4},"39":{"score":0.8983,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":14.7},"40":{"score":0.1484,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":6.7},"41":{"score":0.7455,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":8},"42":{"score":0.6812,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.1},"43":{"score":0.3684,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":8},"44":{"score":0.8214,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.6},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":8.8},"50":{"score":0.5949,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":18.3},"56":{"score":0.911,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":69.6},"57":{"score":0.2917,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":8.7},"58":{"score":0.6677,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":8.2},"59":{"score":0.7736,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":18.3},"61":{"score":0.7495,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":13.7},"62":{"score":0.8677,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":21.1},"64":{"score":0.5928,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":9}},"benchmarks":{"1":{"raw":0.9634,"skill":0.9506,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8784,"private_eq":0.9102},"2":{"raw":0.6846,"skill":0.6358,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6721,"private_eq":0.7184},"3":{"raw":0.8681,"skill":0.8656,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7753,"private_eq":0.7811},"4":{"raw":0.8728,"skill":0.8712,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7696,"private_eq":0.7741},"5":{"raw":0.9242,"skill":0.9237,"coverage":1,"random":0.006,"tracks":[],"combined":0.796,"private_eq":0.7985},"6":{"raw":0.8006,"skill":0.5325,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2955,"skill":0.2955,"coverage":1,"random":0,"tracks":[],"combined":0.1842,"private_eq":0.1925},"10":{"raw":0.7077,"skill":0.5136,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8273,"skill":0.7503,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5268,"skill":0.2912,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3606,"private_eq":0.5773},"20":{"raw":0.5015,"skill":0.003,"coverage":1,"random":0.5,"tracks":[],"combined":0.0973,"private_eq":0.5533},"21":{"raw":0.613,"skill":0.226,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.7836,"skill":0.7819,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6122,"private_eq":0.6186},"23":{"raw":0.6122,"skill":0.2244,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5145,"skill":0.3527,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2857,"skill":0.0476,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2862,"combined":0.0383},"26":{"raw":0.8855,"skill":0.8473,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7457,"skill":0.6608,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6472,"skill":0.2944,"coverage":1,"random":0.5,"tracks":[],"combined":0.0981,"private_eq":0.5537},"29":{"raw":0.7674,"skill":0.6899,"coverage":1,"random":0.25,"tracks":[],"combined":0.498,"private_eq":0.627},"30":{"raw":0.3484,"skill":0.2148,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4496,"random":0.25},{"track":"10-choice","raw":0.2472,"random":0.1}],"combined":0.3193,"private_eq":0.444},"31":{"raw":0.198,"skill":0.1265,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0431,"private_eq":0.1306},"32":{"raw":0.7181,"skill":0.5518,"coverage":1,"random":0.371,"tracks":[],"combined":0.0493,"private_eq":0.4082},"33":{"raw":0.3273,"skill":0.3184,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2766,"private_eq":0.2932},"36":{"raw":0.3595,"skill":0.2754,"coverage":1,"random":0.116,"tracks":[],"combined":0.2165,"private_eq":0.3144},"37":{"raw":0.5042,"skill":0.3781,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4406,"private_eq":0.5583},"38":{"raw":0.2428,"skill":0.2186,"coverage":1,"random":0.031,"tracks":[],"combined":0.3987,"private_eq":0.423},"39":{"raw":0.8983,"skill":0.8504,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3894,"private_eq":0.5889},"40":{"raw":0.1484,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1749,"headline":false},{"track":"A · English","score":0.1484,"headline":true},{"track":"C · Arabic pairs","score":0.805,"headline":false},{"track":"C · English pairs","score":0.84,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7455,"skill":0.6183,"coverage":1,"random":0.3333,"tracks":[],"combined":0.6052,"private_eq":0.7391},"42":{"raw":0.6812,"skill":0.3798,"coverage":1,"random":0.486,"tracks":[],"combined":0.5043,"private_eq":0.7476},"43":{"raw":0.3684,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1649,"private_eq":0.479},"44":{"raw":0.8214,"skill":0.6428,"coverage":1,"random":0.5,"tracks":[],"combined":0.4977,"private_eq":0.7512},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5949,"skill":0.412,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.911,"skill":0.822,"coverage":1,"random":0.5,"tracks":[],"combined":0.2482,"private_eq":0.6279},"57":{"raw":0.2917,"skill":0.2034,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3672,"combined":0.2811},"58":{"raw":0.6677,"skill":0.5183,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3437,"combined":0.0388},"59":{"raw":0.7736,"skill":0.5306,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1424,"private_eq":0.5906},"61":{"raw":0.7495,"skill":0.499,"coverage":1,"random":0.5,"tracks":[],"combined":0.5924,"private_eq":0.798},"62":{"raw":0.8677,"skill":0.8236,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"64":{"raw":0.5928,"skill":0.491,"coverage":1,"random":0.2,"tracks":[],"combined":0.5452,"private_eq":0.6395}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #217).","Replaces the earlier board entry onejev1.5-0.8b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 48.76, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.64% on 827 rows (bar 98.5%)."],"scores":{"balanced_raw":61.63,"balanced_skill":29.58,"breadth_skill":46.13,"public_skill":48.76},"frozen_scores":{"balanced_raw":61.63,"balanced_skill":48.76,"breadth_skill":46.13},"categories":[{"id":"knowledge","raw":0.4338,"skill":0.2609,"combined":0.1758,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6245,"skill":0.4814,"combined":0.3643,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7201,"skill":0.6339,"combined":0.5181,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7492,"skill":0.7286,"combined":0.6265,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6163,"skill":0.3564,"combined":0.3334,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"onejev1.6-0.8b","pub":48.76,"same_skills":40.49,"new_domains":9.28,"v":29.58,"lab":false,"jev":false,"rank_v":169,"rank_pub":124,"name":"OneJev1.6-0.8B","eq":{"S":39.9,"O":14.11},"rank_tie":38,"tied":true,"parts":{"public":0,"same_skills":23.94,"new_domains":5.64}},"new_in_edition":true},{"engine":"garum","name":"garum (Gemma 4 31B)","long_name":"garum (Gemma 4 31B)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/plantwaterer/garum","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":184,"new":true,"show_notes":true},"completed":110201,"counts":{"ok":109958,"unsupported":243,"error":0},"lifted":false,"calibration":{"acc":0.7312,"conf":0.781,"ece":0.0509,"brier":0.367,"over95":0.017,"n":34404,"benchmarks":31,"rel":[{"lo":0,"w":0.004,"conf":0.0885,"acc":0.0695},{"lo":0.1,"w":0.037,"conf":0.1511,"acc":0.1663},{"lo":0.2,"w":0.0244,"conf":0.2485,"acc":0.2255},{"lo":0.3,"w":0.0257,"conf":0.3559,"acc":0.2534},{"lo":0.4,"w":0.0457,"conf":0.4526,"acc":0.442},{"lo":0.5,"w":0.0873,"conf":0.5462,"acc":0.5173},{"lo":0.6,"w":0.0791,"conf":0.6542,"acc":0.5893},{"lo":0.7,"w":0.0982,"conf":0.7488,"acc":0.7172},{"lo":0.8,"w":0.1385,"conf":0.8546,"acc":0.7512},{"lo":0.9,"w":0.4601,"conf":0.9733,"acc":0.9267}],"areas":{"knowledge":{"ece":0.0808,"acc":0.6691,"conf":0.7499},"language":{"ece":0.0374,"acc":0.8431,"conf":0.8069},"retrieval":{"ece":0.2283,"acc":0.6543,"conf":0.8817},"tools":{"ece":0.029,"acc":0.9415,"conf":0.9646},"arts":{"ece":0.0536,"acc":0.5897,"conf":0.6058}}},"gaps":null,"latency":{"median":119,"p95":1705.3,"mean":450.4},"results":{"1":{"score":0.9734,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":429},"2":{"score":0.6571,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":4166},"3":{"score":0.8445,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":9172.5},"4":{"score":0.8826,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":429.5},"5":{"score":0.9354,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":477.9},"6":{"score":0.7998,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1224.9},"9":{"score":0.6705,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":5133.1},"10":{"score":0.0464,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":315.7},"11":{"score":0.7297,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":8956.4},"12":{"score":0.7323,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":91},"20":{"score":0.921,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":91.4},"21":{"score":0.6572,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":80.1},"22":{"score":0.255,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3925.1},"23":{"score":0.6548,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":163.2},"24":{"score":0.8502,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":91},"25":{"score":0.4592,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":114.3},"26":{"score":0.9916,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":81.8},"27":{"score":0.9718,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":82.6},"28":{"score":0.8824,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":79.5},"29":{"score":0.8951,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":115.6},"30":{"score":0.6293,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2122,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":201.3},"32":{"score":0.6423,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":234.6},"33":{"score":0.3097,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":910.1},"36":{"score":0.4875,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5135.9},"37":{"score":0.5457,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":144.2},"38":{"score":0.2491,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2282.7},"39":{"score":0.9297,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":179.1},"40":{"score":0.6832,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4595,"unsupported":5,"errors":0,"median_ms":140.5},"41":{"score":0.7027,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":97},"42":{"score":0.8056,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":146.5},"43":{"score":0.8018,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":93.8},"44":{"score":0.7648,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":92.4},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":141.5},"50":{"score":0.4863,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":221.4},"56":{"score":0.8275,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1010.6},"57":{"score":0.6533,"jev":null,"metric":"accuracy","requests":12032,"answered":11728,"unsupported":304,"errors":0,"median_ms":116.7},"58":{"score":0.7527,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":113.3},"59":{"score":0.7867,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":201.2},"61":{"score":0.704,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":145.1},"62":{"score":0.7081,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":198.8},"64":{"score":0.7614,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":95.9}},"benchmarks":{"1":{"raw":0.9734,"skill":0.9641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.6571,"skill":0.604,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6916,"private_eq":0.7351},"3":{"raw":0.8445,"skill":0.8415,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.8826,"skill":0.8811,"coverage":1,"random":0.0127,"tracks":[],"combined":0.876,"private_eq":0.878},"5":{"raw":0.9354,"skill":0.935,"coverage":1,"random":0.006,"tracks":[],"combined":0.7646,"private_eq":0.7677},"6":{"raw":0.7998,"skill":0.5306,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6705,"skill":0.6705,"coverage":1,"random":0,"tracks":[],"combined":0.6546,"private_eq":0.6575},"10":{"raw":0.0464,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7297,"skill":0.6091,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7323,"skill":0.599,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4227,"private_eq":0.6183},"20":{"raw":0.9203,"skill":0.8407,"coverage":1,"random":0.5,"tracks":[],"combined":0.8782,"private_eq":0.9393},"21":{"raw":0.6572,"skill":0.3143,"coverage":1,"random":0.5,"tracks":[],"combined":0.3472,"private_eq":0.6768},"22":{"raw":0.2449,"skill":0.239,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2644,"private_eq":0.2775},"23":{"raw":0.6497,"skill":0.2994,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8502,"skill":0.8003,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4592,"skill":0.2789,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4711,"combined":0.2877},"26":{"raw":0.9916,"skill":0.9888,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9718,"skill":0.9624,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8824,"skill":0.7648,"coverage":1,"random":0.5,"tracks":[],"combined":0.648,"private_eq":0.8255},"29":{"raw":0.8951,"skill":0.8601,"coverage":1,"random":0.25,"tracks":[],"combined":0.9188,"private_eq":0.939},"30":{"raw":0.6293,"skill":0.556,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7111,"random":0.25},{"track":"10-choice","raw":0.5474,"random":0.1}],"combined":0.6695,"private_eq":0.7296},"31":{"raw":0.2122,"skill":0.142,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1026,"private_eq":0.1846},"32":{"raw":0.6423,"skill":0.4313,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.3097,"skill":0.3005,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2921,"private_eq":0.3083},"36":{"raw":0.4875,"skill":0.4203,"coverage":1,"random":0.116,"tracks":[],"combined":0.3998,"private_eq":0.4746},"37":{"raw":0.5457,"skill":0.4302,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5341,"private_eq":0.632},"38":{"raw":0.2491,"skill":0.2251,"coverage":1,"random":0.031,"tracks":[],"combined":0.4477,"private_eq":0.4699},"39":{"raw":0.9297,"skill":0.8966,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4988,"private_eq":0.6624},"40":{"raw":0.6832,"skill":0.5925,"coverage":0.9986,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1455,"headline":false},{"track":"A · English","score":0.6832,"headline":true},{"track":"C · Arabic pairs","score":0.85,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.5711,"private_eq":0.6696},"41":{"raw":0.7027,"skill":0.5541,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5873,"private_eq":0.7273},"42":{"raw":0.8056,"skill":0.6218,"coverage":1,"random":0.486,"tracks":[],"combined":0.9849,"private_eq":0.9918},"43":{"raw":0.8018,"skill":0.6855,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6122,"private_eq":0.7577},"44":{"raw":0.7648,"skill":0.5296,"coverage":1,"random":0.5,"tracks":[],"combined":0.4817,"private_eq":0.7433},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4863,"skill":0.2544,"coverage":1,"random":0.311,"tracks":[],"combined":0.2539,"private_eq":0.4911},"56":{"raw":0.8275,"skill":0.655,"coverage":1,"random":0.5,"tracks":[],"combined":0.4906,"private_eq":0.7477},"57":{"raw":0.6368,"skill":0.5915,"coverage":0.9747,"random":0.1109,"tracks":[],"private_eq":0.6377,"combined":0.5888},"58":{"raw":0.7527,"skill":0.6415,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5062,"combined":0.277},"59":{"raw":0.7867,"skill":0.5577,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4033,"private_eq":0.715},"61":{"raw":0.704,"skill":0.408,"coverage":1,"random":0.5,"tracks":[],"combined":0.5017,"private_eq":0.7532},"62":{"raw":0.7081,"skill":0.6108,"coverage":1,"random":0.25,"tracks":[],"combined":0.5493,"private_eq":0.6651},"64":{"raw":0.7614,"skill":0.7017,"coverage":1,"random":0.2,"tracks":[],"combined":0.7489,"private_eq":0.8006}},"coverage":0.9995,"pending":0.0005,"notes":["New in the 2026-10-18 PR batch (PR #184).","Submitted public 0.3 run adopted (kit rescore 58.36, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.80% on 831 rows (bar 98.5%).","Two private new-domain tests are not counted for this model: our checks found signs its training data overlapped them. Its new-domain score uses the remaining tests, put on the same scale with the same stock models."],"scores":{"balanced_raw":68.61,"balanced_skill":60.27,"breadth_skill":57.43,"public_skill":58.36},"frozen_scores":{"balanced_raw":68.61,"balanced_skill":58.36,"breadth_skill":57.43},"categories":[{"id":"knowledge","raw":0.5659,"skill":0.4481,"combined":0.4354,"coverage":0.9975,"pending":0,"n":11},{"id":"language","raw":0.7276,"skill":0.6179,"combined":0.6074,"coverage":0.9999,"pending":0,"n":9},{"id":"retrieval","raw":0.7339,"skill":0.6329,"combined":0.6023,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.781,"skill":0.7504,"combined":0.7618,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.62,"skill":0.4416,"combined":0.4985,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"garum","pub":58.36,"same_skills":58.76,"new_domains":48.94,"v":60.27,"lab":false,"jev":false,"rank_v":46,"rank_pub":72,"name":"garum (Gemma 4 31B)","eq":{"S":58.38,"O":63.09},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.03,"new_domains":25.23}},"new_in_edition":true},{"engine":"jebadiah-27b-v2.1-t1e","name":"frontier-infra Jebadiah 27B v2.1 (t1e)","long_name":"frontier-infra Jebadiah 27B v2.1 (t1e)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/frontier-infra/jebadiah-27b-v2-1","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"frontier-infra","name":"Frontier Infra","logo":"data/orgs/frontier-infra.png"},"pr":199,"replaces":"jebadiah-v2.1","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7401,"conf":0.7926,"ece":0.0564,"brier":0.3456,"over95":0.0101,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0132,"conf":0.0761,"acc":0.1084},{"lo":0.1,"w":0.0282,"conf":0.1471,"acc":0.186},{"lo":0.2,"w":0.0185,"conf":0.2504,"acc":0.2748},{"lo":0.3,"w":0.0288,"conf":0.3543,"acc":0.2761},{"lo":0.4,"w":0.037,"conf":0.4552,"acc":0.3858},{"lo":0.5,"w":0.0713,"conf":0.5488,"acc":0.4404},{"lo":0.6,"w":0.0655,"conf":0.649,"acc":0.4667},{"lo":0.7,"w":0.1067,"conf":0.7508,"acc":0.6285},{"lo":0.8,"w":0.153,"conf":0.8556,"acc":0.7894},{"lo":0.9,"w":0.4778,"conf":0.9691,"acc":0.955}],"areas":{"knowledge":{"ece":0.0754,"acc":0.6789,"conf":0.7544},"language":{"ece":0.0167,"acc":0.8597,"conf":0.8599},"retrieval":{"ece":0.1467,"acc":0.6373,"conf":0.784},"tools":{"ece":0.0252,"acc":0.9449,"conf":0.9674},"arts":{"ece":0.0768,"acc":0.6068,"conf":0.6497}}},"gaps":null,"latency":{"median":303.2,"p95":2950.2,"mean":809.5},"results":{"1":{"score":0.9717,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":722.9},"2":{"score":0.6607,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":8765.5},"3":{"score":0.8504,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":4352.4},"4":{"score":0.7963,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":857.1},"5":{"score":0.8933,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1464},"6":{"score":0.7981,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":2492.1},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":8323.4},"10":{"score":0.2126,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":628.2},"11":{"score":0.7448,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":16208.3},"12":{"score":0.7895,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":222.2},"20":{"score":0.9578,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":241.9},"21":{"score":0.6164,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":107.1},"22":{"score":0.359,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3630.4},"23":{"score":0.6614,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":355},"24":{"score":0.8687,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":197.3},"25":{"score":0.5459,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":240.3},"26":{"score":0.9916,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":114},"27":{"score":0.9727,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":118.9},"28":{"score":0.9077,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":196.1},"29":{"score":0.9496,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":290.5},"30":{"score":0.685,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1608,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":552.5},"32":{"score":0.6503,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":617.9},"33":{"score":0.2527,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1760.1},"36":{"score":0.5006,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":13774.1},"37":{"score":0.5787,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":295.2},"38":{"score":0.2391,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":6435.2},"39":{"score":0.9058,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":297.3},"40":{"score":0.6204,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":199.6},"41":{"score":0.7728,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":220.5},"42":{"score":0.8612,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":364.7},"43":{"score":0.8193,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":219.1},"44":{"score":0.7788,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":215.2},"45":{"score":0.1058,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":281.7},"50":{"score":0.4696,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":638.3},"56":{"score":0.6765,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1007.9},"57":{"score":0.6841,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":265.3},"58":{"score":0.8239,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":222.5},"59":{"score":0.8093,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":310.2},"61":{"score":0.793,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":311.7},"62":{"score":0.7705,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":484.7},"64":{"score":0.767,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":261.4}},"benchmarks":{"1":{"raw":0.9717,"skill":0.9617,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6607,"skill":0.6082,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6967,"private_eq":0.7395},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9171,"private_eq":0.9186},"4":{"raw":0.7963,"skill":0.7937,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8796,"private_eq":0.8815},"5":{"raw":0.8933,"skill":0.8927,"coverage":1,"random":0.006,"tracks":[],"combined":0.7463,"private_eq":0.7497},"6":{"raw":0.7981,"skill":0.5266,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.7785,"private_eq":0.78},"10":{"raw":0.2126,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7448,"skill":0.631,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7895,"skill":0.6847,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5104,"private_eq":0.6762},"20":{"raw":0.9598,"skill":0.9196,"coverage":1,"random":0.5,"tracks":[],"combined":0.9288,"private_eq":0.9643},"21":{"raw":0.6164,"skill":0.2329,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.351,"skill":0.3459,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2477,"private_eq":0.2611},"23":{"raw":0.6553,"skill":0.3105,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8687,"skill":0.8249,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5459,"skill":0.3946,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4919,"combined":0.3157},"26":{"raw":0.9916,"skill":0.9888,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9727,"skill":0.9636,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9077,"skill":0.8154,"coverage":1,"random":0.5,"tracks":[],"combined":0.6731,"private_eq":0.8379},"29":{"raw":0.9496,"skill":0.9328,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.685,"skill":0.6252,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7771,"random":0.25},{"track":"10-choice","raw":0.5929,"random":0.1}],"combined":0.6993,"private_eq":0.7539},"31":{"raw":0.1608,"skill":0.086,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0702,"private_eq":0.1552},"32":{"raw":0.6503,"skill":0.444,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.2527,"skill":0.2428,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1995,"private_eq":0.218},"36":{"raw":0.5006,"skill":0.4351,"coverage":1,"random":0.116,"tracks":[],"combined":0.395,"private_eq":0.4704},"37":{"raw":0.5787,"skill":0.4715,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5676,"private_eq":0.6584},"38":{"raw":0.2391,"skill":0.2148,"coverage":1,"random":0.031,"tracks":[],"combined":0.3668,"private_eq":0.3924},"39":{"raw":0.9058,"skill":0.8615,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5467,"private_eq":0.6946},"40":{"raw":0.6204,"skill":0.5117,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4104,"headline":false},{"track":"A · English","score":0.6204,"headline":true},{"track":"C · Arabic pairs","score":0.81,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.5997,"private_eq":0.6916},"41":{"raw":0.7728,"skill":0.6591,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5333,"private_eq":0.6917},"42":{"raw":0.8612,"skill":0.73,"coverage":1,"random":0.486,"tracks":[],"combined":0.9188,"private_eq":0.9582},"43":{"raw":0.8193,"skill":0.7133,"coverage":1,"random":0.3697,"tracks":[],"combined":0.629,"private_eq":0.7682},"44":{"raw":0.7788,"skill":0.5576,"coverage":1,"random":0.5,"tracks":[],"combined":0.5673,"private_eq":0.7856},"45":{"raw":0.1058,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4696,"skill":0.2302,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.6765,"skill":0.353,"coverage":1,"random":0.5,"tracks":[],"combined":0.4906,"private_eq":0.7477},"57":{"raw":0.6841,"skill":0.6447,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5479,"combined":0.4867},"58":{"raw":0.8239,"skill":0.7447,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5082,"combined":0.2799},"59":{"raw":0.8093,"skill":0.6046,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3811,"private_eq":0.7044},"61":{"raw":0.793,"skill":0.586,"coverage":1,"random":0.5,"tracks":[],"combined":0.5357,"private_eq":0.77},"62":{"raw":0.7705,"skill":0.694,"coverage":1,"random":0.25,"tracks":[],"combined":0.563,"private_eq":0.6752},"64":{"raw":0.767,"skill":0.7087,"coverage":1,"random":0.2,"tracks":[],"combined":0.6713,"private_eq":0.7392}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #199).","Replaces the earlier board entry jebadiah-v2.1 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 60.46, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.90% on 831 rows (bar 98.5%).","Same weights as Jebadiah 27B v2.1 with a new inference recipe."],"scores":{"balanced_raw":70.03,"balanced_skill":64.4,"breadth_skill":59.56,"public_skill":60.46},"frozen_scores":{"balanced_raw":70.03,"balanced_skill":60.46,"breadth_skill":59.56},"categories":[{"id":"knowledge","raw":0.5856,"skill":0.4823,"combined":0.4309,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.749,"skill":0.6546,"combined":0.6143,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7086,"skill":0.5994,"combined":0.6089,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8194,"skill":0.7926,"combined":0.79,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6365,"skill":0.458,"combined":0.4806,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jebadiah-27b-v2.1-t1e","pub":60.46,"same_skills":59.35,"new_domains":56.57,"v":64.4,"lab":false,"jev":false,"rank_v":35,"rank_pub":58,"name":"frontier-infra Jebadiah 27B v2.1 (t1e)","eq":{"S":58.98,"O":72.52},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":35.39,"new_domains":29.01}},"new_in_edition":true},{"engine":"jebadiah-9b-v2.1-t1e","name":"frontier-infra Jebadiah 9B v2.1 (t1e)","long_name":"frontier-infra Jebadiah 9B v2.1 (t1e)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/frontier-infra/jebadiah-9b-v2-1","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"frontier-infra","name":"Frontier Infra","logo":"data/orgs/frontier-infra.png"},"pr":188,"replaces":"jebadiah-v2.1-9b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6935,"conf":0.7453,"ece":0.055,"brier":0.3986,"over95":0.0088,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0416,"conf":0.0644,"acc":0.0758},{"lo":0.1,"w":0.0194,"conf":0.1389,"acc":0.1981},{"lo":0.2,"w":0.0134,"conf":0.2531,"acc":0.2339},{"lo":0.3,"w":0.0419,"conf":0.3582,"acc":0.2769},{"lo":0.4,"w":0.0502,"conf":0.4537,"acc":0.3621},{"lo":0.5,"w":0.0862,"conf":0.5496,"acc":0.4804},{"lo":0.6,"w":0.0681,"conf":0.6478,"acc":0.5387},{"lo":0.7,"w":0.1363,"conf":0.7487,"acc":0.6731},{"lo":0.8,"w":0.1597,"conf":0.8535,"acc":0.7894},{"lo":0.9,"w":0.3832,"conf":0.9628,"acc":0.9336}],"areas":{"knowledge":{"ece":0.0806,"acc":0.6178,"conf":0.6963},"language":{"ece":0.0235,"acc":0.8138,"conf":0.8209},"retrieval":{"ece":0.1489,"acc":0.5996,"conf":0.7476},"tools":{"ece":0.0271,"acc":0.929,"conf":0.926},"arts":{"ece":0.0637,"acc":0.5623,"conf":0.5972}}},"gaps":null,"latency":{"median":125.1,"p95":968,"mean":294.2},"results":{"1":{"score":0.9445,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":249.6},"2":{"score":0.6207,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2977.8},"3":{"score":0.8583,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1412.1},"4":{"score":0.7475,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":280.7},"5":{"score":0.8538,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":484.8},"6":{"score":0.7995,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":810.3},"9":{"score":0.3636,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2765.6},"10":{"score":0.134,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":219.5},"11":{"score":0.7115,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":5540.3},"12":{"score":0.6891,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":101.6},"20":{"score":0.9264,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":104.7},"21":{"score":0.621,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":49.9},"22":{"score":0.1515,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1195.1},"23":{"score":0.622,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":131.9},"24":{"score":0.803,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":101.6},"25":{"score":0.4796,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":110.3},"26":{"score":0.984,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":53.4},"27":{"score":0.9599,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":54.4},"28":{"score":0.8129,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":107.4},"29":{"score":0.8822,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":128},"30":{"score":0.4659,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1382,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":196.9},"32":{"score":0.5612,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":221.2},"33":{"score":0.3067,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":614.4},"36":{"score":0.4265,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4612.6},"37":{"score":0.5274,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":126.1},"38":{"score":0.1956,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2191.4},"39":{"score":0.894,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":127.8},"40":{"score":0.5461,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":100.8},"41":{"score":0.6825,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":102.9},"42":{"score":0.8201,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":132.5},"43":{"score":0.607,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":102.6},"44":{"score":0.6912,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":102.4},"45":{"score":0.0978,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":113},"50":{"score":0.4654,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":218.8},"56":{"score":0.7185,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":345.4},"57":{"score":0.5878,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":107.1},"58":{"score":0.7066,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":102.7},"59":{"score":0.749,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":113.3},"61":{"score":0.7402,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":130.2},"62":{"score":0.8031,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":171.8},"64":{"score":0.6686,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":106.6}},"benchmarks":{"1":{"raw":0.9445,"skill":0.9251,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8376,"private_eq":0.8803},"2":{"raw":0.6207,"skill":0.562,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6811,"private_eq":0.7261},"3":{"raw":0.8583,"skill":0.8556,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.7475,"skill":0.7443,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8276,"private_eq":0.8307},"5":{"raw":0.8538,"skill":0.8529,"coverage":1,"random":0.006,"tracks":[],"combined":0.74,"private_eq":0.7435},"6":{"raw":0.7995,"skill":0.5299,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3636,"skill":0.3636,"coverage":1,"random":0,"tracks":[],"combined":0.4042,"private_eq":0.41},"10":{"raw":0.134,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7115,"skill":0.5827,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6891,"skill":0.5343,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5042,"private_eq":0.6721},"20":{"raw":0.9294,"skill":0.8587,"coverage":1,"random":0.5,"tracks":[],"combined":0.8566,"private_eq":0.9286},"21":{"raw":0.621,"skill":0.242,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.1444,"skill":0.1377,"coverage":1,"random":0.0078,"tracks":[],"combined":0.113,"private_eq":0.129},"23":{"raw":0.619,"skill":0.2379,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.803,"skill":0.7373,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3994,"combined":0.191},"26":{"raw":0.984,"skill":0.9787,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9599,"skill":0.9465,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8129,"skill":0.6258,"coverage":1,"random":0.5,"tracks":[],"combined":0.5572,"private_eq":0.7806},"29":{"raw":0.8822,"skill":0.8429,"coverage":1,"random":0.25,"tracks":[],"combined":0.8296,"private_eq":0.8729},"30":{"raw":0.4659,"skill":0.361,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5906,"random":0.25},{"track":"10-choice","raw":0.3412,"random":0.1}],"combined":0.5768,"private_eq":0.654},"31":{"raw":0.1382,"skill":0.0614,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0778,"private_eq":0.1621},"32":{"raw":0.5612,"skill":0.3024,"coverage":1,"random":0.371,"tracks":[],"combined":0.2119,"private_eq":0.5093},"33":{"raw":0.3067,"skill":0.2975,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3331,"private_eq":0.3483},"36":{"raw":0.4265,"skill":0.3513,"coverage":1,"random":0.116,"tracks":[],"combined":0.3506,"private_eq":0.4316},"37":{"raw":0.5274,"skill":0.4072,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5003,"private_eq":0.6053},"38":{"raw":0.1956,"skill":0.1699,"coverage":1,"random":0.031,"tracks":[],"combined":0.2684,"private_eq":0.2982},"39":{"raw":0.894,"skill":0.8441,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4906,"private_eq":0.6569},"40":{"raw":0.5461,"skill":0.4161,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.339,"headline":false},{"track":"A · English","score":0.5461,"headline":true},{"track":"C · Arabic pairs","score":0.73,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"combined":0.4045,"private_eq":0.5416},"41":{"raw":0.6825,"skill":0.5238,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4488,"private_eq":0.636},"42":{"raw":0.8201,"skill":0.65,"coverage":1,"random":0.486,"tracks":[],"combined":0.7426,"private_eq":0.8687},"43":{"raw":0.607,"skill":0.3765,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4408,"private_eq":0.6509},"44":{"raw":0.6912,"skill":0.3824,"coverage":1,"random":0.5,"tracks":[],"combined":0.4817,"private_eq":0.7433},"45":{"raw":0.0978,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4654,"skill":0.2241,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.7185,"skill":0.437,"coverage":1,"random":0.5,"tracks":[],"combined":0.4473,"private_eq":0.7263},"57":{"raw":0.5878,"skill":0.5364,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5215,"combined":0.4566},"58":{"raw":0.7066,"skill":0.5747,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4627,"combined":0.2132},"59":{"raw":0.749,"skill":0.4796,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2066,"private_eq":0.6212},"61":{"raw":0.7402,"skill":0.4804,"coverage":1,"random":0.5,"tracks":[],"combined":0.4677,"private_eq":0.7364},"62":{"raw":0.8031,"skill":0.7375,"coverage":1,"random":0.25,"tracks":[],"combined":0.6108,"private_eq":0.7107},"64":{"raw":0.6686,"skill":0.5857,"coverage":1,"random":0.2,"tracks":[],"combined":0.6228,"private_eq":0.7009}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #188).","Replaces the earlier board entry jebadiah-v2.1-9b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 51.53, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 100.00% on 831 rows (bar 98.5%).","Same weights as Jebadiah 9B v2.1 with a new inference recipe."],"scores":{"balanced_raw":63.27,"balanced_skill":50.9,"breadth_skill":50.2,"public_skill":51.53},"frozen_scores":{"balanced_raw":63.27,"balanced_skill":51.53,"breadth_skill":50.2},"categories":[{"id":"knowledge","raw":0.4993,"skill":0.3527,"combined":0.354,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6898,"skill":0.5658,"combined":0.4955,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6696,"skill":0.555,"combined":0.5632,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7316,"skill":0.7037,"combined":0.6947,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5746,"skill":0.381,"combined":0.4052,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jebadiah-9b-v2.1-t1e","pub":51.53,"same_skills":51.04,"new_domains":39.47,"v":50.9,"lab":false,"jev":false,"rank_v":91,"rank_pub":111,"name":"frontier-infra Jebadiah 9B v2.1 (t1e)","eq":{"S":50.57,"O":51.39},"rank_tie":20,"tied":true,"parts":{"public":0,"same_skills":30.34,"new_domains":20.56}},"new_in_edition":true},{"engine":"xor-26b-a4b-v1.2","name":"Xor 26B-A4B v1.2","long_name":"Xor 26B-A4B v1.2","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/juspay/Xor-26B-A4B","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"juspay","name":"Juspay Technologies","logo":"data/orgs/juspay.png"},"pr":202,"replaces":"xor-26b-a4b-v1.1","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.742,"conf":0.6744,"ece":0.0676,"brier":0.3576,"over95":0.0004,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0165,"conf":0.0807,"acc":0.2477},{"lo":0.1,"w":0.0376,"conf":0.1453,"acc":0.4708},{"lo":0.2,"w":0.0256,"conf":0.2522,"acc":0.4223},{"lo":0.3,"w":0.0549,"conf":0.352,"acc":0.3578},{"lo":0.4,"w":0.0708,"conf":0.4506,"acc":0.51},{"lo":0.5,"w":0.1382,"conf":0.5502,"acc":0.5649},{"lo":0.6,"w":0.1521,"conf":0.6496,"acc":0.6946},{"lo":0.7,"w":0.1402,"conf":0.7503,"acc":0.835},{"lo":0.8,"w":0.1521,"conf":0.8512,"acc":0.9384},{"lo":0.9,"w":0.2119,"conf":0.9456,"acc":0.9912}],"areas":{"knowledge":{"ece":0.056,"acc":0.6575,"conf":0.6613},"language":{"ece":0.0906,"acc":0.8434,"conf":0.7542},"retrieval":{"ece":0.1356,"acc":0.6692,"conf":0.5831},"tools":{"ece":0.1099,"acc":0.9301,"conf":0.8207},"arts":{"ece":0.1471,"acc":0.6737,"conf":0.5424}}},"gaps":null,"latency":{"median":109.9,"p95":479.8,"mean":168.1},"results":{"1":{"score":0.9569,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":821.5},"2":{"score":0.6606,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":7427.5},"3":{"score":0.8602,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3046.6},"4":{"score":0.8259,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":509.8},"5":{"score":0.9325,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":797.1},"6":{"score":0.7988,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1828.5},"9":{"score":0.6023,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1351.9},"10":{"score":0.2512,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":451.4},"11":{"score":0.7668,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":4479.6},"12":{"score":0.7583,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":141.4},"20":{"score":0.9678,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":130.8},"21":{"score":0.6629,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":127.5},"22":{"score":0.6165,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3706.5},"23":{"score":0.6726,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":155},"24":{"score":0.8516,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":139.9},"25":{"score":0.4592,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":156.3},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":133.2},"27":{"score":0.971,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":138},"28":{"score":0.865,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":134.6},"29":{"score":0.9477,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":184.7},"30":{"score":0.7517,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2224,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":430.2},"32":{"score":0.6543,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":401.4},"33":{"score":0.3764,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1274},"36":{"score":0.4886,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":6627.5},"37":{"score":0.5879,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":217.7},"38":{"score":0.2783,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2754.8},"39":{"score":0.9133,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":196.9},"40":{"score":0.6344,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":144.5},"41":{"score":0.6696,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":158.9},"42":{"score":0.8204,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":164.6},"43":{"score":0.7088,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":150},"44":{"score":0.7332,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":139.5},"45":{"score":0.1038,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":217.1},"50":{"score":0.4678,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":383.4},"56":{"score":0.8085,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":538.8},"57":{"score":0.6677,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":152.8},"58":{"score":0.7805,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":154.5},"59":{"score":0.8,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":174.5},"61":{"score":0.7833,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":215.7},"62":{"score":0.6457,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":231.4},"64":{"score":0.7273,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":143.7}},"benchmarks":{"1":{"raw":0.9569,"skill":0.9418,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8921,"private_eq":0.9202},"2":{"raw":0.6606,"skill":0.608,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6854,"private_eq":0.7298},"3":{"raw":0.8602,"skill":0.8575,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8961,"private_eq":0.8982},"4":{"raw":0.8259,"skill":0.8237,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8798,"private_eq":0.8817},"5":{"raw":0.9325,"skill":0.9321,"coverage":1,"random":0.006,"tracks":[],"combined":0.7969,"private_eq":0.7994},"6":{"raw":0.7988,"skill":0.5283,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6023,"skill":0.6023,"coverage":1,"random":0,"tracks":[],"combined":0.518,"private_eq":0.5225},"10":{"raw":0.2512,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7668,"skill":0.6628,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7583,"skill":0.638,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4793,"private_eq":0.6557},"20":{"raw":0.9688,"skill":0.9376,"coverage":1,"random":0.5,"tracks":[],"combined":0.9288,"private_eq":0.9643},"21":{"raw":0.6629,"skill":0.3257,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.613,"skill":0.61,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4492,"private_eq":0.4587},"23":{"raw":0.6758,"skill":0.3517,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8516,"skill":0.8021,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4592,"skill":0.2789,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4877,"combined":0.3101},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.971,"skill":0.9613,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.865,"skill":0.73,"coverage":1,"random":0.5,"tracks":[],"combined":0.6328,"private_eq":0.818},"29":{"raw":0.9477,"skill":0.9303,"coverage":1,"random":0.25,"tracks":[],"combined":0.9979,"private_eq":0.9977},"30":{"raw":0.7517,"skill":0.7031,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8105,"random":0.25},{"track":"10-choice","raw":0.6929,"random":0.1}],"combined":0.7785,"private_eq":0.8185},"31":{"raw":0.2224,"skill":0.1531,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.6543,"skill":0.4504,"coverage":1,"random":0.371,"tracks":[],"combined":0.3047,"private_eq":0.567},"33":{"raw":0.3764,"skill":0.3681,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4154,"private_eq":0.4286},"36":{"raw":0.4886,"skill":0.4215,"coverage":1,"random":0.116,"tracks":[],"combined":0.3847,"private_eq":0.4614},"37":{"raw":0.5879,"skill":0.4831,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5471,"private_eq":0.6422},"38":{"raw":0.2783,"skill":0.2552,"coverage":1,"random":0.031,"tracks":[],"combined":0.4461,"private_eq":0.4684},"39":{"raw":0.9133,"skill":0.8725,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5812,"private_eq":0.7178},"40":{"raw":0.6344,"skill":0.5297,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5769,"headline":false},{"track":"A · English","score":0.6344,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.5204,"private_eq":0.6307},"41":{"raw":0.6696,"skill":0.5045,"coverage":1,"random":0.3333,"tracks":[],"combined":0.539,"private_eq":0.6955},"42":{"raw":0.8204,"skill":0.6506,"coverage":1,"random":0.486,"tracks":[],"combined":0.8398,"private_eq":0.9181},"43":{"raw":0.7088,"skill":0.5379,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6205,"private_eq":0.7629},"44":{"raw":0.7332,"skill":0.4664,"coverage":1,"random":0.5,"tracks":[],"combined":0.4977,"private_eq":0.7512},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4678,"skill":0.2276,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.8085,"skill":0.617,"coverage":1,"random":0.5,"tracks":[],"combined":0.4906,"private_eq":0.7477},"57":{"raw":0.6677,"skill":0.6263,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.626,"combined":0.5755},"58":{"raw":0.7805,"skill":0.6818,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5256,"combined":0.3055},"59":{"raw":0.8,"skill":0.5853,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3177,"private_eq":0.6742},"61":{"raw":0.7833,"skill":0.5666,"coverage":1,"random":0.5,"tracks":[],"combined":0.5642,"private_eq":0.7841},"62":{"raw":0.6457,"skill":0.5276,"coverage":1,"random":0.25,"tracks":[],"combined":0.5356,"private_eq":0.6549},"64":{"raw":0.7273,"skill":0.6591,"coverage":1,"random":0.2,"tracks":[],"combined":0.7003,"private_eq":0.7622}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #202).","Replaces the earlier board entry xor-26b-a4b-v1.1 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 59.46, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.57% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":69.41,"balanced_skill":59.44,"breadth_skill":58.76,"public_skill":59.46},"frozen_scores":{"balanced_raw":69.41,"balanced_skill":59.46,"breadth_skill":58.76},"categories":[{"id":"knowledge","raw":0.5749,"skill":0.4549,"combined":0.4536,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7372,"skill":0.6322,"combined":0.5972,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7388,"skill":0.6484,"combined":0.6175,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7572,"skill":0.7217,"combined":0.7194,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6859,"skill":0.5186,"combined":0.5304,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"xor-26b-a4b-v1.2","pub":59.46,"same_skills":58.74,"new_domains":47.3,"v":59.44,"lab":false,"jev":false,"rank_v":55,"rank_pub":64,"name":"Xor 26B-A4B v1.2","eq":{"S":58.37,"O":61.06},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":35.02,"new_domains":24.42}},"new_in_edition":true},{"engine":"startlux-decision-v1.1-9b-instant","name":"StartLux-Decision v1.1 9B","long_name":"StartLux-Decision v1.1 9B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/startlux-models/StartLux-Decision-v1.1-9B-Instant","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"startlux-models","name":"StartLux","logo":"data/orgs/startlux-models.png"},"pr":208,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8015,"conf":0.8473,"ece":0.0458,"brier":0.265,"over95":0.0142,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0013,"conf":0.0922,"acc":0.0929},{"lo":0.1,"w":0.0049,"conf":0.1556,"acc":0.0954},{"lo":0.2,"w":0.0126,"conf":0.2578,"acc":0.1292},{"lo":0.3,"w":0.0294,"conf":0.3517,"acc":0.2388},{"lo":0.4,"w":0.0346,"conf":0.454,"acc":0.3768},{"lo":0.5,"w":0.0765,"conf":0.5506,"acc":0.4807},{"lo":0.6,"w":0.0722,"conf":0.6489,"acc":0.5375},{"lo":0.7,"w":0.0747,"conf":0.7497,"acc":0.6869},{"lo":0.8,"w":0.0816,"conf":0.8502,"acc":0.7571},{"lo":0.9,"w":0.6121,"conf":0.9846,"acc":0.9647}],"areas":{"knowledge":{"ece":0.107,"acc":0.6859,"conf":0.7929},"language":{"ece":0.0172,"acc":0.9021,"conf":0.912},"retrieval":{"ece":0.0337,"acc":0.8146,"conf":0.8474},"tools":{"ece":0.0073,"acc":0.9687,"conf":0.9663},"arts":{"ece":0.0319,"acc":0.7406,"conf":0.768}}},"gaps":null,"latency":{"median":38.6,"p95":261.4,"mean":93},"results":{"1":{"score":0.9776,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":238},"2":{"score":0.6774,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1715.5},"3":{"score":0.8957,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1756.4},"4":{"score":0.9299,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":174.1},"5":{"score":0.9428,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":193.4},"6":{"score":0.7961,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":303.8},"9":{"score":0.9886,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":461.8},"10":{"score":0.6839,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":169.1},"11":{"score":0.8656,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":443.6},"12":{"score":0.7699,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":75.9},"20":{"score":0.9718,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":65.6},"21":{"score":0.6636,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":65.6},"22":{"score":0.8175,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1083.5},"23":{"score":0.6988,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77},"24":{"score":0.8212,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":66},"25":{"score":0.5051,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":61.3},"26":{"score":0.9878,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":60.9},"27":{"score":0.959,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":59.7},"28":{"score":0.9266,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":61.7},"29":{"score":0.9664,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":70},"30":{"score":0.6994,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2358,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":178.5},"32":{"score":0.7074,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":136.3},"33":{"score":0.5824,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":292.3},"36":{"score":0.4646,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":407.3},"37":{"score":0.6304,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":197.7},"38":{"score":0.6422,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":301.2},"39":{"score":0.9574,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":163.2},"40":{"score":0.6148,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":111.1},"41":{"score":0.7897,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":82.5},"42":{"score":0.8457,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":88.1},"43":{"score":0.6877,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":74.8},"44":{"score":0.9266,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":75.9},"45":{"score":0.1158,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":77.3},"50":{"score":0.6563,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":86.8},"56":{"score":0.9235,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":111.9},"57":{"score":0.6464,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":75},"58":{"score":0.8694,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":74.1},"59":{"score":0.8424,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":90.6},"61":{"score":0.8995,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":89.7},"62":{"score":0.9318,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":86.7},"64":{"score":0.803,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":80.9}},"benchmarks":{"1":{"raw":0.9776,"skill":0.9697,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9192,"private_eq":0.9401},"2":{"raw":0.6774,"skill":0.6274,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6959,"private_eq":0.7388},"3":{"raw":0.8957,"skill":0.8937,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9118,"private_eq":0.9135},"4":{"raw":0.9299,"skill":0.929,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8535,"private_eq":0.856},"5":{"raw":0.9428,"skill":0.9425,"coverage":1,"random":0.006,"tracks":[],"combined":0.8087,"private_eq":0.811},"6":{"raw":0.7961,"skill":0.5219,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9886,"skill":0.9886,"coverage":1,"random":0,"tracks":[],"combined":0.8569,"private_eq":0.8575},"10":{"raw":0.6839,"skill":0.474,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8656,"skill":0.8056,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7699,"skill":0.6553,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5013,"private_eq":0.6702},"20":{"raw":0.9734,"skill":0.9468,"coverage":1,"random":0.5,"tracks":[],"combined":0.9721,"private_eq":0.9857},"21":{"raw":0.6636,"skill":0.3272,"coverage":1,"random":0.5,"tracks":[],"combined":0.3773,"private_eq":0.6917},"22":{"raw":0.8142,"skill":0.8128,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6373,"private_eq":0.6432},"23":{"raw":0.7217,"skill":0.4435,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8212,"skill":0.7616,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4607,"combined":0.2737},"26":{"raw":0.9878,"skill":0.9837,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.959,"skill":0.9453,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9266,"skill":0.8532,"coverage":1,"random":0.5,"tracks":[],"combined":0.5721,"private_eq":0.788},"29":{"raw":0.9664,"skill":0.9552,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6994,"skill":0.64,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7657,"random":0.25},{"track":"10-choice","raw":0.6331,"random":0.1}],"combined":0.6577,"private_eq":0.72},"31":{"raw":0.2358,"skill":0.1677,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1005,"private_eq":0.1827},"32":{"raw":0.7074,"skill":0.5348,"coverage":1,"random":0.371,"tracks":[],"combined":0.4054,"private_eq":0.6296},"33":{"raw":0.5824,"skill":0.5769,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4461,"private_eq":0.4586},"36":{"raw":0.4646,"skill":0.3943,"coverage":1,"random":0.116,"tracks":[],"combined":0.3979,"private_eq":0.4729},"37":{"raw":0.6304,"skill":0.5364,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5341,"private_eq":0.632},"38":{"raw":0.6422,"skill":0.6308,"coverage":1,"random":0.031,"tracks":[],"combined":0.7338,"private_eq":0.744},"39":{"raw":0.9574,"skill":0.9373,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5861,"private_eq":0.7211},"40":{"raw":0.6148,"skill":0.5044,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4721,"headline":false},{"track":"A · English","score":0.6148,"headline":true},{"track":"C · Arabic pairs","score":0.95,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.4962,"private_eq":0.6121},"41":{"raw":0.7897,"skill":0.6845,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3427,"private_eq":0.5661},"42":{"raw":0.8457,"skill":0.6998,"coverage":1,"random":0.486,"tracks":[],"combined":0.8235,"private_eq":0.9098},"43":{"raw":0.6877,"skill":0.5045,"coverage":1,"random":0.3697,"tracks":[],"combined":0.491,"private_eq":0.6822},"44":{"raw":0.9266,"skill":0.8532,"coverage":1,"random":0.5,"tracks":[],"combined":0.8189,"private_eq":0.91},"45":{"raw":0.1158,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6563,"skill":0.5012,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.9235,"skill":0.847,"coverage":1,"random":0.5,"tracks":[],"combined":0.5857,"private_eq":0.7947},"57":{"raw":0.6464,"skill":0.6023,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5234,"combined":0.4588},"58":{"raw":0.8694,"skill":0.8107,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.486,"combined":0.2474},"59":{"raw":0.8424,"skill":0.6732,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2835,"private_eq":0.6579},"61":{"raw":0.8995,"skill":0.799,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.9318,"skill":0.9091,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.803,"skill":0.7537,"coverage":1,"random":0.2,"tracks":[],"combined":0.7101,"private_eq":0.7699}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #208).","Submitted public 0.3 run adopted (kit rescore 69.98, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.85% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":77.28,"balanced_skill":58.07,"breadth_skill":69.08,"public_skill":69.98},"frozen_scores":{"balanced_raw":77.28,"balanced_skill":69.98,"breadth_skill":69.08},"categories":[{"id":"knowledge","raw":0.6263,"skill":0.5337,"combined":0.4428,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8129,"skill":0.7307,"combined":0.6034,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7967,"skill":0.7426,"combined":0.6417,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8974,"skill":0.8817,"combined":0.8063,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.772,"skill":0.6309,"combined":0.5829,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"startlux-decision-v1.1-9b-instant","pub":69.98,"same_skills":61.3,"new_domains":41.38,"v":58.07,"lab":false,"jev":false,"rank_v":64,"rank_pub":12,"name":"StartLux-Decision v1.1 9B","eq":{"S":60.95,"O":53.75},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":36.57,"new_domains":21.5}},"new_in_edition":true},{"engine":"startlux-decision-v1.1-4b-instant","name":"StartLux-Decision v1.1 4B","long_name":"StartLux-Decision v1.1 4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/startlux-models/StartLux-Decision-v1.1-4B-Instant","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"startlux-models","name":"StartLux","logo":"data/orgs/startlux-models.png"},"pr":208,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7812,"conf":0.8329,"ece":0.0517,"brier":0.2956,"over95":0.0217,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0015,"conf":0.0841,"acc":0.026},{"lo":0.1,"w":0.0059,"conf":0.1581,"acc":0.1376},{"lo":0.2,"w":0.0172,"conf":0.2654,"acc":0.0873},{"lo":0.3,"w":0.0292,"conf":0.355,"acc":0.3275},{"lo":0.4,"w":0.0441,"conf":0.4526,"acc":0.357},{"lo":0.5,"w":0.0822,"conf":0.5499,"acc":0.4749},{"lo":0.6,"w":0.0732,"conf":0.6493,"acc":0.5641},{"lo":0.7,"w":0.074,"conf":0.7476,"acc":0.6907},{"lo":0.8,"w":0.0908,"conf":0.8537,"acc":0.8003},{"lo":0.9,"w":0.5818,"conf":0.982,"acc":0.9443}],"areas":{"knowledge":{"ece":0.0918,"acc":0.6717,"conf":0.7635},"language":{"ece":0.0121,"acc":0.8839,"conf":0.8918},"retrieval":{"ece":0.1214,"acc":0.7535,"conf":0.8702},"tools":{"ece":0.0197,"acc":0.9486,"conf":0.9683},"arts":{"ece":0.0273,"acc":0.7375,"conf":0.7548}}},"gaps":null,"latency":{"median":38.3,"p95":165.7,"mean":68.8},"results":{"1":{"score":0.9628,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":195},"2":{"score":0.6773,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1110.6},"3":{"score":0.8701,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1074.1},"4":{"score":0.907,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":175.3},"5":{"score":0.9347,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":193.4},"6":{"score":0.7999,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":187.3},"9":{"score":0.9545,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":237},"10":{"score":0.5477,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":128.8},"11":{"score":0.8551,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":244.2},"12":{"score":0.7465,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":76.9},"20":{"score":0.9586,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":65.1},"21":{"score":0.6514,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":67},"22":{"score":0.8035,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":729.5},"23":{"score":0.6896,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":88.8},"24":{"score":0.7966,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":72.3},"25":{"score":0.4439,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":72},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":71.1},"27":{"score":0.9531,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":68.9},"28":{"score":0.8958,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":65.4},"29":{"score":0.9481,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":68.9},"30":{"score":0.6967,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.221,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":170.2},"32":{"score":0.6902,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":150},"33":{"score":0.5782,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":250.2},"36":{"score":0.4777,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":355.2},"37":{"score":0.621,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":185.8},"38":{"score":0.6313,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":358.1},"39":{"score":0.9544,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":181.8},"40":{"score":0.6167,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":89.3},"41":{"score":0.7863,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":75.4},"42":{"score":0.8111,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":85},"43":{"score":0.6491,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":75.3},"44":{"score":0.917,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.3},"45":{"score":0.1078,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":76.6},"50":{"score":0.6342,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":86},"56":{"score":0.9055,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":115.5},"57":{"score":0.6093,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":74},"58":{"score":0.856,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":72.7},"59":{"score":0.8404,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":91.9},"61":{"score":0.8918,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":89.6},"62":{"score":0.928,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":87.1},"64":{"score":0.7538,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":79.8}},"benchmarks":{"1":{"raw":0.9628,"skill":0.9498,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8171,"private_eq":0.8653},"2":{"raw":0.6773,"skill":0.6274,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6957,"private_eq":0.7386},"3":{"raw":0.8701,"skill":0.8676,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8699,"private_eq":0.8728},"4":{"raw":0.907,"skill":0.9058,"coverage":1,"random":0.0127,"tracks":[],"combined":0.869,"private_eq":0.8711},"5":{"raw":0.9347,"skill":0.9343,"coverage":1,"random":0.006,"tracks":[],"combined":0.797,"private_eq":0.7995},"6":{"raw":0.7999,"skill":0.5308,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9545,"skill":0.9545,"coverage":1,"random":0,"tracks":[],"combined":0.8316,"private_eq":0.8325},"10":{"raw":0.5477,"skill":0.2474,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8551,"skill":0.7904,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7465,"skill":0.6203,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5103,"private_eq":0.6761},"20":{"raw":0.96,"skill":0.9201,"coverage":1,"random":0.5,"tracks":[],"combined":0.9361,"private_eq":0.9679},"21":{"raw":0.6514,"skill":0.3029,"coverage":1,"random":0.5,"tracks":[],"combined":0.3975,"private_eq":0.7017},"22":{"raw":0.799,"skill":0.7975,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6091,"private_eq":0.6156},"23":{"raw":0.7097,"skill":0.4195,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7966,"skill":0.7288,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4275,"combined":0.2289},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9531,"skill":0.9374,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8958,"skill":0.7916,"coverage":1,"random":0.5,"tracks":[],"combined":0.5319,"private_eq":0.7681},"29":{"raw":0.9481,"skill":0.9308,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6967,"skill":0.6368,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7642,"random":0.25},{"track":"10-choice","raw":0.6293,"random":0.1}],"combined":0.5931,"private_eq":0.6673},"31":{"raw":0.221,"skill":0.1515,"coverage":1,"random":0.0819,"tracks":[],"combined":0.095,"private_eq":0.1777},"32":{"raw":0.6902,"skill":0.5075,"coverage":1,"random":0.371,"tracks":[],"combined":0.3434,"private_eq":0.5911},"33":{"raw":0.5782,"skill":0.5726,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4256,"private_eq":0.4386},"36":{"raw":0.4777,"skill":0.4092,"coverage":1,"random":0.116,"tracks":[],"combined":0.3765,"private_eq":0.4542},"37":{"raw":0.621,"skill":0.5247,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5985,"private_eq":0.6827},"38":{"raw":0.6313,"skill":0.6195,"coverage":1,"random":0.031,"tracks":[],"combined":0.7317,"private_eq":0.742},"39":{"raw":0.9544,"skill":0.9329,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6141,"private_eq":0.7399},"40":{"raw":0.6167,"skill":0.5069,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4743,"headline":false},{"track":"A · English","score":0.6167,"headline":true},{"track":"C · Arabic pairs","score":0.935,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0.3559,"private_eq":0.5043},"41":{"raw":0.7863,"skill":0.6795,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0694,"private_eq":0.386},"42":{"raw":0.8111,"skill":0.6325,"coverage":1,"random":0.486,"tracks":[],"combined":0.8792,"private_eq":0.9381},"43":{"raw":0.6491,"skill":0.4433,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4491,"private_eq":0.6561},"44":{"raw":0.917,"skill":0.834,"coverage":1,"random":0.5,"tracks":[],"combined":0.7655,"private_eq":0.8836},"45":{"raw":0.1078,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6342,"skill":0.4692,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.9055,"skill":0.811,"coverage":1,"random":0.5,"tracks":[],"combined":0.577,"private_eq":0.7904},"57":{"raw":0.6093,"skill":0.5606,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4883,"combined":0.4189},"58":{"raw":0.856,"skill":0.7913,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4767,"combined":0.2338},"59":{"raw":0.8404,"skill":0.6691,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2475,"private_eq":0.6407},"61":{"raw":0.8918,"skill":0.7836,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.928,"skill":0.904,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.7538,"skill":0.6923,"coverage":1,"random":0.2,"tracks":[],"combined":0.7198,"private_eq":0.7776}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #208).","Submitted public 0.3 run adopted (kit rescore 67.81, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.51% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":75.72,"balanced_skill":54.8,"breadth_skill":66.78,"public_skill":67.81},"frozen_scores":{"balanced_raw":75.72,"balanced_skill":67.81,"breadth_skill":66.78},"categories":[{"id":"knowledge","raw":0.604,"skill":0.5023,"combined":0.4044,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8009,"skill":0.7119,"combined":0.5609,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7881,"skill":0.7301,"combined":0.6469,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8813,"skill":0.8642,"combined":0.7696,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7513,"skill":0.6002,"combined":0.5717,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"startlux-decision-v1.1-4b-instant","pub":67.81,"same_skills":58.6,"new_domains":38.07,"v":54.8,"lab":false,"jev":false,"rank_v":78,"rank_pub":19,"name":"StartLux-Decision v1.1 4B","eq":{"S":58.22,"O":49.67},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":34.93,"new_domains":19.87}},"new_in_edition":true},{"engine":"startlux-decision-v1.1-2b-instant","name":"StartLux-Decision v1.1 2B","long_name":"StartLux-Decision v1.1 2B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/startlux-models/StartLux-Decision-v1.1-2B-Instant","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"org":{"id":"startlux-models","name":"StartLux","logo":"data/orgs/startlux-models.png"},"pr":208,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7407,"conf":0.7759,"ece":0.0354,"brier":0.3452,"over95":0.0149,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0008,"conf":0.0823,"acc":0.0476},{"lo":0.1,"w":0.0084,"conf":0.1541,"acc":0.1654},{"lo":0.2,"w":0.0262,"conf":0.262,"acc":0.1723},{"lo":0.3,"w":0.0493,"conf":0.3501,"acc":0.2742},{"lo":0.4,"w":0.0545,"conf":0.4535,"acc":0.3861},{"lo":0.5,"w":0.116,"conf":0.5498,"acc":0.5334},{"lo":0.6,"w":0.093,"conf":0.649,"acc":0.6448},{"lo":0.7,"w":0.0904,"conf":0.7502,"acc":0.7242},{"lo":0.8,"w":0.11,"conf":0.8505,"acc":0.789},{"lo":0.9,"w":0.4515,"conf":0.9751,"acc":0.9438}],"areas":{"knowledge":{"ece":0.0806,"acc":0.6067,"conf":0.6865},"language":{"ece":0.0109,"acc":0.8464,"conf":0.8395},"retrieval":{"ece":0.1072,"acc":0.7317,"conf":0.8331},"tools":{"ece":0.0232,"acc":0.9235,"conf":0.9268},"arts":{"ece":0.0308,"acc":0.7159,"conf":0.704}}},"gaps":null,"latency":{"median":30.2,"p95":88.4,"mean":44.3},"results":{"1":{"score":0.9463,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":161.8},"2":{"score":0.659,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":962.5},"3":{"score":0.8012,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1092.8},"4":{"score":0.8981,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":146},"5":{"score":0.9059,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":169.9},"6":{"score":0.7853,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":186.9},"9":{"score":0.7159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":267.9},"10":{"score":0.5127,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":113.7},"11":{"score":0.842,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":269.6},"12":{"score":0.644,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":54.4},"20":{"score":0.909,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.2},"21":{"score":0.6309,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":56.9},"22":{"score":0.777,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":702.1},"23":{"score":0.672,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.2},"24":{"score":0.6681,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":54.9},"25":{"score":0.3418,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":54.5},"26":{"score":0.9579,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":54.6},"27":{"score":0.8874,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":54.7},"28":{"score":0.7806,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":54.6},"29":{"score":0.884,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":54.7},"30":{"score":0.5186,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2122,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":133.6},"32":{"score":0.6503,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":115.2},"33":{"score":0.5479,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":365},"36":{"score":0.4071,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":558.4},"37":{"score":0.6042,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110.9},"38":{"score":0.5788,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":652.6},"39":{"score":0.9319,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":209.3},"40":{"score":0.4732,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":69.8},"41":{"score":0.7762,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":56.1},"42":{"score":0.7219,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":68.5},"43":{"score":0.4035,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":56.7},"44":{"score":0.8326,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.6},"45":{"score":0.1377,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":58.8},"50":{"score":0.6163,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":69.4},"56":{"score":0.892,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":148.3},"57":{"score":0.4441,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":55.2},"58":{"score":0.7699,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":55.2},"59":{"score":0.811,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":71.6},"61":{"score":0.878,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":67.3},"62":{"score":0.9036,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":69.8},"64":{"score":0.6856,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":60.7}},"benchmarks":{"1":{"raw":0.9463,"skill":0.9275,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8717,"private_eq":0.9053},"2":{"raw":0.659,"skill":0.6062,"coverage":1,"random":0.1341,"tracks":[],"combined":0.675,"private_eq":0.7209},"3":{"raw":0.8012,"skill":0.7974,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7648,"private_eq":0.7709},"4":{"raw":0.8981,"skill":0.8968,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8024,"private_eq":0.8061},"5":{"raw":0.9059,"skill":0.9053,"coverage":1,"random":0.006,"tracks":[],"combined":0.7489,"private_eq":0.7522},"6":{"raw":0.7853,"skill":0.4966,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7159,"skill":0.7159,"coverage":1,"random":0,"tracks":[],"combined":0.647,"private_eq":0.65},"10":{"raw":0.5127,"skill":0.1892,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.842,"skill":0.7715,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.644,"skill":0.4667,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4224,"private_eq":0.6181},"20":{"raw":0.91,"skill":0.8201,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.6309,"skill":0.2618,"coverage":1,"random":0.5,"tracks":[],"combined":0.317,"private_eq":0.6619},"22":{"raw":0.7646,"skill":0.7628,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5536,"private_eq":0.5611},"23":{"raw":0.6627,"skill":0.3255,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6681,"skill":0.5575,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3418,"skill":0.1224,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3308,"combined":0.0985},"26":{"raw":0.9579,"skill":0.9439,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8874,"skill":0.8498,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7806,"skill":0.5612,"coverage":1,"random":0.5,"tracks":[],"combined":0.3453,"private_eq":0.6759},"29":{"raw":0.884,"skill":0.8453,"coverage":1,"random":0.25,"tracks":[],"combined":0.8246,"private_eq":0.8692},"30":{"raw":0.5186,"skill":0.4228,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6194,"random":0.25},{"track":"10-choice","raw":0.4177,"random":0.1}],"combined":0.4626,"private_eq":0.5609},"31":{"raw":0.2122,"skill":0.142,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.6503,"skill":0.444,"coverage":1,"random":0.371,"tracks":[],"combined":0.181,"private_eq":0.4901},"33":{"raw":0.5479,"skill":0.5419,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3383,"private_eq":0.3534},"36":{"raw":0.4071,"skill":0.3293,"coverage":1,"random":0.116,"tracks":[],"combined":0.2802,"private_eq":0.3701},"37":{"raw":0.6042,"skill":0.5035,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5524,"private_eq":0.6464},"38":{"raw":0.5788,"skill":0.5653,"coverage":1,"random":0.031,"tracks":[],"combined":0.6825,"private_eq":0.6949},"39":{"raw":0.9319,"skill":0.8998,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5856,"private_eq":0.7208},"40":{"raw":0.4732,"skill":0.3222,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4117,"headline":false},{"track":"A · English","score":0.4732,"headline":true},{"track":"C · Arabic pairs","score":0.905,"headline":false},{"track":"C · English pairs","score":0.87,"headline":false}],"combined":0.3559,"private_eq":0.5043},"41":{"raw":0.7762,"skill":0.6642,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0968,"private_eq":0.404},"42":{"raw":0.7219,"skill":0.4589,"coverage":1,"random":0.486,"tracks":[],"combined":0.8467,"private_eq":0.9216},"43":{"raw":0.4035,"skill":0.0536,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3196,"private_eq":0.5754},"44":{"raw":0.8326,"skill":0.6652,"coverage":1,"random":0.5,"tracks":[],"combined":0.578,"private_eq":0.7909},"45":{"raw":0.1377,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6163,"skill":0.4432,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.892,"skill":0.784,"coverage":1,"random":0.5,"tracks":[],"combined":0.5596,"private_eq":0.7818},"57":{"raw":0.4441,"skill":0.3748,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.333,"combined":0.2422},"58":{"raw":0.7699,"skill":0.6665,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3845,"combined":0.0986},"59":{"raw":0.811,"skill":0.6081,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2527,"private_eq":0.6432},"61":{"raw":0.878,"skill":0.756,"coverage":1,"random":0.5,"tracks":[],"combined":0.6152,"private_eq":0.8093},"62":{"raw":0.9036,"skill":0.8715,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.6856,"skill":0.607,"coverage":1,"random":0.2,"tracks":[],"combined":0.6519,"private_eq":0.7239}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #208).","Submitted public 0.3 run adopted (kit rescore 59.23, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.80% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":69.56,"balanced_skill":40.11,"breadth_skill":57.38,"public_skill":59.23},"frozen_scores":{"balanced_raw":69.56,"balanced_skill":59.23,"breadth_skill":57.38},"categories":[{"id":"knowledge","raw":0.5112,"skill":0.3616,"combined":0.2698,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7413,"skill":0.6239,"combined":0.5139,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7617,"skill":0.6971,"combined":0.5947,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8103,"skill":0.7895,"combined":0.7165,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7117,"skill":0.5367,"combined":0.5208,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"startlux-decision-v1.1-2b-instant","pub":59.23,"same_skills":51.52,"new_domains":17.04,"v":40.11,"lab":false,"jev":false,"rank_v":134,"rank_pub":66,"name":"StartLux-Decision v1.1 2B","eq":{"S":51.07,"O":23.68},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":30.64,"new_domains":9.47}},"new_in_edition":true},{"engine":"startlux-decision-v1.1-0.8b-instant","name":"StartLux-Decision v1.1 0.8B","long_name":"StartLux-Decision v1.1 0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/startlux-models/StartLux-Decision-v1.1-0.8B-Instant","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"startlux-models","name":"StartLux","logo":"data/orgs/startlux-models.png"},"pr":208,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7251,"conf":0.7269,"ece":0.0145,"brier":0.362,"over95":0.0051,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0027,"conf":0.0833,"acc":0.0591},{"lo":0.1,"w":0.0112,"conf":0.1614,"acc":0.2645},{"lo":0.2,"w":0.0332,"conf":0.262,"acc":0.2039},{"lo":0.3,"w":0.0679,"conf":0.3509,"acc":0.332},{"lo":0.4,"w":0.064,"conf":0.4511,"acc":0.4645},{"lo":0.5,"w":0.152,"conf":0.5466,"acc":0.5568},{"lo":0.6,"w":0.1067,"conf":0.6488,"acc":0.6752},{"lo":0.7,"w":0.098,"conf":0.7515,"acc":0.7485},{"lo":0.8,"w":0.1099,"conf":0.8527,"acc":0.8372},{"lo":0.9,"w":0.3545,"conf":0.9698,"acc":0.9618}],"areas":{"knowledge":{"ece":0.0509,"acc":0.5851,"conf":0.6289},"language":{"ece":0.0144,"acc":0.8055,"conf":0.7962},"retrieval":{"ece":0.0175,"acc":0.7836,"conf":0.7818},"tools":{"ece":0.0487,"acc":0.9095,"conf":0.8651},"arts":{"ece":0.0406,"acc":0.7027,"conf":0.6711}}},"gaps":null,"latency":{"median":29.9,"p95":87.1,"mean":40.3},"results":{"1":{"score":0.9351,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":151.4},"2":{"score":0.6401,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":971.9},"3":{"score":0.7835,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1052.7},"4":{"score":0.902,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":142.2},"5":{"score":0.9027,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":164.9},"6":{"score":0.6852,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":182.7},"9":{"score":0.6023,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":257.5},"10":{"score":0.6561,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":106},"11":{"score":0.844,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":293.7},"12":{"score":0.5714,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":54.8},"20":{"score":0.9026,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.4},"21":{"score":0.6126,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":55.7},"22":{"score":0.781,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":742.5},"23":{"score":0.6634,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":71.9},"24":{"score":0.5588,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":55},"25":{"score":0.301,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":55.2},"26":{"score":0.907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":54.4},"27":{"score":0.7816,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":52.3},"28":{"score":0.7064,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":50.2},"29":{"score":0.7857,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":54.7},"30":{"score":0.3768,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2118,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":131.7},"32":{"score":0.5971,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":109.8},"33":{"score":0.5061,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":386.5},"36":{"score":0.4115,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":660},"37":{"score":0.5922,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":75},"38":{"score":0.5819,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":815.1},"39":{"score":0.9177,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":155.8},"40":{"score":0.3657,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":62.8},"41":{"score":0.7566,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":55.9},"42":{"score":0.6709,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":68.8},"43":{"score":0.3474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":56},"44":{"score":0.795,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":55},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":59.1},"50":{"score":0.6032,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":71.7},"56":{"score":0.818,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":159.4},"57":{"score":0.334,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":55.3},"58":{"score":0.7151,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":55},"59":{"score":0.7747,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":72.2},"61":{"score":0.8502,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":66.7},"62":{"score":0.8582,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":70.1},"64":{"score":0.6023,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":59.1}},"benchmarks":{"1":{"raw":0.9351,"skill":0.9123,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8034,"private_eq":0.8553},"2":{"raw":0.6401,"skill":0.5844,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6682,"private_eq":0.7151},"3":{"raw":0.7835,"skill":0.7793,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7018,"private_eq":0.7098},"4":{"raw":0.902,"skill":0.9007,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7721,"private_eq":0.7766},"5":{"raw":0.9027,"skill":0.9021,"coverage":1,"random":0.006,"tracks":[],"combined":0.6479,"private_eq":0.653},"6":{"raw":0.6852,"skill":0.2619,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6023,"skill":0.6023,"coverage":1,"random":0,"tracks":[],"combined":0.3284,"private_eq":0.335},"10":{"raw":0.6561,"skill":0.4278,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.844,"skill":0.7744,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5714,"skill":0.358,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3419,"private_eq":0.565},"20":{"raw":0.9043,"skill":0.8086,"coverage":1,"random":0.5,"tracks":[],"combined":0.7842,"private_eq":0.8928},"21":{"raw":0.6126,"skill":0.2253,"coverage":1,"random":0.5,"tracks":[],"combined":0.2565,"private_eq":0.632},"22":{"raw":0.7711,"skill":0.7693,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6579,"private_eq":0.6634},"23":{"raw":0.668,"skill":0.336,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5588,"skill":0.4117,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.301,"skill":0.068,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3132,"combined":0.0747},"26":{"raw":0.907,"skill":0.876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7816,"skill":0.7087,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7064,"skill":0.4128,"coverage":1,"random":0.5,"tracks":[],"combined":0.2393,"private_eq":0.6235},"29":{"raw":0.7857,"skill":0.7143,"coverage":1,"random":0.25,"tracks":[],"combined":0.587,"private_eq":0.693},"30":{"raw":0.3768,"skill":0.2501,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4829,"random":0.25},{"track":"10-choice","raw":0.2707,"random":0.1}],"combined":0.2788,"private_eq":0.411},"31":{"raw":0.2118,"skill":0.1415,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.5971,"skill":0.3595,"coverage":1,"random":0.371,"tracks":[],"combined":0.1037,"private_eq":0.442},"33":{"raw":0.5061,"skill":0.4995,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3228,"private_eq":0.3383},"36":{"raw":0.4115,"skill":0.3343,"coverage":1,"random":0.116,"tracks":[],"combined":0.2173,"private_eq":0.3151},"37":{"raw":0.5922,"skill":0.4885,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5145,"private_eq":0.6165},"38":{"raw":0.5819,"skill":0.5685,"coverage":1,"random":0.031,"tracks":[],"combined":0.6835,"private_eq":0.6958},"39":{"raw":0.9177,"skill":0.879,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5181,"private_eq":0.6754},"40":{"raw":0.3657,"skill":0.184,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3794,"headline":false},{"track":"A · English","score":0.3657,"headline":true},{"track":"C · Arabic pairs","score":0.895,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"combined":0.0743,"private_eq":0.2879},"41":{"raw":0.7566,"skill":0.6349,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.6709,"skill":0.3597,"coverage":1,"random":0.486,"tracks":[],"combined":0.4171,"private_eq":0.7033},"43":{"raw":0.3474,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1983,"private_eq":0.4998},"44":{"raw":0.795,"skill":0.59,"coverage":1,"random":0.5,"tracks":[],"combined":0.5139,"private_eq":0.7592},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6032,"skill":0.4241,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.818,"skill":0.636,"coverage":1,"random":0.5,"tracks":[],"combined":0.0666,"private_eq":0.5381},"57":{"raw":0.334,"skill":0.2509,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2558,"combined":0.1543},"58":{"raw":0.7151,"skill":0.587,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3286,"combined":0.0166},"59":{"raw":0.7747,"skill":0.5329,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2154,"private_eq":0.6254},"61":{"raw":0.8502,"skill":0.7004,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"62":{"raw":0.8582,"skill":0.8109,"coverage":1,"random":0.25,"tracks":[],"combined":0.5287,"private_eq":0.6498},"64":{"raw":0.6023,"skill":0.5029,"coverage":1,"random":0.2,"tracks":[],"combined":0.5647,"private_eq":0.6549}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #208).","Submitted public 0.3 run adopted (kit rescore 53.79, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.41% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":65.69,"balanced_skill":28.88,"breadth_skill":51.36,"public_skill":53.79},"frozen_scores":{"balanced_raw":65.69,"balanced_skill":53.79,"breadth_skill":51.36},"categories":[{"id":"knowledge","raw":0.4546,"skill":0.2853,"combined":0.1937,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6957,"skill":0.5553,"combined":0.3587,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7454,"skill":0.6651,"combined":0.4824,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7709,"skill":0.7458,"combined":0.617,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6936,"skill":0.511,"combined":0.4962,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"startlux-decision-v1.1-0.8b-instant","pub":53.79,"same_skills":41.29,"new_domains":6.86,"v":28.88,"lab":false,"jev":false,"rank_v":174,"rank_pub":102,"name":"StartLux-Decision v1.1 0.8B","eq":{"S":40.71,"O":11.12},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":24.43,"new_domains":4.45}},"new_in_edition":true},{"engine":"coco-decision-2.4-4b","name":"CoCo-Decision-4B 2.4","long_name":"CoCo-Decision-4B 2.4","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4205751296,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/corners-ai/CoCo-Decision-2.4-4B","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"corners-ai","name":"Corners","logo":"data/orgs/corners-ai.png"},"pr":195,"replaces":"coco-decision-4b-2.3.0","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7263,"conf":0.8401,"ece":0.1138,"brier":0.385,"over95":0.0313,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0,"conf":0.0892,"acc":0},{"lo":0.1,"w":0.0055,"conf":0.1684,"acc":0.0144},{"lo":0.2,"w":0.0164,"conf":0.2541,"acc":0.0548},{"lo":0.3,"w":0.0314,"conf":0.3525,"acc":0.2039},{"lo":0.4,"w":0.0434,"conf":0.4592,"acc":0.2872},{"lo":0.5,"w":0.0772,"conf":0.5459,"acc":0.4478},{"lo":0.6,"w":0.0638,"conf":0.6505,"acc":0.498},{"lo":0.7,"w":0.0727,"conf":0.7495,"acc":0.5106},{"lo":0.8,"w":0.094,"conf":0.8554,"acc":0.6336},{"lo":0.9,"w":0.5956,"conf":0.9831,"acc":0.9125}],"areas":{"knowledge":{"ece":0.1879,"acc":0.6137,"conf":0.8016},"language":{"ece":0.0636,"acc":0.847,"conf":0.9059},"retrieval":{"ece":0.0522,"acc":0.8029,"conf":0.8551},"tools":{"ece":0.0578,"acc":0.9394,"conf":0.9896},"arts":{"ece":0.1519,"acc":0.5451,"conf":0.697}}},"gaps":null,"latency":{"median":62.3,"p95":398.4,"mean":124.8},"results":{"1":{"score":0.9687,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":80.9},"2":{"score":0.5195,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":966.4},"3":{"score":0.8504,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":448.1},"4":{"score":0.8992,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":51},"5":{"score":0.9349,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":84.8},"6":{"score":0.7491,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":240.3},"9":{"score":0.7045,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1046.8},"10":{"score":0.7032,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":67.3},"11":{"score":0.8403,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1747.7},"12":{"score":0.6955,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":20.9},"20":{"score":0.9338,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.6},"21":{"score":0.6621,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":20.6},"22":{"score":0.013,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":361.9},"23":{"score":0.6418,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.3},"24":{"score":0.7519,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":20.7},"25":{"score":0.4337,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":21.1},"26":{"score":0.9823,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":20.4},"27":{"score":0.9437,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":20.6},"28":{"score":0.8311,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":20.4},"29":{"score":0.9441,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":24.9},"30":{"score":0.5182,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.183,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.5},"32":{"score":0.617,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":53},"33":{"score":0.3273,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":169.8},"36":{"score":0.3487,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1779.2},"37":{"score":0.5667,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.8},"38":{"score":0.3594,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":471.8},"39":{"score":0.889,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":26.4},"40":{"score":0.4943,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":20.7},"41":{"score":0.7933,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":20.9},"42":{"score":0.7673,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":29.7},"43":{"score":0.4105,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":21},"44":{"score":0.845,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.9},"45":{"score":0.1038,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":22.1},"50":{"score":0.4737,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":50.7},"56":{"score":0.574,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":119.3},"57":{"score":0.5479,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":21.1},"58":{"score":0.6563,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":20.9},"59":{"score":0.8029,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":50.1},"61":{"score":0.8788,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":28.9},"62":{"score":0.8924,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":49.4},"64":{"score":0.7064,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":20.9}},"benchmarks":{"1":{"raw":0.9687,"skill":0.9578,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.5195,"skill":0.4451,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6861,"private_eq":0.7304},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8278,"private_eq":0.832},"4":{"raw":0.8992,"skill":0.8979,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8664,"private_eq":0.8686},"5":{"raw":0.9349,"skill":0.9345,"coverage":1,"random":0.006,"tracks":[],"combined":0.6759,"private_eq":0.6805},"6":{"raw":0.7491,"skill":0.4117,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7045,"skill":0.7045,"coverage":1,"random":0,"tracks":[],"combined":0.5863,"private_eq":0.59},"10":{"raw":0.7032,"skill":0.5062,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8403,"skill":0.769,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6955,"skill":0.5439,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4171,"private_eq":0.6146},"20":{"raw":0.9319,"skill":0.8639,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6621,"skill":0.3242,"coverage":1,"random":0.5,"tracks":[],"combined":0.3069,"private_eq":0.6569},"22":{"raw":0.0102,"skill":0.0025,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0302,"private_eq":0.0477},"23":{"raw":0.6433,"skill":0.2867,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7519,"skill":0.6692,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4337,"skill":0.2449,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3599,"combined":0.1377},"26":{"raw":0.9823,"skill":0.9764,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9437,"skill":0.9249,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8311,"skill":0.6622,"coverage":1,"random":0.5,"tracks":[],"combined":0.4764,"private_eq":0.7407},"29":{"raw":0.9441,"skill":0.9255,"coverage":1,"random":0.25,"tracks":[],"combined":0.8544,"private_eq":0.8913},"30":{"raw":0.5182,"skill":0.4209,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6065,"random":0.25},{"track":"10-choice","raw":0.4299,"random":0.1}],"combined":0.5481,"private_eq":0.6306},"31":{"raw":0.183,"skill":0.1102,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0766,"private_eq":0.161},"32":{"raw":0.617,"skill":0.3911,"coverage":1,"random":0.371,"tracks":[],"combined":0.1731,"private_eq":0.4852},"33":{"raw":0.3273,"skill":0.3184,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2457,"private_eq":0.2631},"36":{"raw":0.3487,"skill":0.2632,"coverage":1,"random":0.116,"tracks":[],"combined":0.3414,"private_eq":0.4235},"37":{"raw":0.5667,"skill":0.4565,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5057,"private_eq":0.6096},"38":{"raw":0.3594,"skill":0.3389,"coverage":1,"random":0.031,"tracks":[],"combined":0.5024,"private_eq":0.5223},"39":{"raw":0.889,"skill":0.8367,"coverage":1,"random":0.3201,"tracks":[],"combined":0.54,"private_eq":0.6901},"40":{"raw":0.4943,"skill":0.3494,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4235,"headline":false},{"track":"A · English","score":0.4943,"headline":true},{"track":"C · Arabic pairs","score":0.92,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0.35,"private_eq":0.4997},"41":{"raw":0.7933,"skill":0.6899,"coverage":1,"random":0.3333,"tracks":[],"combined":0.7272,"private_eq":0.8195},"42":{"raw":0.7673,"skill":0.5473,"coverage":1,"random":0.486,"tracks":[],"combined":0.8426,"private_eq":0.9195},"43":{"raw":0.4105,"skill":0.0647,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3782,"private_eq":0.6119},"44":{"raw":0.845,"skill":0.69,"coverage":1,"random":0.5,"tracks":[],"combined":0.6583,"private_eq":0.8306},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4737,"skill":0.2362,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.574,"skill":0.148,"coverage":1,"random":0.5,"tracks":[],"combined":0.0146,"private_eq":0.5124},"57":{"raw":0.5479,"skill":0.4915,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5255,"combined":0.4612},"58":{"raw":0.6563,"skill":0.5018,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4013,"combined":0.1232},"59":{"raw":0.8029,"skill":0.5913,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2026,"private_eq":0.6193},"61":{"raw":0.8788,"skill":0.7576,"coverage":1,"random":0.5,"tracks":[],"combined":0.5924,"private_eq":0.798},"62":{"raw":0.8924,"skill":0.8565,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.7064,"skill":0.633,"coverage":1,"random":0.2,"tracks":[],"combined":0.6326,"private_eq":0.7086}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #195).","Replaces the earlier board entry coco-decision-4b-2.3.0 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 55.19, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.69% on 753 rows (bar 98.5%)."],"scores":{"balanced_raw":66.25,"balanced_skill":45.52,"breadth_skill":53.38,"public_skill":55.19},"frozen_scores":{"balanced_raw":66.25,"balanced_skill":55.19,"breadth_skill":53.38},"categories":[{"id":"knowledge","raw":0.499,"skill":0.3563,"combined":0.3257,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7355,"skill":0.6261,"combined":0.5584,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7029,"skill":0.5874,"combined":0.5111,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7962,"skill":0.7727,"combined":0.7398,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5713,"skill":0.3911,"combined":0.4073,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"coco-decision-2.4-4b","pub":55.19,"same_skills":51.67,"new_domains":27.81,"v":45.52,"lab":false,"jev":false,"rank_v":110,"rank_pub":98,"name":"CoCo-Decision-4B 2.4","eq":{"S":51.21,"O":36.99},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":30.73,"new_domains":14.8}},"new_in_edition":true},{"engine":"jegue-0.1","name":"Jegue 0.1 (9B)","long_name":"Jegue 0.1 (9B)","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/fdemelo/jegue","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"pr":221,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7018,"conf":0.8179,"ece":0.1162,"brier":0.4207,"over95":0.0172,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0012,"conf":0.0889,"acc":0.0326},{"lo":0.1,"w":0.0302,"conf":0.1626,"acc":0.0169},{"lo":0.2,"w":0.0256,"conf":0.2493,"acc":0.0958},{"lo":0.3,"w":0.0155,"conf":0.3482,"acc":0.1855},{"lo":0.4,"w":0.018,"conf":0.4572,"acc":0.2158},{"lo":0.5,"w":0.0524,"conf":0.5537,"acc":0.3396},{"lo":0.6,"w":0.0678,"conf":0.645,"acc":0.4702},{"lo":0.7,"w":0.0812,"conf":0.7631,"acc":0.561},{"lo":0.8,"w":0.1309,"conf":0.8648,"acc":0.6492},{"lo":0.9,"w":0.5772,"conf":0.9442,"acc":0.8867}],"areas":{"knowledge":{"ece":0.1489,"acc":0.5947,"conf":0.7436},"language":{"ece":0.0944,"acc":0.8299,"conf":0.924},"retrieval":{"ece":0.0741,"acc":0.7932,"conf":0.8673},"tools":{"ece":0.0432,"acc":0.9083,"conf":0.8833},"arts":{"ece":0.2087,"acc":0.4882,"conf":0.6969}}},"gaps":null,"latency":{"median":45.7,"p95":398.8,"mean":129},"results":{"1":{"score":0.9534,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":134.1},"2":{"score":0.655,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1529.8},"3":{"score":0.7815,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":519.6},"4":{"score":0.8742,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":95.9},"5":{"score":0.9353,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":164.4},"6":{"score":0.7895,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":240.9},"9":{"score":0.4091,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":627.8},"10":{"score":0.6983,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":97.2},"11":{"score":0.8137,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":769.1},"12":{"score":0.6608,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":33.5},"20":{"score":0.7786,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.7},"21":{"score":0.6545,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":24.9},"22":{"score":0.0185,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":591.1},"23":{"score":0.5596,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":56.2},"24":{"score":0.7621,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":30.6},"25":{"score":0.3878,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":33.6},"26":{"score":0.9819,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":25.1},"27":{"score":0.9411,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":25.7},"28":{"score":0.8248,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":22.6},"29":{"score":0.9421,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":46},"30":{"score":0.6035,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1198,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":97.8},"32":{"score":0.5811,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":97.1},"33":{"score":0.3248,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":306.5},"36":{"score":0.4513,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1847.7},"37":{"score":0.5635,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.6},"38":{"score":0.5214,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2128.1},"39":{"score":0.9428,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":66.9},"40":{"score":0.5781,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":23.1},"41":{"score":0.5619,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":33.2},"42":{"score":0.7882,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":57.6},"43":{"score":0.5386,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":33.1},"44":{"score":0.6536,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.5},"45":{"score":0.0699,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":45.7},"50":{"score":0.466,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":83.6},"56":{"score":0.97,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":263.9},"57":{"score":0.5533,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":33.8},"58":{"score":0.671,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":33.6},"59":{"score":0.8268,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":82.2},"61":{"score":0.653,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":52.1},"62":{"score":0.8776,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":72.7},"64":{"score":0.6553,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":33.3}},"benchmarks":{"1":{"raw":0.9534,"skill":0.937,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9056,"private_eq":0.9301},"2":{"raw":0.655,"skill":0.6016,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6667,"private_eq":0.7138},"3":{"raw":0.7815,"skill":0.7773,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8331,"private_eq":0.8371},"4":{"raw":0.8742,"skill":0.8726,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8117,"private_eq":0.8152},"5":{"raw":0.9353,"skill":0.9349,"coverage":1,"random":0.006,"tracks":[],"combined":0.6828,"private_eq":0.6873},"6":{"raw":0.7895,"skill":0.5064,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4091,"skill":0.4091,"coverage":1,"random":0,"tracks":[],"combined":0.255,"private_eq":0.2625},"10":{"raw":0.6983,"skill":0.498,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8137,"skill":0.7305,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6608,"skill":0.4919,"coverage":1,"random":0.3324,"tracks":[],"combined":0.444,"private_eq":0.6324},"20":{"raw":0.7795,"skill":0.559,"coverage":1,"random":0.5,"tracks":[],"combined":0.7769,"private_eq":0.8892},"21":{"raw":0.6545,"skill":0.309,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.0168,"skill":0.0092,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5584,"skill":0.1168,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7621,"skill":0.6828,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3412,"combined":0.1125},"26":{"raw":0.9819,"skill":0.9759,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9411,"skill":0.9214,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8248,"skill":0.6496,"coverage":1,"random":0.5,"tracks":[],"combined":0.4461,"private_eq":0.7257},"29":{"raw":0.9421,"skill":0.9228,"coverage":1,"random":0.25,"tracks":[],"combined":0.9384,"private_eq":0.9536},"30":{"raw":0.6035,"skill":0.5248,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6884,"random":0.25},{"track":"10-choice","raw":0.5186,"random":0.1}],"combined":0.5542,"private_eq":0.6356},"31":{"raw":0.1198,"skill":0.0413,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0431,"private_eq":0.1306},"32":{"raw":0.5811,"skill":0.334,"coverage":1,"random":0.371,"tracks":[],"combined":0.0649,"private_eq":0.4179},"33":{"raw":0.3248,"skill":0.3158,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2251,"private_eq":0.243},"36":{"raw":0.4513,"skill":0.3793,"coverage":1,"random":0.116,"tracks":[],"combined":0.346,"private_eq":0.4276},"37":{"raw":0.5635,"skill":0.4526,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4843,"private_eq":0.5927},"38":{"raw":0.5214,"skill":0.5061,"coverage":1,"random":0.031,"tracks":[],"combined":0.6764,"private_eq":0.689},"39":{"raw":0.9428,"skill":0.9159,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4779,"private_eq":0.6484},"40":{"raw":0.5781,"skill":0.4573,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2362,"headline":false},{"track":"A · English","score":0.5781,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.9,"headline":false}],"combined":0.4146,"private_eq":0.5494},"41":{"raw":0.5619,"skill":0.3429,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4224,"private_eq":0.6186},"42":{"raw":0.7882,"skill":0.5879,"coverage":1,"random":0.486,"tracks":[],"combined":0.5836,"private_eq":0.7879},"43":{"raw":0.5386,"skill":0.2679,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4616,"private_eq":0.6639},"44":{"raw":0.6536,"skill":0.3072,"coverage":1,"random":0.5,"tracks":[],"combined":0.4174,"private_eq":0.7115},"45":{"raw":0.0699,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.466,"skill":0.225,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.97,"skill":0.94,"coverage":1,"random":0.5,"tracks":[],"combined":0.5337,"private_eq":0.769},"57":{"raw":0.5533,"skill":0.4976,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3789,"combined":0.2944},"58":{"raw":0.671,"skill":0.5231,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3966,"combined":0.1163},"59":{"raw":0.8268,"skill":0.6409,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2777,"private_eq":0.6551},"61":{"raw":0.653,"skill":0.306,"coverage":1,"random":0.5,"tracks":[],"combined":0.3712,"private_eq":0.6887},"62":{"raw":0.8776,"skill":0.8368,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.6553,"skill":0.5691,"coverage":1,"random":0.2,"tracks":[],"combined":0.5259,"private_eq":0.6242}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #221).","Submitted public 0.3 run adopted (kit rescore 54.13). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.75% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":65.39,"balanced_skill":44.16,"breadth_skill":51.78,"public_skill":54.13},"frozen_scores":{"balanced_raw":65.39,"balanced_skill":54.13,"breadth_skill":51.78},"categories":[{"id":"knowledge","raw":0.4858,"skill":0.3347,"combined":0.2712,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.74,"skill":0.6254,"combined":0.5371,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7423,"skill":0.655,"combined":0.5451,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7451,"skill":0.7231,"combined":0.6689,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5217,"skill":0.298,"combined":0.3659,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jegue-0.1","pub":54.13,"same_skills":48.78,"new_domains":28.6,"v":44.16,"lab":false,"jev":false,"rank_v":116,"rank_pub":101,"name":"Jegue 0.1 (9B)","eq":{"S":48.29,"O":37.96},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":28.97,"new_domains":15.19}},"new_in_edition":true},{"engine":"nari-decision-4b-v3.2","name":"Nari Decision 4B v3.2","long_name":"Nari Decision 4B v3.2","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4216792578,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/blackmallangcow/nari-decision-4b","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"blackmallangcow","name":"blackmallangcow","logo":"data/orgs/blackmallangcow.png"},"pr":216,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6539,"conf":0.7144,"ece":0.0689,"brier":0.4623,"over95":0.0089,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0257,"conf":0.0846,"acc":0.0237},{"lo":0.1,"w":0.0433,"conf":0.1345,"acc":0.1306},{"lo":0.2,"w":0.0164,"conf":0.2527,"acc":0.3843},{"lo":0.3,"w":0.0303,"conf":0.3563,"acc":0.4241},{"lo":0.4,"w":0.0574,"conf":0.4544,"acc":0.36},{"lo":0.5,"w":0.1138,"conf":0.5483,"acc":0.4761},{"lo":0.6,"w":0.1094,"conf":0.6529,"acc":0.5651},{"lo":0.7,"w":0.122,"conf":0.7515,"acc":0.6574},{"lo":0.8,"w":0.1835,"conf":0.8527,"acc":0.7647},{"lo":0.9,"w":0.2983,"conf":0.9504,"acc":0.9097}],"areas":{"knowledge":{"ece":0.0909,"acc":0.6016,"conf":0.6862},"language":{"ece":0.0415,"acc":0.7587,"conf":0.8},"retrieval":{"ece":0.0939,"acc":0.6419,"conf":0.7177},"tools":{"ece":0.0817,"acc":0.8112,"conf":0.7441},"arts":{"ece":0.112,"acc":0.4851,"conf":0.5963}}},"gaps":null,"latency":{"median":23.1,"p95":124.8,"mean":65.4},"results":{"1":{"score":0.9244,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":79.9},"2":{"score":0.6818,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":597.6},"3":{"score":0.5039,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":670.4},"4":{"score":0.8187,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.2},"5":{"score":0.8968,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":56.1},"6":{"score":0.7902,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":193.8},"9":{"score":0.4205,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":208.5},"10":{"score":0.1746,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":128.7},"11":{"score":0.6615,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":252.3},"12":{"score":0.5817,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.2},"20":{"score":0.7722,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.4},"21":{"score":0.5963,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":18.2},"22":{"score":0.012,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":368},"23":{"score":0.6312,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":55.2},"24":{"score":0.7239,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":19.8},"25":{"score":0.4592,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":25.6},"26":{"score":0.9701,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.3},"27":{"score":0.9181,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.4},"28":{"score":0.8074,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":18.2},"29":{"score":0.7426,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":28.3},"30":{"score":0.3889,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1066,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":55.2},"32":{"score":0.5864,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":80.1},"33":{"score":0.2461,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":80.5},"36":{"score":0.3194,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":836.1},"37":{"score":0.5217,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.2},"38":{"score":0.2233,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":556},"39":{"score":0.8257,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":28.8},"40":{"score":0.4572,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":18.2},"41":{"score":0.4913,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":24.6},"42":{"score":0.7631,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":47.1},"43":{"score":0.4982,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":24.4},"44":{"score":0.6178,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.3},"45":{"score":0.1038,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":27.9},"50":{"score":0.3962,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":71.6},"56":{"score":0.739,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":75.6},"57":{"score":0.4924,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":25.6},"58":{"score":0.6268,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":24.4},"59":{"score":0.717,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":67.7},"61":{"score":0.8695,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":44.3},"62":{"score":0.7229,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":79.6},"64":{"score":0.5436,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":25.6}},"benchmarks":{"1":{"raw":0.9244,"skill":0.898,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7013,"private_eq":0.7805},"2":{"raw":0.6818,"skill":0.6325,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6876,"private_eq":0.7317},"3":{"raw":0.5039,"skill":0.4943,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6809,"private_eq":0.6895},"4":{"raw":0.8187,"skill":0.8164,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7268,"private_eq":0.7323},"5":{"raw":0.8968,"skill":0.8962,"coverage":1,"random":0.006,"tracks":[],"combined":0.6106,"private_eq":0.6163},"6":{"raw":0.7902,"skill":0.5081,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4205,"skill":0.4205,"coverage":1,"random":0,"tracks":[],"combined":0.3208,"private_eq":0.3275},"10":{"raw":0.1746,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6615,"skill":0.5105,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5817,"skill":0.3734,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3556,"private_eq":0.574},"20":{"raw":0.7742,"skill":0.5485,"coverage":1,"random":0.5,"tracks":[],"combined":0.7046,"private_eq":0.8535},"21":{"raw":0.5963,"skill":0.1925,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.0115,"skill":0.0038,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0094},"23":{"raw":0.6128,"skill":0.2255,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7239,"skill":0.6319,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4592,"skill":0.2789,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3402,"combined":0.1111},"26":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9181,"skill":0.8908,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8074,"skill":0.6148,"coverage":1,"random":0.5,"tracks":[],"combined":0.3755,"private_eq":0.6908},"29":{"raw":0.7426,"skill":0.6568,"coverage":1,"random":0.25,"tracks":[],"combined":0.592,"private_eq":0.6967},"30":{"raw":0.3889,"skill":0.2671,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5148,"random":0.25},{"track":"10-choice","raw":0.2631,"random":0.1}],"combined":0.4711,"private_eq":0.5678},"31":{"raw":0.1066,"skill":0.027,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0539,"private_eq":0.1404},"32":{"raw":0.5864,"skill":0.3424,"coverage":1,"random":0.371,"tracks":[],"combined":0.1037,"private_eq":0.442},"33":{"raw":0.2461,"skill":0.2361,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1071,"private_eq":0.1278},"36":{"raw":0.3194,"skill":0.2301,"coverage":1,"random":0.116,"tracks":[],"combined":0.2753,"private_eq":0.3658},"37":{"raw":0.5217,"skill":0.4001,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5296,"private_eq":0.6284},"38":{"raw":0.2233,"skill":0.1985,"coverage":1,"random":0.031,"tracks":[],"combined":0.2548,"private_eq":0.2852},"39":{"raw":0.8257,"skill":0.7436,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3793,"private_eq":0.5821},"40":{"raw":0.4572,"skill":0.3018,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.294,"headline":false},{"track":"A · English","score":0.4572,"headline":true},{"track":"C · Arabic pairs","score":0.56,"headline":false},{"track":"C · English pairs","score":0.785,"headline":false}],"combined":0.3302,"private_eq":0.4845},"41":{"raw":0.4913,"skill":0.237,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3457,"private_eq":0.5681},"42":{"raw":0.7631,"skill":0.5391,"coverage":1,"random":0.486,"tracks":[],"combined":0.6281,"private_eq":0.8105},"43":{"raw":0.4982,"skill":0.2039,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4074,"private_eq":0.6301},"44":{"raw":0.6178,"skill":0.2356,"coverage":1,"random":0.5,"tracks":[],"combined":0.3907,"private_eq":0.6983},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3962,"skill":0.1237,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.739,"skill":0.478,"coverage":1,"random":0.5,"tracks":[],"combined":0.6462,"private_eq":0.8246},"57":{"raw":0.4924,"skill":0.4291,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4229,"combined":0.3445},"58":{"raw":0.6268,"skill":0.4591,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4024,"combined":0.1248},"59":{"raw":0.717,"skill":0.4132,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2181,"private_eq":0.6267},"61":{"raw":0.8695,"skill":0.739,"coverage":1,"random":0.5,"tracks":[],"combined":0.5756,"private_eq":0.7897},"62":{"raw":0.7229,"skill":0.6305,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.5436,"skill":0.4295,"coverage":1,"random":0.2,"tracks":[],"combined":0.4677,"private_eq":0.5782}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #216).","Submitted public 0.3 run adopted (kit rescore 44.80, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.55% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":58.19,"balanced_skill":40.87,"breadth_skill":42.9,"public_skill":44.8},"frozen_scores":{"balanced_raw":58.19,"balanced_skill":44.8,"breadth_skill":42.9},"categories":[{"id":"knowledge","raw":0.4526,"skill":0.2875,"combined":0.2482,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6094,"skill":0.4447,"combined":0.3921,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6927,"skill":0.5982,"combined":0.5586,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6554,"skill":0.6212,"combined":0.6057,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4891,"skill":0.2539,"combined":0.3296,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"nari-decision-4b-v3.2","pub":44.8,"same_skills":43.38,"new_domains":28.58,"v":40.87,"lab":false,"jev":false,"rank_v":130,"rank_pub":136,"name":"Nari Decision 4B v3.2","eq":{"S":42.83,"O":37.94},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.7,"new_domains":15.17}},"new_in_edition":true},{"engine":"latentmesh-blitz-2b-v1","name":"LatentMesh Blitz 2B v1","long_name":"LatentMesh Blitz 2B v1","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/LatentMesh-Labs/latentmesh-blitz-2b-v1","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"org":{"id":"LatentMesh-Labs","name":"LatentMesh Labs","logo":"data/orgs/LatentMesh-Labs.png"},"pr":196,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6926,"conf":0.8271,"ece":0.137,"brier":0.4416,"over95":0.0484,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0046,"conf":0.0823,"acc":0.1028},{"lo":0.1,"w":0.0114,"conf":0.1497,"acc":0.1747},{"lo":0.2,"w":0.0133,"conf":0.2632,"acc":0.329},{"lo":0.3,"w":0.0333,"conf":0.3556,"acc":0.2768},{"lo":0.4,"w":0.0404,"conf":0.4487,"acc":0.328},{"lo":0.5,"w":0.0725,"conf":0.5484,"acc":0.4322},{"lo":0.6,"w":0.0702,"conf":0.6509,"acc":0.4782},{"lo":0.7,"w":0.0812,"conf":0.7541,"acc":0.554},{"lo":0.8,"w":0.113,"conf":0.8549,"acc":0.5903},{"lo":0.9,"w":0.5602,"conf":0.9789,"acc":0.8689}],"areas":{"knowledge":{"ece":0.1735,"acc":0.5498,"conf":0.7162},"language":{"ece":0.0978,"acc":0.7998,"conf":0.8974},"retrieval":{"ece":0.1229,"acc":0.7642,"conf":0.8871},"tools":{"ece":0.1136,"acc":0.8627,"conf":0.9762},"arts":{"ece":0.1609,"acc":0.626,"conf":0.785}}},"gaps":null,"latency":{"median":51.5,"p95":168.3,"mean":88},"results":{"1":{"score":0.9109,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":53.3},"2":{"score":0.6295,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":685.2},"3":{"score":0.7343,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":920.9},"4":{"score":0.8747,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":64.5},"5":{"score":0.921,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":98.1},"6":{"score":0.7821,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":178.8},"9":{"score":0.3636,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":418.2},"10":{"score":0.579,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":120.8},"11":{"score":0.826,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":913.8},"12":{"score":0.5625,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":12.6},"20":{"score":0.9074,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":9.2},"21":{"score":0.6195,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":9.5},"22":{"score":0.677,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":802.7},"23":{"score":0.5946,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.2},"24":{"score":0.5749,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13},"25":{"score":0.3214,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":13.6},"26":{"score":0.9175,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6.1},"27":{"score":0.802,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":12.8},"28":{"score":0.6827,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":5.1},"29":{"score":0.844,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":17.2},"30":{"score":0.4052,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1574,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.7},"32":{"score":0.5519,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":72.1},"33":{"score":0.2685,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":73.2},"36":{"score":0.3537,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1130.4},"37":{"score":0.5144,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23},"38":{"score":0.3919,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":468.5},"39":{"score":0.9218,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":17.5},"40":{"score":0.3489,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":4.8},"41":{"score":0.7401,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":14.8},"42":{"score":0.689,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":33.5},"43":{"score":0.3386,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":10.8},"44":{"score":0.7812,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.8},"45":{"score":0.1577,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":16.6},"50":{"score":0.3622,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":65.4},"56":{"score":0.7115,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":79.2},"57":{"score":0.3141,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":14.5},"58":{"score":0.5228,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":14},"59":{"score":0.7787,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":53.3},"61":{"score":0.848,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":33.5},"62":{"score":0.7941,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":47},"64":{"score":0.6212,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":13.3}},"benchmarks":{"1":{"raw":0.9109,"skill":0.8797,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8444,"private_eq":0.8853},"2":{"raw":0.6295,"skill":0.5721,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6732,"private_eq":0.7194},"3":{"raw":0.7343,"skill":0.7292,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7123,"private_eq":0.72},"4":{"raw":0.8747,"skill":0.8731,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7906,"private_eq":0.7946},"5":{"raw":0.921,"skill":0.9205,"coverage":1,"random":0.006,"tracks":[],"combined":0.7062,"private_eq":0.7103},"6":{"raw":0.7821,"skill":0.4891,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3636,"skill":0.3636,"coverage":1,"random":0,"tracks":[],"combined":0.3031,"private_eq":0.31},"10":{"raw":0.579,"skill":0.2995,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.826,"skill":0.7484,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5625,"skill":0.3447,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3778,"private_eq":0.5887},"20":{"raw":0.9036,"skill":0.8071,"coverage":1,"random":0.5,"tracks":[],"combined":0.8637,"private_eq":0.9321},"21":{"raw":0.6195,"skill":0.239,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.6786,"skill":0.6761,"coverage":1,"random":0.0078,"tracks":[],"combined":0.637,"private_eq":0.6429},"23":{"raw":0.5915,"skill":0.1831,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5749,"skill":0.4332,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3183,"combined":0.0816},"26":{"raw":0.9175,"skill":0.89,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.802,"skill":0.7359,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6827,"skill":0.3654,"coverage":1,"random":0.5,"tracks":[],"combined":0.6429,"private_eq":0.823},"29":{"raw":0.844,"skill":0.792,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"30":{"raw":0.4052,"skill":0.2862,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5239,"random":0.25},{"track":"10-choice","raw":0.2866,"random":0.1}],"combined":0.3598,"private_eq":0.477},"31":{"raw":0.1574,"skill":0.0823,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0983,"private_eq":0.1807},"32":{"raw":0.5519,"skill":0.2876,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.2685,"skill":0.2588,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2714,"private_eq":0.2881},"36":{"raw":0.3537,"skill":0.2689,"coverage":1,"random":0.116,"tracks":[],"combined":0.214,"private_eq":0.3122},"37":{"raw":0.5144,"skill":0.3909,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5222,"private_eq":0.6226},"38":{"raw":0.3919,"skill":0.3724,"coverage":1,"random":0.031,"tracks":[],"combined":0.5417,"private_eq":0.56},"39":{"raw":0.9218,"skill":0.885,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4055,"private_eq":0.5997},"40":{"raw":0.3489,"skill":0.1624,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3762,"headline":false},{"track":"A · English","score":0.3489,"headline":true},{"track":"C · Arabic pairs","score":0.835,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"combined":0.2319,"private_eq":0.409},"41":{"raw":0.7401,"skill":0.6102,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2365,"private_eq":0.4961},"42":{"raw":0.689,"skill":0.3949,"coverage":1,"random":0.486,"tracks":[],"combined":0.4579,"private_eq":0.724},"43":{"raw":0.3386,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2652,"private_eq":0.5415},"44":{"raw":0.7812,"skill":0.5624,"coverage":1,"random":0.5,"tracks":[],"combined":0.5994,"private_eq":0.8015},"45":{"raw":0.1577,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3622,"skill":0.0743,"coverage":1,"random":0.311,"tracks":[],"combined":0.1306,"private_eq":0.4071},"56":{"raw":0.7115,"skill":0.423,"coverage":1,"random":0.5,"tracks":[],"combined":0.1962,"private_eq":0.6022},"57":{"raw":0.3141,"skill":0.2285,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3789,"combined":0.2944},"58":{"raw":0.5228,"skill":0.3083,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3437,"combined":0.0388},"59":{"raw":0.7787,"skill":0.5412,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1694,"private_eq":0.6035},"61":{"raw":0.848,"skill":0.696,"coverage":1,"random":0.5,"tracks":[],"combined":0.5699,"private_eq":0.7869},"62":{"raw":0.7941,"skill":0.7255,"coverage":1,"random":0.25,"tracks":[],"combined":0.5493,"private_eq":0.6651},"64":{"raw":0.6212,"skill":0.5265,"coverage":1,"random":0.2,"tracks":[],"combined":0.5452,"private_eq":0.6395}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #196).","Submitted public 0.3 run adopted (kit rescore 48.15, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.48% on 827 rows (bar 98.5%)."],"scores":{"balanced_raw":61.26,"balanced_skill":34.44,"breadth_skill":45.36,"public_skill":48.15},"frozen_scores":{"balanced_raw":61.26,"balanced_skill":48.15,"breadth_skill":45.36},"categories":[{"id":"knowledge","raw":0.4084,"skill":0.2228,"combined":0.2654,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6792,"skill":0.5403,"combined":0.3747,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.705,"skill":0.6038,"combined":0.5063,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6966,"skill":0.6652,"combined":0.6284,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6294,"skill":0.4177,"combined":0.5007,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"latentmesh-blitz-2b-v1","pub":48.15,"same_skills":44.23,"new_domains":14.52,"v":34.44,"lab":false,"jev":false,"rank_v":151,"rank_pub":127,"name":"LatentMesh Blitz 2B v1","eq":{"S":43.68,"O":20.57},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":26.21,"new_domains":8.23}},"new_in_edition":true},{"engine":"sieve-2b-v3","name":"Sieve 2B v3","long_name":"Sieve 2B v3","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/Sieve-2B-v3","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"},"pr":191,"replaces":"sieve-2b-plus","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7128,"conf":0.7832,"ece":0.0723,"brier":0.3897,"over95":0.0341,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0044,"conf":0.0859,"acc":0.1812},{"lo":0.1,"w":0.0212,"conf":0.1485,"acc":0.1271},{"lo":0.2,"w":0.0186,"conf":0.2593,"acc":0.1884},{"lo":0.3,"w":0.0489,"conf":0.3552,"acc":0.3668},{"lo":0.4,"w":0.0568,"conf":0.4496,"acc":0.3788},{"lo":0.5,"w":0.0961,"conf":0.5488,"acc":0.5173},{"lo":0.6,"w":0.0849,"conf":0.6496,"acc":0.5733},{"lo":0.7,"w":0.0875,"conf":0.7509,"acc":0.6504},{"lo":0.8,"w":0.0964,"conf":0.8529,"acc":0.7369},{"lo":0.9,"w":0.4852,"conf":0.9812,"acc":0.907}],"areas":{"knowledge":{"ece":0.0779,"acc":0.5972,"conf":0.6725},"language":{"ece":0.0626,"acc":0.7921,"conf":0.8543},"retrieval":{"ece":0.1347,"acc":0.6843,"conf":0.8154},"tools":{"ece":0.0322,"acc":0.9341,"conf":0.9611},"arts":{"ece":0.0634,"acc":0.6915,"conf":0.7439}}},"gaps":null,"latency":{"median":43,"p95":97.5,"mean":85.5},"results":{"1":{"score":0.8843,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":188.9},"2":{"score":0.6301,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1589.1},"3":{"score":0.8661,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":497.2},"4":{"score":0.8811,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":85.2},"5":{"score":0.8178,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":121.5},"6":{"score":0.6687,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":275.5},"9":{"score":0.9659,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":478.9},"10":{"score":0.4355,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":92},"11":{"score":0.8091,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":501.5},"12":{"score":0.5065,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":54.7},"20":{"score":0.9188,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.9},"21":{"score":0.5852,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":53.3},"22":{"score":0.7925,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":510.6},"23":{"score":0.602,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":74.3},"24":{"score":0.5585,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":54},"25":{"score":0.2704,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":54.8},"26":{"score":0.9137,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":53.6},"27":{"score":0.7884,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":53.7},"28":{"score":0.7751,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":54.1},"29":{"score":0.8734,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":54.6},"30":{"score":0.3897,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2246,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":70.2},"32":{"score":0.742,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":84},"33":{"score":0.2873,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":249.1},"36":{"score":0.3526,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2771.6},"37":{"score":0.557,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.2},"38":{"score":0.4395,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":734.2},"39":{"score":0.8478,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":149.4},"40":{"score":0.2865,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":55.6},"41":{"score":0.7257,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":54.7},"42":{"score":0.691,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":61.7},"43":{"score":0.4842,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":54.2},"44":{"score":0.8844,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":54.9},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":56.5},"50":{"score":0.5919,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":79.8},"56":{"score":0.964,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":201.3},"57":{"score":0.2975,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":55.1},"58":{"score":0.6076,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":54.3},"59":{"score":0.6789,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":72.8},"61":{"score":0.829,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":60.9},"62":{"score":0.8198,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":71},"64":{"score":0.6004,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":55}},"benchmarks":{"1":{"raw":0.8843,"skill":0.8438,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7626,"private_eq":0.8254},"2":{"raw":0.6301,"skill":0.5728,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6702,"private_eq":0.7168},"3":{"raw":0.8661,"skill":0.8635,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7963,"private_eq":0.8014},"4":{"raw":0.8811,"skill":0.8796,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7782,"private_eq":0.7825},"5":{"raw":0.8178,"skill":0.8167,"coverage":1,"random":0.006,"tracks":[],"combined":0.5724,"private_eq":0.5788},"6":{"raw":0.6687,"skill":0.2232,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9659,"skill":0.9659,"coverage":1,"random":0,"tracks":[],"combined":0.5585,"private_eq":0.5625},"10":{"raw":0.4355,"skill":0.0607,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8091,"skill":0.7239,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5065,"skill":0.2608,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4753,"private_eq":0.653},"20":{"raw":0.9142,"skill":0.8284,"coverage":1,"random":0.5,"tracks":[],"combined":0.8637,"private_eq":0.9321},"21":{"raw":0.5852,"skill":0.1705,"coverage":1,"random":0.5,"tracks":[],"combined":0.0957,"private_eq":0.5525},"22":{"raw":0.7915,"skill":0.7899,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6254,"private_eq":0.6316},"23":{"raw":0.5806,"skill":0.1613,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5585,"skill":0.4113,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2704,"skill":0.0272,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2872,"combined":0.0397},"26":{"raw":0.9137,"skill":0.8849,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7884,"skill":0.7178,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7751,"skill":0.5502,"coverage":1,"random":0.5,"tracks":[],"combined":0.3553,"private_eq":0.6808},"29":{"raw":0.8734,"skill":0.8312,"coverage":1,"random":0.25,"tracks":[],"combined":0.592,"private_eq":0.6967},"30":{"raw":0.3897,"skill":0.2657,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4943,"random":0.25},{"track":"10-choice","raw":0.2851,"random":0.1}],"combined":0.3593,"private_eq":0.4766},"31":{"raw":0.2246,"skill":0.1555,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0886,"private_eq":0.1719},"32":{"raw":0.742,"skill":0.5898,"coverage":1,"random":0.371,"tracks":[],"combined":0.0958,"private_eq":0.4371},"33":{"raw":0.2873,"skill":0.2778,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3279,"private_eq":0.3433},"36":{"raw":0.3526,"skill":0.2676,"coverage":1,"random":0.116,"tracks":[],"combined":0.2402,"private_eq":0.3351},"37":{"raw":0.557,"skill":0.4443,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5546,"private_eq":0.6481},"38":{"raw":0.4395,"skill":0.4216,"coverage":1,"random":0.031,"tracks":[],"combined":0.5257,"private_eq":0.5447},"39":{"raw":0.8478,"skill":0.7761,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5483,"private_eq":0.6957},"40":{"raw":0.2865,"skill":0.0822,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4077,"headline":false},{"track":"A · English","score":0.2865,"headline":true},{"track":"C · Arabic pairs","score":0.925,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"combined":0.1165,"private_eq":0.3203},"41":{"raw":0.7257,"skill":0.5885,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4624,"private_eq":0.645},"42":{"raw":0.691,"skill":0.3988,"coverage":1,"random":0.486,"tracks":[],"combined":0.5966,"private_eq":0.7945},"43":{"raw":0.4842,"skill":0.1817,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2358,"private_eq":0.5232},"44":{"raw":0.8844,"skill":0.7688,"coverage":1,"random":0.5,"tracks":[],"combined":0.7441,"private_eq":0.873},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5919,"skill":0.4077,"coverage":1,"random":0.311,"tracks":[],"combined":0.1814,"private_eq":0.4417},"56":{"raw":0.964,"skill":0.928,"coverage":1,"random":0.5,"tracks":[],"combined":0.3433,"private_eq":0.6749},"57":{"raw":0.2975,"skill":0.2099,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2911,"combined":0.1945},"58":{"raw":0.6076,"skill":0.4312,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3367,"combined":0.0285},"59":{"raw":0.6789,"skill":0.3342,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2598,"private_eq":0.6466},"61":{"raw":0.829,"skill":0.658,"coverage":1,"random":0.5,"tracks":[],"combined":0.5131,"private_eq":0.7588},"62":{"raw":0.8198,"skill":0.7597,"coverage":1,"random":0.25,"tracks":[],"combined":0.5013,"private_eq":0.6295},"64":{"raw":0.6004,"skill":0.5005,"coverage":1,"random":0.2,"tracks":[],"combined":0.5355,"private_eq":0.6318}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #191).","Replaces the earlier board entry sieve-2b-plus (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 53.51, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.90% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":65.39,"balanced_skill":34.45,"breadth_skill":51.09,"public_skill":53.51},"frozen_scores":{"balanced_raw":65.39,"balanced_skill":53.51,"breadth_skill":51.09},"categories":[{"id":"knowledge","raw":0.4588,"skill":0.3085,"combined":0.2401,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6526,"skill":0.4932,"combined":0.4512,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7291,"skill":0.6647,"combined":0.503,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8363,"skill":0.805,"combined":0.6668,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6773,"skill":0.4764,"combined":0.4603,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-2b-v3","pub":53.51,"same_skills":45.8,"new_domains":12.61,"v":34.45,"lab":false,"jev":false,"rank_v":150,"rank_pub":104,"name":"Sieve 2B v3","eq":{"S":45.27,"O":18.22},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":27.16,"new_domains":7.29}},"new_in_edition":true},{"engine":"vera-core-2","name":"vera-core-2","long_name":"vera-core-2","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":1883395910,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/scar-ai/vera-core-2","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"pr":223,"replaces":"vera-core","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6439,"conf":0.6877,"ece":0.0461,"brier":0.4556,"over95":0.009,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0567,"conf":0.0453,"acc":0.0624},{"lo":0.1,"w":0.0096,"conf":0.1409,"acc":0.1023},{"lo":0.2,"w":0.0147,"conf":0.2633,"acc":0.2773},{"lo":0.3,"w":0.052,"conf":0.354,"acc":0.2931},{"lo":0.4,"w":0.0689,"conf":0.4518,"acc":0.4085},{"lo":0.5,"w":0.1497,"conf":0.5474,"acc":0.5373},{"lo":0.6,"w":0.1204,"conf":0.6495,"acc":0.5992},{"lo":0.7,"w":0.1167,"conf":0.7517,"acc":0.6779},{"lo":0.8,"w":0.1307,"conf":0.8521,"acc":0.7747},{"lo":0.9,"w":0.2807,"conf":0.9661,"acc":0.923}],"areas":{"knowledge":{"ece":0.0935,"acc":0.5553,"conf":0.6453},"language":{"ece":0.0372,"acc":0.7406,"conf":0.7684},"retrieval":{"ece":0.0482,"acc":0.725,"conf":0.743},"tools":{"ece":0.0309,"acc":0.8197,"conf":0.8411},"arts":{"ece":0.0339,"acc":0.4768,"conf":0.4909}}},"gaps":null,"latency":{"median":89.9,"p95":169.8,"mean":108.7},"results":{"1":{"score":0.9209,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":79.9},"2":{"score":0.6411,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":358.6},"3":{"score":0.6142,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":172.6},"4":{"score":0.8762,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":78.7},"5":{"score":0.8685,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":79.8},"6":{"score":0.7547,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":118.7},"9":{"score":0.1932,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":347.8},"10":{"score":0.5806,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":78.8},"11":{"score":0.6587,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":557.6},"12":{"score":0.5334,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":76.9},"20":{"score":0.7224,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":76.9},"21":{"score":0.6263,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":76.9},"22":{"score":0.03,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":156.5},"23":{"score":0.6008,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77.9},"24":{"score":0.64,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":76.9},"25":{"score":0.3673,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":76.9},"26":{"score":0.952,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":76.9},"27":{"score":0.8695,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":76.9},"28":{"score":0.689,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":76.6},"29":{"score":0.8401,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":76.9},"30":{"score":0.4431,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1032,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77.9},"32":{"score":0.6011,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":78.9},"33":{"score":0.2721,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":106.9},"36":{"score":0.4302,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":591},"37":{"score":0.4275,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77.7},"38":{"score":0.1779,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":266.5},"39":{"score":0.8714,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":77.8},"40":{"score":0.4067,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":76.9},"41":{"score":0.5397,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":76.9},"42":{"score":0.7045,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":77.9},"43":{"score":0.3772,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":76.9},"44":{"score":0.5892,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":76.9},"45":{"score":0.1257,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":76.9},"50":{"score":0.4248,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":78.7},"56":{"score":0.5735,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":90.9},"57":{"score":0.4067,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":76.9},"58":{"score":0.5112,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":76.9},"59":{"score":0.7002,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":78.9},"61":{"score":0.803,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":77.9},"62":{"score":0.807,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":78.8},"64":{"score":0.5095,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":76.9}},"benchmarks":{"1":{"raw":0.9209,"skill":0.8932,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8852,"private_eq":0.9152},"2":{"raw":0.6411,"skill":0.5855,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6707,"private_eq":0.7172},"3":{"raw":0.6142,"skill":0.6068,"coverage":1,"random":0.0189,"tracks":[],"combined":0.4499,"private_eq":0.4655},"4":{"raw":0.8762,"skill":0.8746,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7665,"private_eq":0.7711},"5":{"raw":0.8685,"skill":0.8677,"coverage":1,"random":0.006,"tracks":[],"combined":0.7137,"private_eq":0.7177},"6":{"raw":0.7547,"skill":0.4249,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1932,"skill":0.1932,"coverage":1,"random":0,"tracks":[],"combined":0.1362,"private_eq":0.145},"10":{"raw":0.5806,"skill":0.3022,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6587,"skill":0.5064,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5334,"skill":0.3011,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4689,"private_eq":0.6488},"20":{"raw":0.7295,"skill":0.4591,"coverage":1,"random":0.5,"tracks":[],"combined":0.5529,"private_eq":0.7785},"21":{"raw":0.6263,"skill":0.2527,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.0265,"skill":0.0189,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0097},"23":{"raw":0.5766,"skill":0.1533,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.64,"skill":0.52,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3673,"skill":0.1565,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.308,"combined":0.0677},"26":{"raw":0.952,"skill":0.936,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8695,"skill":0.826,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.689,"skill":0.378,"coverage":1,"random":0.5,"tracks":[],"combined":0.3755,"private_eq":0.6908},"29":{"raw":0.8401,"skill":0.7868,"coverage":1,"random":0.25,"tracks":[],"combined":0.6564,"private_eq":0.7445},"30":{"raw":0.4431,"skill":0.3316,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5527,"random":0.25},{"track":"10-choice","raw":0.3336,"random":0.1}],"combined":0.4289,"private_eq":0.5334},"31":{"raw":0.1032,"skill":0.0232,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0377,"private_eq":0.1257},"32":{"raw":0.6011,"skill":0.3658,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.2721,"skill":0.2624,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1275,"private_eq":0.1478},"36":{"raw":0.4302,"skill":0.3554,"coverage":1,"random":0.116,"tracks":[],"combined":0.3103,"private_eq":0.3964},"37":{"raw":0.4275,"skill":0.2819,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5143,"private_eq":0.6164},"38":{"raw":0.1779,"skill":0.1516,"coverage":1,"random":0.031,"tracks":[],"combined":0.2918,"private_eq":0.3206},"39":{"raw":0.8714,"skill":0.8109,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4562,"private_eq":0.6338},"40":{"raw":0.4067,"skill":0.2367,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3128,"headline":false},{"track":"A · English","score":0.4067,"headline":true},{"track":"C · Arabic pairs","score":0.55,"headline":false},{"track":"C · English pairs","score":0.85,"headline":false}],"combined":0.2233,"private_eq":0.4024},"41":{"raw":0.5397,"skill":0.3096,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.7045,"skill":0.4251,"coverage":1,"random":0.486,"tracks":[],"combined":0.6108,"private_eq":0.8017},"43":{"raw":0.3772,"skill":0.0119,"coverage":1,"random":0.3697,"tracks":[],"combined":0.353,"private_eq":0.5962},"44":{"raw":0.5892,"skill":0.1784,"coverage":1,"random":0.5,"tracks":[],"combined":0.155,"private_eq":0.5818},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4248,"skill":0.1652,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.5735,"skill":0.147,"coverage":1,"random":0.5,"tracks":[],"combined":0.1703,"private_eq":0.5894},"57":{"raw":0.4067,"skill":0.3327,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3711,"combined":0.2855},"58":{"raw":0.5112,"skill":0.2915,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3752,"combined":0.0849},"59":{"raw":0.7002,"skill":0.3784,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1367,"private_eq":0.5879},"61":{"raw":0.803,"skill":0.606,"coverage":1,"random":0.5,"tracks":[],"combined":0.6662,"private_eq":0.8345},"62":{"raw":0.807,"skill":0.7427,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.5095,"skill":0.3869,"coverage":1,"random":0.2,"tracks":[],"combined":0.458,"private_eq":0.5705}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #223).","Replaces the earlier board entry vera-core (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 41.28, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.55% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":56.27,"balanced_skill":30.6,"breadth_skill":38.79,"public_skill":41.28},"frozen_scores":{"balanced_raw":56.27,"balanced_skill":41.28,"breadth_skill":38.79},"categories":[{"id":"knowledge","raw":0.4088,"skill":0.2136,"combined":0.1927,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6072,"skill":0.4387,"combined":0.3641,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6688,"skill":0.5382,"combined":0.5302,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6451,"skill":0.6151,"combined":0.5567,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4822,"skill":0.2394,"combined":0.3048,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vera-core-2","pub":41.28,"same_skills":39.55,"new_domains":12.49,"v":30.6,"lab":false,"jev":false,"rank_v":167,"rank_pub":148,"name":"vera-core-2","eq":{"S":38.95,"O":18.07},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":23.37,"new_domains":7.23}},"new_in_edition":true},{"engine":"aurelis-2b","name":"Aurelis-2B","long_name":"Aurelis-2B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2516756480,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Leptos-ai/Aurelis-2B","code_url":null,"base_model":"openbmb/MiniCPM5-2B","base_url":"https://huggingface.co/openbmb/MiniCPM5-2B","weights_repo":null,"served_checkpoint":null,"org":{"id":"Leptos-ai","name":"Leptos Labs","logo":"data/orgs/Leptos-ai.png"},"pr":212,"new":true},"completed":110201,"counts":{"ok":110149,"unsupported":52,"error":0},"lifted":false,"calibration":{"acc":0.6966,"conf":0.7603,"ece":0.0653,"brier":0.4044,"over95":0.011,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0301,"conf":0.0562,"acc":0.067},{"lo":0.1,"w":0.0105,"conf":0.1431,"acc":0.186},{"lo":0.2,"w":0.0145,"conf":0.26,"acc":0.1512},{"lo":0.3,"w":0.0413,"conf":0.3619,"acc":0.3483},{"lo":0.4,"w":0.0612,"conf":0.453,"acc":0.3643},{"lo":0.5,"w":0.086,"conf":0.5505,"acc":0.4758},{"lo":0.6,"w":0.0966,"conf":0.6511,"acc":0.5878},{"lo":0.7,"w":0.1098,"conf":0.7501,"acc":0.6333},{"lo":0.8,"w":0.1387,"conf":0.8533,"acc":0.7413},{"lo":0.9,"w":0.4113,"conf":0.9718,"acc":0.9327}],"areas":{"knowledge":{"ece":0.1127,"acc":0.5729,"conf":0.6854},"language":{"ece":0.0369,"acc":0.7953,"conf":0.8251},"retrieval":{"ece":0.0719,"acc":0.7657,"conf":0.8347},"tools":{"ece":0.034,"acc":0.9367,"conf":0.9707},"arts":{"ece":0.0597,"acc":0.5669,"conf":0.6077}}},"gaps":null,"latency":{"median":74.1,"p95":501.8,"mean":134.2},"results":{"1":{"score":0.9433,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":226.8},"2":{"score":0.6329,"jev":null,"metric":"nDCG@10","requests":685,"answered":676,"unsupported":9,"errors":0,"median_ms":1788.6},"3":{"score":0.8524,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2088.3},"4":{"score":0.8643,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":715.2},"5":{"score":0.8962,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":443.9},"6":{"score":0.7861,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":311},"9":{"score":0.7273,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":241},"10":{"score":0.6469,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":163.3},"11":{"score":0.8257,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":758.5},"12":{"score":0.6406,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":38.7},"20":{"score":0.945,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":38.9},"21":{"score":0.629,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":24.5},"22":{"score":0.096,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1053.8},"23":{"score":0.555,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.3},"24":{"score":0.6012,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":29.8},"25":{"score":0.4133,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":38.5},"26":{"score":0.899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":28.5},"27":{"score":0.7747,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":28.9},"28":{"score":0.7585,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":24.8},"29":{"score":0.8469,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":45.4},"30":{"score":0.4003,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1642,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":195.1},"32":{"score":0.5638,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":108.2},"33":{"score":0.3939,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":125.8},"36":{"score":0.3842,"jev":null,"metric":"nDCG@10","requests":220,"answered":163,"unsupported":57,"errors":0,"median_ms":925.2},"37":{"score":0.532,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":59.6},"38":{"score":0.381,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":672.8},"39":{"score":0.8154,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":46.9},"40":{"score":0.3735,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":22.7},"41":{"score":0.729,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":28.4},"42":{"score":0.7217,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":64.4},"43":{"score":0.4263,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":35.7},"44":{"score":0.8552,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.7},"45":{"score":0.1317,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":100.8},"50":{"score":0.608,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":224.3},"56":{"score":0.821,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":196.3},"57":{"score":0.3938,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":105.5},"58":{"score":0.7383,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":39.9},"59":{"score":0.7558,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":77.2},"61":{"score":0.5907,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":100.7},"62":{"score":0.9025,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":62.3},"64":{"score":0.4867,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":50.5}},"benchmarks":{"1":{"raw":0.9433,"skill":0.9235,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8784,"private_eq":0.9102},"2":{"raw":0.6246,"skill":0.5665,"coverage":0.9869,"random":0.1341,"tracks":[],"combined":0.6335,"private_eq":0.6854},"3":{"raw":0.8524,"skill":0.8496,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7753,"private_eq":0.7811},"4":{"raw":0.8643,"skill":0.8626,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7715,"private_eq":0.776},"5":{"raw":0.8962,"skill":0.8956,"coverage":1,"random":0.006,"tracks":[],"combined":0.6065,"private_eq":0.6123},"6":{"raw":0.7861,"skill":0.4985,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7273,"skill":0.7273,"coverage":1,"random":0,"tracks":[],"combined":0.5484,"private_eq":0.5525},"10":{"raw":0.6469,"skill":0.4125,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8257,"skill":0.7479,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6406,"skill":0.4617,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3789,"private_eq":0.5894},"20":{"raw":0.9424,"skill":0.8847,"coverage":1,"random":0.5,"tracks":[],"combined":0.7698,"private_eq":0.8857},"21":{"raw":0.629,"skill":0.258,"coverage":1,"random":0.5,"tracks":[],"combined":0.2365,"private_eq":0.6221},"22":{"raw":0.0908,"skill":0.0837,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1013,"private_eq":0.1175},"23":{"raw":0.5705,"skill":0.141,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6012,"skill":0.4683,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3215,"combined":0.0859},"26":{"raw":0.899,"skill":0.8653,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7747,"skill":0.6995,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7585,"skill":0.517,"coverage":1,"random":0.5,"tracks":[],"combined":0.3603,"private_eq":0.6833},"29":{"raw":0.8469,"skill":0.7959,"coverage":1,"random":0.25,"tracks":[],"combined":0.6266,"private_eq":0.7224},"30":{"raw":0.4003,"skill":0.2785,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5034,"random":0.25},{"track":"10-choice","raw":0.2972,"random":0.1}],"combined":0.366,"private_eq":0.4821},"31":{"raw":0.1642,"skill":0.0897,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1113,"private_eq":0.1925},"32":{"raw":0.5638,"skill":0.3065,"coverage":1,"random":0.371,"tracks":[],"combined":0.0649,"private_eq":0.4179},"33":{"raw":0.3939,"skill":0.3859,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3383,"private_eq":0.3534},"36":{"raw":0.2847,"skill":0.1908,"coverage":0.7409,"random":0.116,"tracks":[],"combined":0.2667,"private_eq":0.3583},"37":{"raw":0.532,"skill":0.413,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3846,"private_eq":0.5141},"38":{"raw":0.381,"skill":0.3612,"coverage":1,"random":0.031,"tracks":[],"combined":0.561,"private_eq":0.5785},"39":{"raw":0.8154,"skill":0.7285,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5847,"private_eq":0.7202},"40":{"raw":0.3735,"skill":0.1941,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3329,"headline":false},{"track":"A · English","score":0.3735,"headline":true},{"track":"C · Arabic pairs","score":0.81,"headline":false},{"track":"C · English pairs","score":0.86,"headline":false}],"combined":0.2038,"private_eq":0.3874},"41":{"raw":0.729,"skill":0.5935,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.7217,"skill":0.4586,"coverage":1,"random":0.486,"tracks":[],"combined":0.6861,"private_eq":0.84},"43":{"raw":0.4263,"skill":0.0898,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3488,"private_eq":0.5936},"44":{"raw":0.8552,"skill":0.7104,"coverage":1,"random":0.5,"tracks":[],"combined":0.6369,"private_eq":0.82},"45":{"raw":0.1317,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.608,"skill":0.4311,"coverage":1,"random":0.311,"tracks":[],"combined":0.174,"private_eq":0.4367},"56":{"raw":0.821,"skill":0.642,"coverage":1,"random":0.5,"tracks":[],"combined":0.404,"private_eq":0.7049},"57":{"raw":0.3938,"skill":0.3182,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4394,"combined":0.3632},"58":{"raw":0.7383,"skill":0.6207,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3814,"combined":0.094},"59":{"raw":0.7558,"skill":0.4937,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3534,"private_eq":0.6912},"61":{"raw":0.5907,"skill":0.1814,"coverage":1,"random":0.5,"tracks":[],"combined":0.3203,"private_eq":0.6635},"62":{"raw":0.9025,"skill":0.87,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.4867,"skill":0.3584,"coverage":1,"random":0.2,"tracks":[],"combined":0.3706,"private_eq":0.5014}},"coverage":0.9901,"pending":0.0099,"notes":["New in the 2026-10-18 PR batch (PR #212).","Submitted public 0.3 run adopted (kit rescore 51.29, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (GGUF Q8_0); decision agreement with the submitted rows 99.00% on 826 rows (bar 96%).","Served as GGUF Q8_0 through Ollama, as in the author's run, so the quantized agreement bar applies."],"scores":{"balanced_raw":63.71,"balanced_skill":32.55,"breadth_skill":49.22,"public_skill":51.29},"frozen_scores":{"balanced_raw":63.71,"balanced_skill":51.29,"breadth_skill":49.22},"categories":[{"id":"knowledge","raw":0.4772,"skill":0.3224,"combined":0.2732,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6795,"skill":0.5411,"combined":0.4268,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6664,"skill":0.5417,"combined":0.467,"coverage":0.9529,"pending":0,"n":6},{"id":"tools","raw":0.8165,"skill":0.7947,"combined":0.6982,"coverage":0.9976,"pending":0,"n":5},{"id":"arts","raw":0.5546,"skill":0.3595,"combined":0.3305,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"aurelis-2b","pub":51.29,"same_skills":44.71,"new_domains":10.1,"v":32.55,"lab":false,"jev":false,"rank_v":159,"rank_pub":113,"name":"Aurelis-2B","eq":{"S":44.17,"O":15.11},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":26.5,"new_domains":6.05}},"new_in_edition":true},{"engine":"saina-helm-2-v2","name":"Saina Helm 2.0 v2","long_name":"Saina Helm 2.0 v2","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":852985920,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/run-saina/saina-helm-2-0.8b","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"run-saina","name":"Saina","logo":"data/orgs/run-saina.png"},"pr":209,"replaces":"saina-helm-2.0-06f2fec","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6366,"conf":0.805,"ece":0.1684,"brier":0.52,"over95":0.0649,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0027,"conf":0.0844,"acc":0.0583},{"lo":0.1,"w":0.0224,"conf":0.1498,"acc":0.1089},{"lo":0.2,"w":0.0195,"conf":0.2526,"acc":0.1488},{"lo":0.3,"w":0.0363,"conf":0.3569,"acc":0.2222},{"lo":0.4,"w":0.0505,"conf":0.456,"acc":0.3337},{"lo":0.5,"w":0.0775,"conf":0.5495,"acc":0.4557},{"lo":0.6,"w":0.0769,"conf":0.6498,"acc":0.4583},{"lo":0.7,"w":0.084,"conf":0.7524,"acc":0.4779},{"lo":0.8,"w":0.1032,"conf":0.8539,"acc":0.6},{"lo":0.9,"w":0.527,"conf":0.9803,"acc":0.8226}],"areas":{"knowledge":{"ece":0.1905,"acc":0.515,"conf":0.7039},"language":{"ece":0.1389,"acc":0.7427,"conf":0.8816},"retrieval":{"ece":0.2232,"acc":0.7206,"conf":0.9431},"tools":{"ece":0.0752,"acc":0.8864,"conf":0.9617},"arts":{"ece":0.1931,"acc":0.4716,"conf":0.6647}}},"gaps":null,"latency":{"median":38.5,"p95":49.3,"mean":53.5},"results":{"1":{"score":0.925,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":42.5},"2":{"score":0.4841,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":389.1},"3":{"score":0.8169,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":287.9},"4":{"score":0.9096,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":61.2},"5":{"score":0.9262,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":100.5},"6":{"score":0.7853,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":97.2},"9":{"score":0.0682,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":111.5},"10":{"score":0.4393,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":51.7},"11":{"score":0.8543,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":146.3},"12":{"score":0.4819,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":39.6},"20":{"score":0.6082,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.8},"21":{"score":0.5989,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":39},"22":{"score":0.0385,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":259.4},"23":{"score":0.6126,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.8},"24":{"score":0.4918,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":39.4},"25":{"score":0.2449,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":40.5},"26":{"score":0.8535,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":39},"27":{"score":0.7167,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":39.1},"28":{"score":0.6243,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":38.8},"29":{"score":0.6553,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":40},"30":{"score":0.3465,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.218,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":51.1},"32":{"score":0.5492,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":46.1},"33":{"score":0.1418,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":45.3},"36":{"score":0.2116,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":497.1},"37":{"score":0.4792,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.9},"38":{"score":0.3726,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":303},"39":{"score":0.748,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":40.6},"40":{"score":0.2911,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":39.6},"41":{"score":0.6955,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":40.3},"42":{"score":0.6336,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":41.7},"43":{"score":0.3088,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":39.6},"44":{"score":0.7652,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.5},"45":{"score":0.1357,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":40},"50":{"score":0.5179,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":44.2},"56":{"score":0.5095,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":43.6},"57":{"score":0.2472,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":39.9},"58":{"score":0.4287,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":39.7},"59":{"score":0.775,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":43.3},"61":{"score":0.8195,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":41},"62":{"score":0.8384,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":42.6},"64":{"score":0.5341,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":39.5}},"benchmarks":{"1":{"raw":0.925,"skill":0.8988,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8239,"private_eq":0.8703},"2":{"raw":0.4841,"skill":0.4042,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6622,"private_eq":0.71},"3":{"raw":0.8169,"skill":0.8134,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6704,"private_eq":0.6793},"4":{"raw":0.9096,"skill":0.9084,"coverage":1,"random":0.0127,"tracks":[],"combined":0.748,"private_eq":0.753},"5":{"raw":0.9262,"skill":0.9258,"coverage":1,"random":0.006,"tracks":[],"combined":0.5399,"private_eq":0.5469},"6":{"raw":0.7853,"skill":0.4966,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0682,"skill":0.0682,"coverage":1,"random":0,"tracks":[],"combined":0.035,"private_eq":0.045},"10":{"raw":0.4393,"skill":0.0671,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8543,"skill":0.7893,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4819,"skill":0.2239,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3034,"private_eq":0.5396},"20":{"raw":0.6019,"skill":0.2037,"coverage":1,"random":0.5,"tracks":[],"combined":0.3431,"private_eq":0.6748},"21":{"raw":0.5989,"skill":0.1979,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.0387,"skill":0.0312,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0923,"private_eq":0.1086},"23":{"raw":0.6009,"skill":0.2019,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4918,"skill":0.3224,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3132,"combined":0.0747},"26":{"raw":0.8535,"skill":0.8046,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7167,"skill":0.6222,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6243,"skill":0.2486,"coverage":1,"random":0.5,"tracks":[],"combined":0.1283,"private_eq":0.5686},"29":{"raw":0.6553,"skill":0.5404,"coverage":1,"random":0.25,"tracks":[],"combined":0.3446,"private_eq":0.5133},"30":{"raw":0.3465,"skill":0.213,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4519,"random":0.25},{"track":"10-choice","raw":0.2411,"random":0.1}],"combined":0.2514,"private_eq":0.3886},"31":{"raw":0.218,"skill":0.1483,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0962,"private_eq":0.1788},"32":{"raw":0.5492,"skill":0.2833,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1418,"skill":0.1304,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0175},"36":{"raw":0.2116,"skill":0.1081,"coverage":1,"random":0.116,"tracks":[],"combined":0.1729,"private_eq":0.2763},"37":{"raw":0.4792,"skill":0.3467,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5018,"private_eq":0.6065},"38":{"raw":0.3726,"skill":0.3525,"coverage":1,"random":0.031,"tracks":[],"combined":0.4371,"private_eq":0.4598},"39":{"raw":0.748,"skill":0.6294,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4735,"private_eq":0.6454},"40":{"raw":0.2911,"skill":0.088,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3226,"headline":false},{"track":"A · English","score":0.2911,"headline":true},{"track":"C · Arabic pairs","score":0.81,"headline":false},{"track":"C · English pairs","score":0.815,"headline":false}],"combined":0.0797,"private_eq":0.292},"41":{"raw":0.6955,"skill":0.5433,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.6336,"skill":0.2872,"coverage":1,"random":0.486,"tracks":[],"combined":0.5334,"private_eq":0.7624},"43":{"raw":0.3088,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2066,"private_eq":0.505},"44":{"raw":0.7652,"skill":0.5304,"coverage":1,"random":0.5,"tracks":[],"combined":0.4388,"private_eq":0.7221},"45":{"raw":0.1357,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5179,"skill":0.3003,"coverage":1,"random":0.311,"tracks":[],"combined":0.283,"private_eq":0.5109},"56":{"raw":0.5095,"skill":0.019,"coverage":1,"random":0.5,"tracks":[],"combined":0.0146,"private_eq":0.5124},"57":{"raw":0.2472,"skill":0.1533,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4102,"combined":0.33},"58":{"raw":0.4287,"skill":0.1719,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3394,"combined":0.0324},"59":{"raw":0.775,"skill":0.5335,"coverage":1,"random":0.5177,"tracks":[],"combined":0.272,"private_eq":0.6524},"61":{"raw":0.8195,"skill":0.639,"coverage":1,"random":0.5,"tracks":[],"combined":0.4847,"private_eq":0.7448},"62":{"raw":0.8384,"skill":0.7845,"coverage":1,"random":0.25,"tracks":[],"combined":0.5082,"private_eq":0.6346},"64":{"raw":0.5341,"skill":0.4176,"coverage":1,"random":0.2,"tracks":[],"combined":0.4968,"private_eq":0.6012}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #209).","Replaces the earlier board entry saina-helm-2.0-06f2fec (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 39.26, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.41% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":54.68,"balanced_skill":25.29,"breadth_skill":36.12,"public_skill":39.26},"frozen_scores":{"balanced_raw":54.68,"balanced_skill":39.26,"breadth_skill":36.12},"categories":[{"id":"knowledge","raw":0.3622,"skill":0.1662,"combined":0.1533,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6101,"skill":0.4405,"combined":0.3051,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6462,"skill":0.5054,"combined":0.4173,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6446,"skill":0.6132,"combined":0.5553,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4821,"skill":0.2254,"combined":0.3004,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"saina-helm-2-v2","pub":39.26,"same_skills":34.65,"new_domains":7.75,"v":25.29,"lab":false,"jev":false,"rank_v":181,"rank_pub":157,"name":"Saina Helm 2.0 v2","eq":{"S":34,"O":12.21},"rank_tie":41,"tied":true,"parts":{"public":0,"same_skills":20.4,"new_domains":4.89}},"new_in_edition":true},{"engine":"saina-helm-2-omni","name":"Saina Helm 2 Omni (0.8B)","long_name":"Saina Helm 2 Omni (0.8B)","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":852985920,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/run-saina/saina-helm-2-omni-0.8b","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"run-saina","name":"Saina","logo":"data/orgs/run-saina.png"},"pr":214,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6144,"conf":0.713,"ece":0.0999,"brier":0.4961,"over95":0.0223,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0325,"conf":0.0357,"acc":0.0547},{"lo":0.1,"w":0.0151,"conf":0.1552,"acc":0.1561},{"lo":0.2,"w":0.0278,"conf":0.2575,"acc":0.1754},{"lo":0.3,"w":0.0539,"conf":0.3556,"acc":0.3163},{"lo":0.4,"w":0.0715,"conf":0.4507,"acc":0.3449},{"lo":0.5,"w":0.1272,"conf":0.5492,"acc":0.4839},{"lo":0.6,"w":0.1114,"conf":0.6474,"acc":0.531},{"lo":0.7,"w":0.1015,"conf":0.7503,"acc":0.6068},{"lo":0.8,"w":0.1138,"conf":0.852,"acc":0.686},{"lo":0.9,"w":0.3452,"conf":0.9726,"acc":0.8783}],"areas":{"knowledge":{"ece":0.1155,"acc":0.4958,"conf":0.6111},"language":{"ece":0.0813,"acc":0.7141,"conf":0.793},"retrieval":{"ece":0.1713,"acc":0.7027,"conf":0.8707},"tools":{"ece":0.0856,"acc":0.8396,"conf":0.9247},"arts":{"ece":0.0619,"acc":0.4663,"conf":0.5194}}},"gaps":null,"latency":{"median":38.5,"p95":48.3,"mean":43.2},"results":{"1":{"score":0.9191,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":43.2},"2":{"score":0.5643,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":401.6},"3":{"score":0.7677,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":304.7},"4":{"score":0.8745,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":63.2},"5":{"score":0.8993,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":103.5},"6":{"score":0.7651,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":101.1},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":115.6},"10":{"score":0.4101,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":53.8},"11":{"score":0.8441,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":153.3},"12":{"score":0.4487,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":39.9},"20":{"score":0.702,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.7},"21":{"score":0.6058,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":38.7},"22":{"score":0.0535,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":269.9},"23":{"score":0.5824,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.9},"24":{"score":0.4964,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":39.4},"25":{"score":0.2908,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":39.8},"26":{"score":0.8603,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":39},"27":{"score":0.7125,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":39.1},"28":{"score":0.614,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":38.8},"29":{"score":0.5628,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":40.8},"30":{"score":0.3127,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.188,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.3},"32":{"score":0.496,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":48.4},"33":{"score":0.2,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":48},"36":{"score":0.3006,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":520.5},"37":{"score":0.5096,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.8},"38":{"score":0.2453,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":318.1},"39":{"score":0.7087,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":40.8},"40":{"score":0.3073,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":39.7},"41":{"score":0.6793,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":40.9},"42":{"score":0.6163,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":42.9},"43":{"score":0.307,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":40.8},"44":{"score":0.6912,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":40.2},"45":{"score":0.1397,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":40.9},"50":{"score":0.4368,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":47.1},"56":{"score":0.5145,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":46.1},"57":{"score":0.2522,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":41.1},"58":{"score":0.4532,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":40.4},"59":{"score":0.7745,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":45.4},"61":{"score":0.763,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":42.4},"62":{"score":0.805,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":44.5},"64":{"score":0.5417,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":40.2}},"benchmarks":{"1":{"raw":0.9191,"skill":0.8908,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8852,"private_eq":0.9152},"2":{"raw":0.5643,"skill":0.4968,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6684,"private_eq":0.7153},"3":{"raw":0.7677,"skill":0.7632,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5444,"private_eq":0.5571},"4":{"raw":0.8745,"skill":0.8729,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7099,"private_eq":0.7159},"5":{"raw":0.8993,"skill":0.8987,"coverage":1,"random":0.006,"tracks":[],"combined":0.5767,"private_eq":0.583},"6":{"raw":0.7651,"skill":0.4492,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0.0097,"private_eq":0.02},"10":{"raw":0.4101,"skill":0.0185,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8441,"skill":0.7745,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4487,"skill":0.1742,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3751,"private_eq":0.5869},"20":{"raw":0.6984,"skill":0.3967,"coverage":1,"random":0.5,"tracks":[],"combined":0.437,"private_eq":0.7212},"21":{"raw":0.6058,"skill":0.2116,"coverage":1,"random":0.5,"tracks":[],"combined":0.1157,"private_eq":0.5624},"22":{"raw":0.0478,"skill":0.0404,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0159,"private_eq":0.0337},"23":{"raw":0.5822,"skill":0.1644,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4964,"skill":0.3285,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2908,"skill":0.0544,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3101,"combined":0.0705},"26":{"raw":0.8603,"skill":0.8137,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7125,"skill":0.6166,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.614,"skill":0.228,"coverage":1,"random":0.5,"tracks":[],"combined":0.0123,"private_eq":0.5113},"29":{"raw":0.5628,"skill":0.4171,"coverage":1,"random":0.25,"tracks":[],"combined":0.3248,"private_eq":0.4986},"30":{"raw":0.3127,"skill":0.1723,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.423,"random":0.25},{"track":"10-choice","raw":0.2024,"random":0.1}],"combined":0.2266,"private_eq":0.3684},"31":{"raw":0.188,"skill":0.1156,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1015,"private_eq":0.1836},"32":{"raw":0.496,"skill":0.1987,"coverage":1,"random":0.371,"tracks":[],"combined":0.0184,"private_eq":0.389},"33":{"raw":0.2,"skill":0.1894,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1379,"private_eq":0.1579},"36":{"raw":0.3006,"skill":0.2088,"coverage":1,"random":0.116,"tracks":[],"combined":0.1596,"private_eq":0.2647},"37":{"raw":0.5096,"skill":0.3849,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4658,"private_eq":0.5781},"38":{"raw":0.2453,"skill":0.2212,"coverage":1,"random":0.031,"tracks":[],"combined":0.2399,"private_eq":0.2709},"39":{"raw":0.7087,"skill":0.5716,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4693,"private_eq":0.6426},"40":{"raw":0.3073,"skill":0.1088,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1269,"headline":false},{"track":"A · English","score":0.3073,"headline":true},{"track":"C · Arabic pairs","score":0.385,"headline":false},{"track":"C · English pairs","score":0.59,"headline":false}],"combined":0.3287,"private_eq":0.4834},"41":{"raw":0.6793,"skill":0.519,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3026,"private_eq":0.5397},"42":{"raw":0.6163,"skill":0.2535,"coverage":1,"random":0.486,"tracks":[],"combined":0.557,"private_eq":0.7744},"43":{"raw":0.307,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1439,"private_eq":0.4659},"44":{"raw":0.6912,"skill":0.3824,"coverage":1,"random":0.5,"tracks":[],"combined":0.3959,"private_eq":0.7009},"45":{"raw":0.1397,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4368,"skill":0.1825,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.5145,"skill":0.029,"coverage":1,"random":0.5,"tracks":[],"combined":0.1358,"private_eq":0.5723},"57":{"raw":0.2522,"skill":0.1589,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4463,"combined":0.3711},"58":{"raw":0.4532,"skill":0.2074,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3523,"combined":0.0514},"59":{"raw":0.7745,"skill":0.5324,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2659,"private_eq":0.6495},"61":{"raw":0.763,"skill":0.526,"coverage":1,"random":0.5,"tracks":[],"combined":0.5131,"private_eq":0.7588},"62":{"raw":0.805,"skill":0.74,"coverage":1,"random":0.25,"tracks":[],"combined":0.4741,"private_eq":0.6093},"64":{"raw":0.5417,"skill":0.4271,"coverage":1,"random":0.2,"tracks":[],"combined":0.3998,"private_eq":0.5245}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #214).","Submitted public 0.3 run adopted (kit rescore 37.44, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.36% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":53.39,"balanced_skill":26.54,"breadth_skill":34.37,"public_skill":37.44},"frozen_scores":{"balanced_raw":53.39,"balanced_skill":37.44,"breadth_skill":34.37},"categories":[{"id":"knowledge","raw":0.3579,"skill":0.1531,"combined":0.151,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5733,"skill":0.3926,"combined":0.3575,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6479,"skill":0.5025,"combined":0.4318,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6284,"skill":0.5966,"combined":0.5311,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4854,"skill":0.2371,"combined":0.243,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"saina-helm-2-omni","pub":37.44,"same_skills":35.21,"new_domains":9.59,"v":26.54,"lab":false,"jev":false,"rank_v":177,"rank_pub":168,"name":"Saina Helm 2 Omni (0.8B)","eq":{"S":34.57,"O":14.49},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":20.74,"new_domains":5.8}},"new_in_edition":true},{"engine":"wyat-1-800m","name":"wyat-1-800m","long_name":"wyat-1-800m","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/teilomillet/wyat-1-800m","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B-Base","weights_repo":null,"served_checkpoint":null,"pr":222,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5223,"conf":0.7543,"ece":0.232,"brier":0.6916,"over95":0.0925,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0138,"conf":0.073,"acc":0.0457},{"lo":0.1,"w":0.0132,"conf":0.141,"acc":0.0662},{"lo":0.2,"w":0.0112,"conf":0.2501,"acc":0.2432},{"lo":0.3,"w":0.0313,"conf":0.3568,"acc":0.2212},{"lo":0.4,"w":0.0502,"conf":0.4539,"acc":0.3766},{"lo":0.5,"w":0.1388,"conf":0.5459,"acc":0.4491},{"lo":0.6,"w":0.1225,"conf":0.6505,"acc":0.4843},{"lo":0.7,"w":0.1104,"conf":0.7493,"acc":0.5065},{"lo":0.8,"w":0.132,"conf":0.8523,"acc":0.5378},{"lo":0.9,"w":0.3767,"conf":0.9663,"acc":0.6469}],"areas":{"knowledge":{"ece":0.2424,"acc":0.4525,"conf":0.6949},"language":{"ece":0.2317,"acc":0.5541,"conf":0.7856},"retrieval":{"ece":0.2204,"acc":0.6263,"conf":0.8467},"tools":{"ece":0.2141,"acc":0.6484,"conf":0.8313},"arts":{"ece":0.2506,"acc":0.4458,"conf":0.6964}}},"gaps":null,"latency":{"median":111.8,"p95":923.6,"mean":311.5},"results":{"1":{"score":0.7834,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":179.9},"2":{"score":0.5834,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1869.5},"3":{"score":0.5197,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1066.7},"4":{"score":0.8051,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":380.5},"5":{"score":0.8591,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":822.8},"6":{"score":0.6539,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":443.7},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1559.7},"10":{"score":0.1838,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":96.6},"11":{"score":0.475,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3080.8},"12":{"score":0.414,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":53.6},"20":{"score":0.563,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.5},"21":{"score":0.5559,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":52},"22":{"score":0.0145,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1807.3},"23":{"score":0.57,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":61.2},"24":{"score":0.4697,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":52.4},"25":{"score":0.301,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":55.2},"26":{"score":0.8312,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":52},"27":{"score":0.6775,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":51.9},"28":{"score":0.5896,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":51.8},"29":{"score":0.4966,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":56.5},"30":{"score":0.2945,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.086,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":208.7},"32":{"score":0.5093,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":97.3},"33":{"score":0.1812,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":461.8},"36":{"score":0.3015,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2171.4},"37":{"score":0.4272,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.4},"38":{"score":0.0805,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3411.1},"39":{"score":0.7002,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":107},"40":{"score":0.2884,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":52.2},"41":{"score":0.45,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":55.1},"42":{"score":0.6241,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":61.6},"43":{"score":0.3456,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":53.7},"44":{"score":0.5514,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.6},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":56.5},"50":{"score":0.4218,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":108.3},"56":{"score":0.5745,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":499.9},"57":{"score":0.2541,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":56.4},"58":{"score":0.4364,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":53.8},"59":{"score":0.141,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":82},"61":{"score":0.5805,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":60.5},"62":{"score":0.46,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":76.3},"64":{"score":0.4129,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":55.1}},"benchmarks":{"1":{"raw":0.7834,"skill":0.7075,"coverage":1,"random":0.2592,"tracks":[],"combined":0.3608,"private_eq":0.5311},"2":{"raw":0.5834,"skill":0.5189,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6137,"private_eq":0.6684},"3":{"raw":0.5197,"skill":0.5104,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3817,"private_eq":0.3993},"4":{"raw":0.8051,"skill":0.8026,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7008,"private_eq":0.707},"5":{"raw":0.8591,"skill":0.8582,"coverage":1,"random":0.006,"tracks":[],"combined":0.5776,"private_eq":0.5839},"6":{"raw":0.6539,"skill":0.1885,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.1838,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.475,"skill":0.2408,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.414,"skill":0.1222,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3545,"private_eq":0.5733},"20":{"raw":0.5621,"skill":0.1242,"coverage":1,"random":0.5,"tracks":[],"combined":0.1335,"private_eq":0.5712},"21":{"raw":0.5559,"skill":0.1119,"coverage":1,"random":0.5,"tracks":[],"combined":0.0554,"private_eq":0.5326},"22":{"raw":0.015,"skill":0.0073,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0573,"private_eq":0.0743},"23":{"raw":0.56,"skill":0.12,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4697,"skill":0.2929,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.301,"skill":0.068,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2976,"combined":0.0537},"26":{"raw":0.8312,"skill":0.7749,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.6775,"skill":0.5699,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5896,"skill":0.1792,"coverage":1,"random":0.5,"tracks":[],"combined":0.1536,"private_eq":0.5811},"29":{"raw":0.4966,"skill":0.3288,"coverage":1,"random":0.25,"tracks":[],"combined":0.2406,"private_eq":0.4362},"30":{"raw":0.2945,"skill":0.1499,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4041,"random":0.25},{"track":"10-choice","raw":0.185,"random":0.1}],"combined":0.1851,"private_eq":0.3345},"31":{"raw":0.086,"skill":0.0045,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0892},"32":{"raw":0.5093,"skill":0.2199,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1812,"skill":0.1703,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0453,"private_eq":0.0676},"36":{"raw":0.3015,"skill":0.2097,"coverage":1,"random":0.116,"tracks":[],"combined":0.2102,"private_eq":0.3089},"37":{"raw":0.4272,"skill":0.2816,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3045,"private_eq":0.451},"38":{"raw":0.0805,"skill":0.0511,"coverage":1,"random":0.031,"tracks":[],"combined":0.1063,"private_eq":0.1429},"39":{"raw":0.7002,"skill":0.5591,"coverage":1,"random":0.3201,"tracks":[],"combined":0.2327,"private_eq":0.4836},"40":{"raw":0.2884,"skill":0.0845,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2483,"headline":false},{"track":"A · English","score":0.2884,"headline":true},{"track":"C · Arabic pairs","score":0.655,"headline":false},{"track":"C · English pairs","score":0.73,"headline":false}],"combined":0.0376,"private_eq":0.2597},"41":{"raw":0.45,"skill":0.1751,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3216,"private_eq":0.5522},"42":{"raw":0.6241,"skill":0.2687,"coverage":1,"random":0.486,"tracks":[],"combined":0.4834,"private_eq":0.737},"43":{"raw":0.3456,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1524,"private_eq":0.4712},"44":{"raw":0.5514,"skill":0.1028,"coverage":1,"random":0.5,"tracks":[],"combined":0.2834,"private_eq":0.6453},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4218,"skill":0.1609,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.5745,"skill":0.149,"coverage":1,"random":0.5,"tracks":[],"combined":0.2221,"private_eq":0.615},"57":{"raw":0.2541,"skill":0.1611,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2207,"combined":0.1144},"58":{"raw":0.4364,"skill":0.1831,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3223,"combined":0.0074},"59":{"raw":0.141,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5805,"skill":0.161,"coverage":1,"random":0.5,"tracks":[],"combined":0.3486,"private_eq":0.6775},"62":{"raw":0.46,"skill":0.28,"coverage":1,"random":0.25,"tracks":[],"combined":0.2549,"private_eq":0.4468},"64":{"raw":0.4129,"skill":0.2661,"coverage":1,"random":0.2,"tracks":[],"combined":0.2931,"private_eq":0.4401}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #222).","Submitted public 0.3 run adopted (kit rescore 25.78). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.06% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":44.22,"balanced_skill":21.49,"breadth_skill":23.34,"public_skill":25.78},"frozen_scores":{"balanced_raw":44.22,"balanced_skill":25.78,"breadth_skill":23.34},"categories":[{"id":"knowledge","raw":0.3339,"skill":0.1131,"combined":0.097,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4098,"skill":0.2043,"combined":0.2245,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5971,"skill":0.4297,"combined":0.4032,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4828,"skill":0.4186,"combined":0.3239,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4213,"skill":0.1317,"combined":0.1557,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"wyat-1-800m","pub":25.78,"same_skills":25.21,"new_domains":11.68,"v":21.49,"lab":false,"jev":false,"rank_v":192,"rank_pub":203,"name":"wyat-1-800m","eq":{"S":24.45,"O":17.07},"rank_tie":44,"tied":true,"parts":{"public":0,"same_skills":14.67,"new_domains":6.83}},"new_in_edition":true},{"engine":"amber-0.4b","name":"Amber 0.4B","long_name":"Amber 0.4B","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":395881664,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/MLukauskas/amber-0.4b","code_url":null,"base_model":"answerdotai/ModernBERT-large","base_url":"https://huggingface.co/answerdotai/ModernBERT-large","weights_repo":null,"served_checkpoint":null,"pr":201,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6509,"conf":0.7253,"ece":0.0747,"brier":0.433,"over95":0.0172,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0269,"conf":0.0554,"acc":0.0598},{"lo":0.1,"w":0.007,"conf":0.1506,"acc":0.0339},{"lo":0.2,"w":0.0283,"conf":0.2659,"acc":0.2514},{"lo":0.3,"w":0.0892,"conf":0.3519,"acc":0.2743},{"lo":0.4,"w":0.0713,"conf":0.4491,"acc":0.3322},{"lo":0.5,"w":0.1201,"conf":0.548,"acc":0.477},{"lo":0.6,"w":0.083,"conf":0.6485,"acc":0.5616},{"lo":0.7,"w":0.0818,"conf":0.7503,"acc":0.6207},{"lo":0.8,"w":0.0914,"conf":0.8515,"acc":0.7324},{"lo":0.9,"w":0.4009,"conf":0.9801,"acc":0.928}],"areas":{"knowledge":{"ece":0.0957,"acc":0.4052,"conf":0.5002},"language":{"ece":0.0745,"acc":0.7717,"conf":0.8462},"retrieval":{"ece":0.0966,"acc":0.7528,"conf":0.8494},"tools":{"ece":0.0352,"acc":0.9043,"conf":0.9265},"arts":{"ece":0.0482,"acc":0.6908,"conf":0.738}}},"gaps":null,"latency":{"median":42.5,"p95":47.2,"mean":37.5},"results":{"1":{"score":0.9392,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":47.2},"2":{"score":0.6492,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":313.1},"3":{"score":0.7894,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":300.5},"4":{"score":0.9016,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":25},"5":{"score":0.9153,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":25.1},"6":{"score":0.6879,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":97.3},"9":{"score":0.9545,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":508},"10":{"score":0.5156,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":43},"11":{"score":0.7922,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":978.6},"12":{"score":0.4996,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":27.2},"20":{"score":0.9392,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":49.2},"21":{"score":0.6119,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":48.9},"22":{"score":0.813,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":49.8},"23":{"score":0.6106,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.5},"24":{"score":0.2984,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":52},"25":{"score":0.2602,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":54.7},"26":{"score":0.5063,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":51.9},"27":{"score":0.3439,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":52.3},"28":{"score":0.693,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":25.9},"29":{"score":0.7169,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":54.8},"30":{"score":0.1729,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.093,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.1},"32":{"score":0.3963,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":52.7},"33":{"score":0.3758,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":91.7},"36":{"score":0.2492,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":483.1},"37":{"score":0.5178,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.8},"38":{"score":0.4195,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":169.1},"39":{"score":0.9142,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":28.3},"40":{"score":0.3519,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":26.6},"41":{"score":0.7717,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":28.2},"42":{"score":0.6358,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":28},"43":{"score":0.4561,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":53.6},"44":{"score":0.83,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.3},"45":{"score":0.1737,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":55},"50":{"score":0.5805,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":55.7},"56":{"score":0.884,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":57.8},"57":{"score":0.1321,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":54.6},"58":{"score":0.615,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":28.1},"59":{"score":0.7292,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":29.8},"61":{"score":0.8625,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":27.1},"62":{"score":0.7519,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":56.2},"64":{"score":0.5909,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":28.7}},"benchmarks":{"1":{"raw":0.9392,"skill":0.9179,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7898,"private_eq":0.8453},"2":{"raw":0.6492,"skill":0.5948,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6259,"private_eq":0.6789},"3":{"raw":0.7894,"skill":0.7853,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5181,"private_eq":0.5316},"4":{"raw":0.9016,"skill":0.9003,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6016,"private_eq":0.6101},"5":{"raw":0.9153,"skill":0.9148,"coverage":1,"random":0.006,"tracks":[],"combined":0.4531,"private_eq":0.4616},"6":{"raw":0.6879,"skill":0.2682,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9545,"skill":0.9545,"coverage":1,"random":0,"tracks":[],"combined":0.4573,"private_eq":0.4625},"10":{"raw":0.5156,"skill":0.194,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7922,"skill":0.6995,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4996,"skill":0.2504,"coverage":1,"random":0.3324,"tracks":[],"combined":0.291,"private_eq":0.5314},"20":{"raw":0.9347,"skill":0.8694,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.6119,"skill":0.2237,"coverage":1,"random":0.5,"tracks":[],"combined":0.4981,"private_eq":0.7514},"22":{"raw":0.816,"skill":0.8146,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5852,"private_eq":0.5921},"23":{"raw":0.5928,"skill":0.1856,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2984,"skill":0.0645,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2508,"combined":0},"26":{"raw":0.5063,"skill":0.3416,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.3439,"skill":0.125,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.693,"skill":0.386,"coverage":1,"random":0.5,"tracks":[],"combined":0.6075,"private_eq":0.8055},"29":{"raw":0.7169,"skill":0.6225,"coverage":1,"random":0.25,"tracks":[],"combined":0.4188,"private_eq":0.5683},"30":{"raw":0.1729,"skill":0,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2479,"random":0.25},{"track":"10-choice","raw":0.0978,"random":0.1}],"combined":0.0147,"private_eq":0.1956},"31":{"raw":0.093,"skill":0.0121,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0873},"32":{"raw":0.3963,"skill":0.0402,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.3758,"skill":0.3675,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1379,"private_eq":0.1579},"36":{"raw":0.2492,"skill":0.1507,"coverage":1,"random":0.116,"tracks":[],"combined":0.1607,"private_eq":0.2656},"37":{"raw":0.5178,"skill":0.3952,"coverage":1,"random":0.2027,"tracks":[],"combined":0.142,"private_eq":0.3229},"38":{"raw":0.4195,"skill":0.4009,"coverage":1,"random":0.031,"tracks":[],"combined":0.5661,"private_eq":0.5834},"39":{"raw":0.9142,"skill":0.8738,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4322,"private_eq":0.6177},"40":{"raw":0.3519,"skill":0.1662,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2466,"headline":false},{"track":"A · English","score":0.3519,"headline":true},{"track":"C · Arabic pairs","score":0.75,"headline":false},{"track":"C · English pairs","score":0.865,"headline":false}],"combined":0.2181,"private_eq":0.3984},"41":{"raw":0.7717,"skill":0.6575,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1175,"private_eq":0.4177},"42":{"raw":0.6358,"skill":0.2914,"coverage":1,"random":0.486,"tracks":[],"combined":0.2821,"private_eq":0.6347},"43":{"raw":0.4561,"skill":0.1371,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.83,"skill":0.66,"coverage":1,"random":0.5,"tracks":[],"combined":0.5351,"private_eq":0.7697},"45":{"raw":0.1737,"skill":0.0115,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5805,"skill":0.3912,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.884,"skill":0.768,"coverage":1,"random":0.5,"tracks":[],"combined":0.4732,"private_eq":0.7391},"57":{"raw":0.1321,"skill":0.0238,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.615,"skill":0.4419,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.341,"combined":0.0348},"59":{"raw":0.7292,"skill":0.4385,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1329,"private_eq":0.5861},"61":{"raw":0.8625,"skill":0.725,"coverage":1,"random":0.5,"tracks":[],"combined":0.5187,"private_eq":0.7616},"62":{"raw":0.7519,"skill":0.6692,"coverage":1,"random":0.25,"tracks":[],"combined":0.3919,"private_eq":0.5484},"64":{"raw":0.5909,"skill":0.4886,"coverage":1,"random":0.2,"tracks":[],"combined":0.4968,"private_eq":0.6012}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #201).","Submitted public 0.3 run adopted (kit rescore 49.74, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.51% on 895 rows (bar 98.5%)."],"scores":{"balanced_raw":62.82,"balanced_skill":26,"breadth_skill":45.47,"public_skill":49.74},"frozen_scores":{"balanced_raw":62.82,"balanced_skill":49.74,"breadth_skill":45.47},"categories":[{"id":"knowledge","raw":0.3811,"skill":0.1891,"combined":0.1305,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6462,"skill":0.4867,"combined":0.3097,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7187,"skill":0.6435,"combined":0.3928,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8204,"skill":0.7893,"combined":0.5638,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6878,"skill":0.4955,"combined":0.5134,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"amber-0.4b","pub":49.74,"same_skills":35.73,"new_domains":7.86,"v":26,"lab":false,"jev":false,"rank_v":179,"rank_pub":121,"name":"Amber 0.4B","eq":{"S":35.09,"O":12.35},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":21.06,"new_domains":4.94}},"new_in_edition":true},{"engine":"crivo-1b","name":"Crivo-1B","long_name":"Crivo-1B","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":1031276289,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tardellirs/crivo-1b","code_url":null,"base_model":"cross-encoder/ettin-reranker-1b-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-1b-v1","weights_repo":null,"served_checkpoint":null,"pr":220,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6624,"conf":0.8131,"ece":0.1507,"brier":0.479,"over95":0.0435,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0015,"conf":0.0908,"acc":0.0526},{"lo":0.1,"w":0.0184,"conf":0.1515,"acc":0.0559},{"lo":0.2,"w":0.0138,"conf":0.2489,"acc":0.0822},{"lo":0.3,"w":0.0412,"conf":0.3517,"acc":0.2708},{"lo":0.4,"w":0.0518,"conf":0.4494,"acc":0.3376},{"lo":0.5,"w":0.0815,"conf":0.5475,"acc":0.4152},{"lo":0.6,"w":0.0665,"conf":0.6485,"acc":0.5118},{"lo":0.7,"w":0.0686,"conf":0.7514,"acc":0.5609},{"lo":0.8,"w":0.1059,"conf":0.8553,"acc":0.6061},{"lo":0.9,"w":0.5508,"conf":0.9787,"acc":0.8368}],"areas":{"knowledge":{"ece":0.1652,"acc":0.4986,"conf":0.6638},"language":{"ece":0.0721,"acc":0.819,"conf":0.8902},"retrieval":{"ece":0.1568,"acc":0.7617,"conf":0.9155},"tools":{"ece":0.0896,"acc":0.9063,"conf":0.9903},"arts":{"ece":0.3097,"acc":0.4823,"conf":0.7849}}},"gaps":null,"latency":{"median":16.4,"p95":70.7,"mean":27.5},"results":{"1":{"score":0.9303,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":25.2},"2":{"score":0.6404,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":298.9},"3":{"score":0.8071,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":201.8},"4":{"score":0.9127,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":19.9},"5":{"score":0.9302,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":25.3},"6":{"score":0.675,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":65.5},"9":{"score":0.875,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":231.2},"10":{"score":0.5294,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":24.5},"11":{"score":0.8387,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":586.9},"12":{"score":0.5553,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":11.8},"20":{"score":0.9488,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.8},"21":{"score":0.543,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":11.8},"22":{"score":0.0145,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":126.6},"23":{"score":0.5108,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.2},"24":{"score":0.4472,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":11.8},"25":{"score":0.2602,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":11.9},"26":{"score":0.8283,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":11.8},"27":{"score":0.6613,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":11.8},"28":{"score":0.7553,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":11.8},"29":{"score":0.8451,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":12.1},"30":{"score":0.202,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0716,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.1},"32":{"score":0.6277,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":21.8},"33":{"score":0.1661,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":50.9},"36":{"score":0.3653,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":590.1},"37":{"score":0.4947,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.7},"38":{"score":0.3496,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":111.3},"39":{"score":0.9198,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":12.7},"40":{"score":0.5514,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":12.5},"41":{"score":0.7806,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":12.6},"42":{"score":0.6714,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.1},"43":{"score":0.3368,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":12.3},"44":{"score":0.8246,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":12.3},"45":{"score":0.1497,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":12.4},"50":{"score":0.4057,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":19.7},"56":{"score":0.5185,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":33},"57":{"score":0.207,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":12.5},"58":{"score":0.4311,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13},"59":{"score":0.7605,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":18.9},"61":{"score":0.8665,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":13.8},"62":{"score":0.8059,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":18.7},"64":{"score":0.6761,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":12.7}},"benchmarks":{"1":{"raw":0.9303,"skill":0.906,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8034,"private_eq":0.8553},"2":{"raw":0.6404,"skill":0.5847,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6464,"private_eq":0.6964},"3":{"raw":0.8071,"skill":0.8034,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6704,"private_eq":0.6793},"4":{"raw":0.9127,"skill":0.9116,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7805,"private_eq":0.7848},"5":{"raw":0.9302,"skill":0.9298,"coverage":1,"random":0.006,"tracks":[],"combined":0.7025,"private_eq":0.7067},"6":{"raw":0.675,"skill":0.238,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.875,"skill":0.875,"coverage":1,"random":0,"tracks":[],"combined":0.3992,"private_eq":0.405},"10":{"raw":0.5294,"skill":0.217,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8387,"skill":0.7668,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5553,"skill":0.3339,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3965,"private_eq":0.601},"20":{"raw":0.9465,"skill":0.8931,"coverage":1,"random":0.5,"tracks":[],"combined":0.8347,"private_eq":0.9178},"21":{"raw":0.543,"skill":0.086,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.014,"skill":0.0063,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5073,"skill":0.0147,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4472,"skill":0.2629,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2685,"combined":0.0144},"26":{"raw":0.8283,"skill":0.771,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.6613,"skill":0.5483,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7553,"skill":0.5106,"coverage":1,"random":0.5,"tracks":[],"combined":0.5268,"private_eq":0.7656},"29":{"raw":0.8451,"skill":0.7935,"coverage":1,"random":0.25,"tracks":[],"combined":0.7751,"private_eq":0.8325},"30":{"raw":0.202,"skill":0.0337,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2828,"random":0.25},{"track":"10-choice","raw":0.1213,"random":0.1}],"combined":0.0783,"private_eq":0.2474},"31":{"raw":0.0716,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0834},"32":{"raw":0.6277,"skill":0.4081,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1661,"skill":0.155,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0864,"private_eq":0.1077},"36":{"raw":0.3653,"skill":0.2819,"coverage":1,"random":0.116,"tracks":[],"combined":0.3066,"private_eq":0.3931},"37":{"raw":0.4947,"skill":0.3663,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4486,"private_eq":0.5646},"38":{"raw":0.3496,"skill":0.3288,"coverage":1,"random":0.031,"tracks":[],"combined":0.5279,"private_eq":0.5468},"39":{"raw":0.9198,"skill":0.882,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4177,"private_eq":0.6079},"40":{"raw":0.5514,"skill":0.4229,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0339,"headline":false},{"track":"A · English","score":0.5514,"headline":true},{"track":"C · Arabic pairs","score":0.49,"headline":false},{"track":"C · English pairs","score":0.68,"headline":false}],"combined":0.2844,"private_eq":0.4493},"41":{"raw":0.7806,"skill":0.6709,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0701,"private_eq":0.3864},"42":{"raw":0.6714,"skill":0.3607,"coverage":1,"random":0.486,"tracks":[],"combined":0.1648,"private_eq":0.5751},"43":{"raw":0.3368,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1021,"private_eq":0.4399},"44":{"raw":0.8246,"skill":0.6492,"coverage":1,"random":0.5,"tracks":[],"combined":0.6261,"private_eq":0.8147},"45":{"raw":0.1497,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4057,"skill":0.1375,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.5185,"skill":0.037,"coverage":1,"random":0.5,"tracks":[],"combined":0.0146,"private_eq":0.5124},"57":{"raw":0.207,"skill":0.1081,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2061,"combined":0.0978},"58":{"raw":0.4311,"skill":0.1754,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3678,"combined":0.0741},"59":{"raw":0.7605,"skill":0.5034,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1348,"private_eq":0.587},"61":{"raw":0.8665,"skill":0.733,"coverage":1,"random":0.5,"tracks":[],"combined":0.6096,"private_eq":0.8065},"62":{"raw":0.8059,"skill":0.7412,"coverage":1,"random":0.25,"tracks":[],"combined":0.4672,"private_eq":0.6042},"64":{"raw":0.6761,"skill":0.5951,"coverage":1,"random":0.2,"tracks":[],"combined":0.5549,"private_eq":0.6472}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #220).","Submitted public 0.3 run adopted (kit rescore 47.77). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.85% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":61.29,"balanced_skill":29.25,"breadth_skill":43.3,"public_skill":47.77},"frozen_scores":{"balanced_raw":61.29,"balanced_skill":47.77,"breadth_skill":43.3},"categories":[{"id":"knowledge","raw":0.3661,"skill":0.1846,"combined":0.1601,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6971,"skill":0.5626,"combined":0.3578,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6863,"skill":0.5582,"combined":0.4879,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.816,"skill":0.7874,"combined":0.6076,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5154,"skill":0.2888,"combined":0.3129,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"crivo-1b","pub":47.77,"same_skills":38.75,"new_domains":10.75,"v":29.25,"lab":false,"jev":false,"rank_v":172,"rank_pub":128,"name":"Crivo-1B","eq":{"S":38.15,"O":15.92},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":22.89,"new_domains":6.37}},"new_in_edition":true},{"engine":"crivo-150m-v2","name":"Crivo-150M v2","long_name":"Crivo-150M v2","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":149610241,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tardellirs/crivo-150m","code_url":null,"base_model":"cross-encoder/ettin-reranker-150m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-150m-v1","weights_repo":null,"served_checkpoint":null,"pr":220,"replaces":"crivo-150m","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.587,"conf":0.7024,"ece":0.1154,"brier":0.5356,"over95":0.0242,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0004,"conf":0.0956,"acc":0.1818},{"lo":0.1,"w":0.0111,"conf":0.1573,"acc":0.0534},{"lo":0.2,"w":0.0328,"conf":0.2628,"acc":0.1807},{"lo":0.3,"w":0.0767,"conf":0.35,"acc":0.2877},{"lo":0.4,"w":0.0781,"conf":0.4484,"acc":0.3431},{"lo":0.5,"w":0.166,"conf":0.5489,"acc":0.4664},{"lo":0.6,"w":0.1384,"conf":0.647,"acc":0.4795},{"lo":0.7,"w":0.1005,"conf":0.7477,"acc":0.6208},{"lo":0.8,"w":0.0994,"conf":0.8514,"acc":0.667},{"lo":0.9,"w":0.2964,"conf":0.9771,"acc":0.874}],"areas":{"knowledge":{"ece":0.1151,"acc":0.4278,"conf":0.5427},"language":{"ece":0.044,"acc":0.7213,"conf":0.7653},"retrieval":{"ece":0.2078,"acc":0.6529,"conf":0.8601},"tools":{"ece":0.1093,"acc":0.8441,"conf":0.9461},"arts":{"ece":0.1829,"acc":0.4569,"conf":0.6364}}},"gaps":null,"latency":{"median":13.4,"p95":24.5,"mean":16},"results":{"1":{"score":0.7633,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":19.1},"2":{"score":0.6277,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":124.9},"3":{"score":0.7067,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":92.3},"4":{"score":0.8583,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":19.9},"5":{"score":0.8955,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":21},"6":{"score":0.6871,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":31.1},"9":{"score":0.6364,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":91.6},"10":{"score":0.2005,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":19.8},"11":{"score":0.8025,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":236.1},"12":{"score":0.3829,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":17.7},"20":{"score":0.8646,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.8},"21":{"score":0.5046,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":17.6},"22":{"score":0.02,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":58.6},"23":{"score":0.4922,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.4},"24":{"score":0.3614,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":17.5},"25":{"score":0.2602,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":17.4},"26":{"score":0.6646,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":17.4},"27":{"score":0.407,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":17.4},"28":{"score":0.547,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":17.5},"29":{"score":0.5425,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":17.8},"30":{"score":0.1854,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0802,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":19.2},"32":{"score":0.504,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":20},"33":{"score":0.0388,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":22.1},"36":{"score":0.3096,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":245.9},"37":{"score":0.3741,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.1},"38":{"score":0.221,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":56},"39":{"score":0.854,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":17.8},"40":{"score":0.019,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":17.5},"41":{"score":0.7234,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":17.6},"42":{"score":0.5737,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":18.5},"43":{"score":0.4579,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":17.7},"44":{"score":0.6606,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.8},"45":{"score":0.1677,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":18},"50":{"score":0.3914,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":19.9},"56":{"score":0.5005,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":19.7},"57":{"score":0.1488,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":17.9},"58":{"score":0.3668,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":17.8},"59":{"score":0.6312,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":19.4},"61":{"score":0.8015,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":18.6},"62":{"score":0.6443,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":19.2},"64":{"score":0.5473,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":17.9}},"benchmarks":{"1":{"raw":0.7633,"skill":0.6804,"coverage":1,"random":0.2592,"tracks":[],"combined":0.4765,"private_eq":0.6159},"2":{"raw":0.6277,"skill":0.5701,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6201,"private_eq":0.6739},"3":{"raw":0.7067,"skill":0.701,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6442,"private_eq":0.6539},"4":{"raw":0.8583,"skill":0.8565,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7193,"private_eq":0.725},"5":{"raw":0.8955,"skill":0.8949,"coverage":1,"random":0.006,"tracks":[],"combined":0.6107,"private_eq":0.6164},"6":{"raw":0.6871,"skill":0.2664,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6364,"skill":0.6364,"coverage":1,"random":0,"tracks":[],"combined":0.3031,"private_eq":0.31},"10":{"raw":0.2005,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8025,"skill":0.7144,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3829,"skill":0.0756,"coverage":1,"random":0.3324,"tracks":[],"combined":0.1671,"private_eq":0.4496},"20":{"raw":0.8649,"skill":0.7298,"coverage":1,"random":0.5,"tracks":[],"combined":0.6468,"private_eq":0.8249},"21":{"raw":0.5046,"skill":0.0091,"coverage":1,"random":0.5,"tracks":[],"combined":0.0051,"private_eq":0.5077},"22":{"raw":0.0186,"skill":0.011,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5021,"skill":0.0042,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3614,"skill":0.1485,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2727,"combined":0.0201},"26":{"raw":0.6646,"skill":0.5527,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.407,"skill":0.2091,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.547,"skill":0.094,"coverage":1,"random":0.5,"tracks":[],"combined":0.1333,"private_eq":0.5711},"29":{"raw":0.5425,"skill":0.39,"coverage":1,"random":0.25,"tracks":[],"combined":0.2356,"private_eq":0.4325},"30":{"raw":0.1854,"skill":0.0124,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2578,"random":0.25},{"track":"10-choice","raw":0.113,"random":0.1}],"combined":0.0502,"private_eq":0.2245},"31":{"raw":0.0802,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.504,"skill":0.2114,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0388,"skill":0.026,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.3096,"skill":0.219,"coverage":1,"random":0.116,"tracks":[],"combined":0.2578,"private_eq":0.3505},"37":{"raw":0.3741,"skill":0.215,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1306,"private_eq":0.3139},"38":{"raw":0.221,"skill":0.1961,"coverage":1,"random":0.031,"tracks":[],"combined":0.2593,"private_eq":0.2895},"39":{"raw":0.854,"skill":0.7853,"coverage":1,"random":0.3201,"tracks":[],"combined":0.2406,"private_eq":0.4889},"40":{"raw":0.019,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.019,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.295,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7234,"skill":0.5851,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0356,"private_eq":0.3637},"42":{"raw":0.5737,"skill":0.1706,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4579,"skill":0.1399,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.6606,"skill":0.3212,"coverage":1,"random":0.5,"tracks":[],"combined":0.1764,"private_eq":0.5924},"45":{"raw":0.1677,"skill":0.0043,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3914,"skill":0.1167,"coverage":1,"random":0.311,"tracks":[],"combined":0.1596,"private_eq":0.4269},"56":{"raw":0.5005,"skill":0.001,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1488,"skill":0.0426,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2061,"combined":0.0978},"58":{"raw":0.3668,"skill":0.0822,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3426,"combined":0.0371},"59":{"raw":0.6312,"skill":0.2353,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8015,"skill":0.603,"coverage":1,"random":0.5,"tracks":[],"combined":0.4847,"private_eq":0.7448},"62":{"raw":0.6443,"skill":0.5257,"coverage":1,"random":0.25,"tracks":[],"combined":0.3713,"private_eq":0.5331},"64":{"raw":0.5473,"skill":0.4341,"coverage":1,"random":0.2,"tracks":[],"combined":0.4385,"private_eq":0.5551}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #220).","Replaces the earlier board entry crivo-150m (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 34.38). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.26% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":51.32,"balanced_skill":18.26,"breadth_skill":29.64,"public_skill":34.38},"frozen_scores":{"balanced_raw":51.32,"balanced_skill":34.38,"breadth_skill":29.64},"categories":[{"id":"knowledge","raw":0.3096,"skill":0.0829,"combined":0.0491,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.525,"skill":0.3453,"combined":0.1175,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6291,"skill":0.4823,"combined":0.3803,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6801,"skill":0.6278,"combined":0.4888,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4715,"skill":0.2175,"combined":0.2479,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"crivo-150m-v2","pub":34.38,"same_skills":24.93,"new_domains":5.48,"v":18.26,"lab":false,"jev":false,"rank_v":201,"rank_pub":174,"name":"Crivo-150M v2","eq":{"S":24.16,"O":9.41},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":14.5,"new_domains":3.76}},"new_in_edition":true},{"engine":"crivo-68m","name":"Crivo-68M","long_name":"Crivo-68M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":68410881,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tardellirs/crivo-68m","code_url":null,"base_model":"cross-encoder/ettin-reranker-68m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-68m-v1","weights_repo":null,"served_checkpoint":null,"pr":220,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5648,"conf":0.6681,"ece":0.1033,"brier":0.5453,"over95":0.0207,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0007,"conf":0.0936,"acc":0.0535},{"lo":0.1,"w":0.0192,"conf":0.1617,"acc":0.0143},{"lo":0.2,"w":0.0522,"conf":0.2587,"acc":0.1529},{"lo":0.3,"w":0.1072,"conf":0.3493,"acc":0.2844},{"lo":0.4,"w":0.0875,"conf":0.4479,"acc":0.3722},{"lo":0.5,"w":0.1754,"conf":0.5444,"acc":0.4845},{"lo":0.6,"w":0.1124,"conf":0.6461,"acc":0.5366},{"lo":0.7,"w":0.0821,"conf":0.7484,"acc":0.5905},{"lo":0.8,"w":0.0842,"conf":0.8501,"acc":0.6382},{"lo":0.9,"w":0.2792,"conf":0.9804,"acc":0.8812}],"areas":{"knowledge":{"ece":0.12,"acc":0.3818,"conf":0.5018},"language":{"ece":0.0364,"acc":0.6836,"conf":0.7201},"retrieval":{"ece":0.1334,"acc":0.7136,"conf":0.8466},"tools":{"ece":0.138,"acc":0.8109,"conf":0.9489},"arts":{"ece":0.1455,"acc":0.4503,"conf":0.5959}}},"gaps":null,"latency":{"median":12.4,"p95":17.6,"mean":14.1},"results":{"1":{"score":0.8253,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":17.6},"2":{"score":0.5999,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":80.6},"3":{"score":0.6043,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":61.3},"4":{"score":0.8595,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":18.2},"5":{"score":0.8765,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":19.3},"6":{"score":0.5702,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":21.9},"9":{"score":0.75,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":56.5},"10":{"score":0.4576,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":18},"11":{"score":0.7981,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":147.3},"12":{"score":0.3596,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":16},"20":{"score":0.867,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16},"21":{"score":0.4954,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":15.8},"22":{"score":0.0145,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":42.1},"23":{"score":0.4974,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.6},"24":{"score":0.3262,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.8},"25":{"score":0.25,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":16},"26":{"score":0.5829,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":15.8},"27":{"score":0.3618,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":15.9},"28":{"score":0.5296,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":15.7},"29":{"score":0.3926,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":16.2},"30":{"score":0.1766,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0734,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.4},"32":{"score":0.4973,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":18.2},"33":{"score":0.0067,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":18.4},"36":{"score":0.3011,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":160.1},"37":{"score":0.4075,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.7},"38":{"score":0.2213,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":48.3},"39":{"score":0.8531,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":16.2},"40":{"score":0,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":15.8},"41":{"score":0.6955,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":16.1},"42":{"score":0.5522,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":16.8},"43":{"score":0.4474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":16},"44":{"score":0.5404,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.1},"45":{"score":0.1537,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":16.3},"50":{"score":0.3962,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":18.1},"56":{"score":0.502,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":17.9},"57":{"score":0.1418,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":16.1},"58":{"score":0.3521,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":16},"59":{"score":0.6436,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":17.6},"61":{"score":0.8143,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":16.9},"62":{"score":0.6525,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":17.3},"64":{"score":0.5114,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":16.2}},"benchmarks":{"1":{"raw":0.8253,"skill":0.7641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5447,"private_eq":0.6658},"2":{"raw":0.5999,"skill":0.538,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6295,"private_eq":0.682},"3":{"raw":0.6043,"skill":0.5967,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6126,"private_eq":0.6233},"4":{"raw":0.8595,"skill":0.8577,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7006,"private_eq":0.7068},"5":{"raw":0.8765,"skill":0.8758,"coverage":1,"random":0.006,"tracks":[],"combined":0.6141,"private_eq":0.6198},"6":{"raw":0.5702,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.75,"skill":0.75,"coverage":1,"random":0,"tracks":[],"combined":0.3511,"private_eq":0.3575},"10":{"raw":0.4576,"skill":0.0975,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7981,"skill":0.708,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3596,"skill":0.0407,"coverage":1,"random":0.3324,"tracks":[],"combined":0.1801,"private_eq":0.4582},"20":{"raw":0.8636,"skill":0.7272,"coverage":1,"random":0.5,"tracks":[],"combined":0.6903,"private_eq":0.8464},"21":{"raw":0.4954,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.014,"skill":0.0063,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5041,"skill":0.0081,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3262,"skill":0.1016,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.25,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2664,"combined":0.0116},"26":{"raw":0.5829,"skill":0.4437,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.3618,"skill":0.1488,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5296,"skill":0.0592,"coverage":1,"random":0.5,"tracks":[],"combined":0.0577,"private_eq":0.5337},"29":{"raw":0.3926,"skill":0.1901,"coverage":1,"random":0.25,"tracks":[],"combined":0.206,"private_eq":0.4105},"30":{"raw":0.1766,"skill":0.0051,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2441,"random":0.25},{"track":"10-choice","raw":0.1092,"random":0.1}],"combined":0.0328,"private_eq":0.2103},"31":{"raw":0.0734,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.4973,"skill":0.2008,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.3746},"33":{"raw":0.0067,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0042,"private_eq":0.0275},"36":{"raw":0.3011,"skill":0.2093,"coverage":1,"random":0.116,"tracks":[],"combined":0.2115,"private_eq":0.31},"37":{"raw":0.4075,"skill":0.2569,"coverage":1,"random":0.2027,"tracks":[],"combined":0.2262,"private_eq":0.3893},"38":{"raw":0.2213,"skill":0.1964,"coverage":1,"random":0.031,"tracks":[],"combined":0.3122,"private_eq":0.3401},"39":{"raw":0.8531,"skill":0.7839,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3897,"private_eq":0.5891},"40":{"raw":0,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.26,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.6955,"skill":0.5432,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5522,"skill":0.1288,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4474,"skill":0.1232,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3722},"44":{"raw":0.5404,"skill":0.0808,"coverage":1,"random":0.5,"tracks":[],"combined":0.0158,"private_eq":0.513},"45":{"raw":0.1537,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3962,"skill":0.1237,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.502,"skill":0.004,"coverage":1,"random":0.5,"tracks":[],"combined":0.0492,"private_eq":0.5295},"57":{"raw":0.1418,"skill":0.0348,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2012,"combined":0.0922},"58":{"raw":0.3521,"skill":0.0609,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3433,"combined":0.0382},"59":{"raw":0.6436,"skill":0.261,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8143,"skill":0.6286,"coverage":1,"random":0.5,"tracks":[],"combined":0.5017,"private_eq":0.7532},"62":{"raw":0.6525,"skill":0.5367,"coverage":1,"random":0.25,"tracks":[],"combined":0.3439,"private_eq":0.5128},"64":{"raw":0.5114,"skill":0.3892,"coverage":1,"random":0.2,"tracks":[],"combined":0.3998,"private_eq":0.5245}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #220).","Submitted public 0.3 run adopted (kit rescore 32.82). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.41% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":50.15,"balanced_skill":19.51,"breadth_skill":26.96,"public_skill":32.82},"frozen_scores":{"balanced_raw":50.15,"balanced_skill":32.82,"breadth_skill":26.96},"categories":[{"id":"knowledge","raw":0.2879,"skill":0.0496,"combined":0.0251,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4964,"skill":0.3083,"combined":0.1337,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6316,"skill":0.488,"combined":0.3952,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6885,"skill":0.6403,"combined":0.5025,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4641,"skill":0.2091,"combined":0.2428,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"crivo-68m","pub":32.82,"same_skills":25.27,"new_domains":7.59,"v":19.51,"lab":false,"jev":false,"rank_v":198,"rank_pub":179,"name":"Crivo-68M","eq":{"S":24.51,"O":12.02},"rank_tie":46,"tied":true,"parts":{"public":0,"same_skills":14.71,"new_domains":4.81}},"new_in_edition":true},{"engine":"crivo-32m-v2","name":"Crivo-32M v2","long_name":"Crivo-32M v2","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":32033665,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tardellirs/crivo-32m","code_url":null,"base_model":"cross-encoder/ettin-reranker-32m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-32m-v1","weights_repo":null,"served_checkpoint":null,"pr":220,"replaces":"crivo-32m","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5269,"conf":0.6599,"ece":0.133,"brier":0.5777,"over95":0.0282,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0014,"conf":0.0808,"acc":0},{"lo":0.1,"w":0.0177,"conf":0.1651,"acc":0.0511},{"lo":0.2,"w":0.0572,"conf":0.2556,"acc":0.1032},{"lo":0.3,"w":0.1042,"conf":0.3488,"acc":0.2409},{"lo":0.4,"w":0.0885,"conf":0.4478,"acc":0.3042},{"lo":0.5,"w":0.1963,"conf":0.542,"acc":0.4626},{"lo":0.6,"w":0.1028,"conf":0.6457,"acc":0.4733},{"lo":0.7,"w":0.081,"conf":0.7487,"acc":0.568},{"lo":0.8,"w":0.0795,"conf":0.8511,"acc":0.6699},{"lo":0.9,"w":0.2714,"conf":0.977,"acc":0.845}],"areas":{"knowledge":{"ece":0.1937,"acc":0.3182,"conf":0.512},"language":{"ece":0.0529,"acc":0.638,"conf":0.6903},"retrieval":{"ece":0.1483,"acc":0.7142,"conf":0.8622},"tools":{"ece":0.1399,"acc":0.7963,"conf":0.9363},"arts":{"ece":0.1406,"acc":0.4327,"conf":0.5733}}},"gaps":null,"latency":{"median":7.6,"p95":11.6,"mean":8.9},"results":{"1":{"score":0.7704,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":27.5},"2":{"score":0.5309,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":87},"3":{"score":0.5709,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":80.5},"4":{"score":0.8613,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":27.7},"5":{"score":0.8479,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":29.6},"6":{"score":0.5418,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":35.2},"9":{"score":0.2386,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":65.4},"10":{"score":0.4742,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":27.8},"11":{"score":0.712,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":150.1},"12":{"score":0.3458,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":26.8},"20":{"score":0.8478,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.9},"21":{"score":0.492,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":26.7},"22":{"score":0.0005,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":55.1},"23":{"score":0.5204,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.1},"24":{"score":0.2994,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":26.6},"25":{"score":0.2755,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":26.7},"26":{"score":0.4823,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":26.6},"27":{"score":0.2944,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":26.6},"28":{"score":0.5051,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":26.6},"29":{"score":0.3295,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":26.7},"30":{"score":0.1782,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0858,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.3},"32":{"score":0.3949,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":27.8},"33":{"score":0.0012,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":29.7},"36":{"score":0.2596,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":163.2},"37":{"score":0.3601,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.9},"38":{"score":0.1725,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":72.4},"39":{"score":0.7601,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":26.7},"40":{"score":0.0636,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":26.6},"41":{"score":0.649,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":26.7},"42":{"score":0.5133,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":27},"43":{"score":0.4263,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":26.7},"44":{"score":0.515,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.7},"45":{"score":0.1537,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":26.8},"50":{"score":0.3711,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":27.9},"56":{"score":0.4635,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":27.6},"57":{"score":0.1213,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":26.7},"58":{"score":0.3637,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":26.7},"59":{"score":0.6609,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":27.6},"61":{"score":0.7837,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":26.9},"62":{"score":0.6235,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":27.3},"64":{"score":0.4621,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":26.7}},"benchmarks":{"1":{"raw":0.7704,"skill":0.69,"coverage":1,"random":0.2592,"tracks":[],"combined":0.4697,"private_eq":0.6109},"2":{"raw":0.5309,"skill":0.4583,"coverage":1,"random":0.1341,"tracks":[],"combined":0.5273,"private_eq":0.5945},"3":{"raw":0.5709,"skill":0.5626,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3187,"private_eq":0.3382},"4":{"raw":0.8613,"skill":0.8595,"coverage":1,"random":0.0127,"tracks":[],"combined":0.5822,"private_eq":0.5912},"5":{"raw":0.8479,"skill":0.847,"coverage":1,"random":0.006,"tracks":[],"combined":0.5537,"private_eq":0.5604},"6":{"raw":0.5418,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2386,"skill":0.2386,"coverage":1,"random":0,"tracks":[],"combined":0.1134,"private_eq":0.1225},"10":{"raw":0.4742,"skill":0.1251,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.712,"skill":0.5835,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.3458,"skill":0.0201,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.8414,"skill":0.6827,"coverage":1,"random":0.5,"tracks":[],"combined":0.7336,"private_eq":0.8678},"21":{"raw":0.492,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0013,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5225,"skill":0.045,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2994,"skill":0.0659,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2529,"combined":0},"26":{"raw":0.4823,"skill":0.3095,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.2944,"skill":0.0589,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5051,"skill":0.0102,"coverage":1,"random":0.5,"tracks":[],"combined":0.0678,"private_eq":0.5387},"29":{"raw":0.3295,"skill":0.106,"coverage":1,"random":0.25,"tracks":[],"combined":0.1366,"private_eq":0.3591},"30":{"raw":0.1782,"skill":0.0062,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2593,"random":0.25},{"track":"10-choice","raw":0.097,"random":0.1}],"combined":0.0041,"private_eq":0.1869},"31":{"raw":0.0858,"skill":0.0043,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0041,"private_eq":0.0952},"32":{"raw":0.3949,"skill":0.038,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.0012,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.2596,"skill":0.1624,"coverage":1,"random":0.116,"tracks":[],"combined":0.1717,"private_eq":0.2752},"37":{"raw":0.3601,"skill":0.1974,"coverage":1,"random":0.2027,"tracks":[],"combined":0.0097,"private_eq":0.2186},"38":{"raw":0.1725,"skill":0.146,"coverage":1,"random":0.031,"tracks":[],"combined":0.2532,"private_eq":0.2836},"39":{"raw":0.7601,"skill":0.6472,"coverage":1,"random":0.3201,"tracks":[],"combined":0.433,"private_eq":0.6182},"40":{"raw":0.0636,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.0636,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.43,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.649,"skill":0.4735,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5133,"skill":0.0531,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4263,"skill":0.0898,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.515,"skill":0.03,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"45":{"raw":0.1537,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3711,"skill":0.0873,"coverage":1,"random":0.311,"tracks":[],"combined":0.0943,"private_eq":0.3824},"56":{"raw":0.4635,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1213,"skill":0.0117,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1348,"combined":0.0167},"58":{"raw":0.3637,"skill":0.0777,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3103,"combined":0},"59":{"raw":0.6609,"skill":0.2969,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.7837,"skill":0.5674,"coverage":1,"random":0.5,"tracks":[],"combined":0.3939,"private_eq":0.6999},"62":{"raw":0.6235,"skill":0.498,"coverage":1,"random":0.25,"tracks":[],"combined":0.2413,"private_eq":0.4367},"64":{"raw":0.4621,"skill":0.3276,"coverage":1,"random":0.2,"tracks":[],"combined":0.3416,"private_eq":0.4785}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #220).","Replaces the earlier board entry crivo-32m (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 27.34). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.65% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":45.75,"balanced_skill":14.23,"breadth_skill":22.25,"public_skill":27.34},"frozen_scores":{"balanced_raw":45.75,"balanced_skill":27.34,"breadth_skill":22.25},"categories":[{"id":"knowledge","raw":0.2754,"skill":0.0274,"combined":0.0056,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4619,"skill":0.2502,"combined":0.0947,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6015,"skill":0.4557,"combined":0.2973,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.556,"skill":0.4996,"combined":0.3385,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4484,"skill":0.1904,"combined":0.2297,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"crivo-32m-v2","pub":27.34,"same_skills":18.44,"new_domains":5.29,"v":14.23,"lab":false,"jev":false,"rank_v":217,"rank_pub":199,"name":"Crivo-32M v2","eq":{"S":17.6,"O":9.18},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":10.56,"new_domains":3.67}},"new_in_edition":true},{"engine":"crivo-17m","name":"Crivo-17M","long_name":"Crivo-17M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":16865025,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tardellirs/crivo-17m","code_url":null,"base_model":"cross-encoder/ettin-reranker-17m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-17m-v1","weights_repo":null,"served_checkpoint":null,"pr":220,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5054,"conf":0.5855,"ece":0.0801,"brier":0.5765,"over95":0.0196,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0338,"conf":0.0464,"acc":0.0222},{"lo":0.1,"w":0.0196,"conf":0.1487,"acc":0.0504},{"lo":0.2,"w":0.0498,"conf":0.2683,"acc":0.1832},{"lo":0.3,"w":0.1468,"conf":0.3488,"acc":0.2539},{"lo":0.4,"w":0.0923,"conf":0.4465,"acc":0.3187},{"lo":0.5,"w":0.2475,"conf":0.5419,"acc":0.4957},{"lo":0.6,"w":0.1009,"conf":0.6473,"acc":0.6099},{"lo":0.7,"w":0.0836,"conf":0.7477,"acc":0.6874},{"lo":0.8,"w":0.0551,"conf":0.8472,"acc":0.7379},{"lo":0.9,"w":0.1706,"conf":0.9767,"acc":0.8529}],"areas":{"knowledge":{"ece":0.1298,"acc":0.3052,"conf":0.435},"language":{"ece":0.0528,"acc":0.6278,"conf":0.6142},"retrieval":{"ece":0.1229,"acc":0.6672,"conf":0.7901},"tools":{"ece":0.1914,"acc":0.7246,"conf":0.9101},"arts":{"ece":0.0557,"acc":0.4246,"conf":0.4765}}},"gaps":null,"latency":{"median":5.9,"p95":10,"mean":7.1},"results":{"1":{"score":0.6122,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":8.7},"2":{"score":0.4763,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":32.3},"3":{"score":0.5,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":25.1},"4":{"score":0.7961,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":9.4},"5":{"score":0.7971,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":10.6},"6":{"score":0.5306,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":11.1},"9":{"score":0.0341,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":18.4},"10":{"score":0.3865,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":9.3},"11":{"score":0.7163,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":40.4},"12":{"score":0.2927,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":7.3},"20":{"score":0.8078,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.4},"21":{"score":0.5118,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":7.1},"22":{"score":0.0185,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":18.6},"23":{"score":0.5084,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.7},"24":{"score":0.282,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":7.2},"25":{"score":0.2194,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":7.4},"26":{"score":0.4158,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":7.1},"27":{"score":0.2526,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":7.2},"28":{"score":0.5217,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":7.1},"29":{"score":0.3057,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":7.6},"30":{"score":0.1816,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.075,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":8.8},"32":{"score":0.379,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":9.6},"33":{"score":0.0024,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":9.9},"36":{"score":0.2339,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":62.5},"37":{"score":0.2581,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.8},"38":{"score":0.1081,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":30.8},"39":{"score":0.6746,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":7.5},"40":{"score":0,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":7.1},"41":{"score":0.598,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":7.4},"42":{"score":0.496,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":8},"43":{"score":0.4018,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":7.4},"44":{"score":0.5,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.4},"45":{"score":0.1437,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":7.6},"50":{"score":0.364,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":9.3},"56":{"score":0.5805,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":9.2},"57":{"score":0.1099,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":7.5},"58":{"score":0.3241,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":7.4},"59":{"score":0.6523,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":8.9},"61":{"score":0.6745,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":8},"62":{"score":0.5359,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":8.6},"64":{"score":0.4148,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":7.5}},"benchmarks":{"1":{"raw":0.6122,"skill":0.4764,"coverage":1,"random":0.2592,"tracks":[],"combined":0.3608,"private_eq":0.5311},"2":{"raw":0.4763,"skill":0.3952,"coverage":1,"random":0.1341,"tracks":[],"combined":0.4491,"private_eq":0.5275},"3":{"raw":0.5,"skill":0.4904,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3713,"private_eq":0.3892},"4":{"raw":0.7961,"skill":0.7935,"coverage":1,"random":0.0127,"tracks":[],"combined":0.4646,"private_eq":0.4764},"5":{"raw":0.7971,"skill":0.7959,"coverage":1,"random":0.006,"tracks":[],"combined":0.3991,"private_eq":0.4085},"6":{"raw":0.5306,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0341,"skill":0.0341,"coverage":1,"random":0,"tracks":[],"combined":0.0198,"private_eq":0.03},"10":{"raw":0.3865,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7163,"skill":0.5898,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.2927,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.8078,"skill":0.6157,"coverage":1,"random":0.5,"tracks":[],"combined":0.5889,"private_eq":0.7963},"21":{"raw":0.5118,"skill":0.0236,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0175,"skill":0.0098,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0164},"23":{"raw":0.5131,"skill":0.0262,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.282,"skill":0.0427,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2194,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2727,"combined":0.0201},"26":{"raw":0.4158,"skill":0.2209,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.2526,"skill":0.0032,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5217,"skill":0.0434,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5038},"29":{"raw":0.3057,"skill":0.0743,"coverage":1,"random":0.25,"tracks":[],"combined":0.1565,"private_eq":0.3738},"30":{"raw":0.1816,"skill":0.0086,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2616,"random":0.25},{"track":"10-choice","raw":0.1016,"random":0.1}],"combined":0.058,"private_eq":0.2309},"31":{"raw":0.075,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0824},"32":{"raw":0.379,"skill":0.0127,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0024,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.2339,"skill":0.1333,"coverage":1,"random":0.116,"tracks":[],"combined":0.1455,"private_eq":0.2523},"37":{"raw":0.2581,"skill":0.0694,"coverage":1,"random":0.2027,"tracks":[],"combined":0.0539,"private_eq":0.2535},"38":{"raw":0.1081,"skill":0.0796,"coverage":1,"random":0.031,"tracks":[],"combined":0.0337,"private_eq":0.0734},"39":{"raw":0.6746,"skill":0.5214,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3986,"private_eq":0.5951},"40":{"raw":0,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0,"headline":true},{"track":"C · Arabic pairs","score":0.435,"headline":false},{"track":"C · English pairs","score":0.345,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.598,"skill":0.397,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.496,"skill":0.0195,"coverage":1,"random":0.486,"tracks":[],"combined":0.2329,"private_eq":0.6097},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.1121,"private_eq":0.5606},"45":{"raw":0.1437,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.364,"skill":0.0769,"coverage":1,"random":0.311,"tracks":[],"combined":0.0437,"private_eq":0.3479},"56":{"raw":0.5805,"skill":0.161,"coverage":1,"random":0.5,"tracks":[],"combined":0.0146,"private_eq":0.5124},"57":{"raw":0.1099,"skill":0,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1231,"combined":0.0034},"58":{"raw":0.3241,"skill":0.0203,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3101,"combined":0},"59":{"raw":0.6523,"skill":0.2791,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6745,"skill":0.349,"coverage":1,"random":0.5,"tracks":[],"combined":0.3429,"private_eq":0.6747},"62":{"raw":0.5359,"skill":0.3812,"coverage":1,"random":0.25,"tracks":[],"combined":0.1659,"private_eq":0.3808},"64":{"raw":0.4148,"skill":0.2685,"coverage":1,"random":0.2,"tracks":[],"combined":0.2154,"private_eq":0.3787}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #220).","Submitted public 0.3 run adopted (kit rescore 22.15). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.46% on 831 rows (bar 98.5%)."],"scores":{"balanced_raw":41.31,"balanced_skill":11.65,"breadth_skill":18.1,"public_skill":22.15},"frozen_scores":{"balanced_raw":41.31,"balanced_skill":22.15,"breadth_skill":18.1},"categories":[{"id":"knowledge","raw":0.2602,"skill":0.0124,"combined":0.013,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4216,"skill":0.2102,"combined":0.0963,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5615,"skill":0.401,"combined":0.2458,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.4409,"skill":0.3649,"combined":0.2802,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4382,"skill":0.1701,"combined":0.1672,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"crivo-17m","pub":22.15,"same_skills":15.88,"new_domains":3.21,"v":11.65,"lab":false,"jev":false,"rank_v":225,"rank_pub":208,"name":"Crivo-17M","eq":{"S":15.01,"O":6.61},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":9.01,"new_domains":2.64}},"new_in_edition":true},{"engine":"mola-150m","name":"Mola-150M","long_name":"Mola-150M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":149311489,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/codavidgarcia/mola-150m","code_url":null,"base_model":"cross-encoder/ettin-reranker-150m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-150m-v1","weights_repo":null,"served_checkpoint":null,"pr":187,"new":true},"completed":110201,"counts":{"ok":110195,"unsupported":6,"error":0},"lifted":false,"calibration":{"acc":0.5764,"conf":0.6566,"ece":0.0802,"brier":0.5222,"over95":0.0104,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0528,"conf":0.0743,"acc":0.0393},{"lo":0.1,"w":0.0124,"conf":0.1313,"acc":0.1092},{"lo":0.2,"w":0.0238,"conf":0.2659,"acc":0.1769},{"lo":0.3,"w":0.0708,"conf":0.3545,"acc":0.2555},{"lo":0.4,"w":0.0818,"conf":0.4505,"acc":0.403},{"lo":0.5,"w":0.1784,"conf":0.548,"acc":0.5214},{"lo":0.6,"w":0.1423,"conf":0.6478,"acc":0.551},{"lo":0.7,"w":0.0954,"conf":0.7481,"acc":0.6036},{"lo":0.8,"w":0.0953,"conf":0.8508,"acc":0.6604},{"lo":0.9,"w":0.2469,"conf":0.9732,"acc":0.9141}],"areas":{"knowledge":{"ece":0.0995,"acc":0.4641,"conf":0.5636},"language":{"ece":0.0427,"acc":0.7053,"conf":0.7444},"retrieval":{"ece":0.0345,"acc":0.7532,"conf":0.7845},"tools":{"ece":0.1934,"acc":0.618,"conf":0.8096},"arts":{"ece":0.0895,"acc":0.4024,"conf":0.4901}}},"gaps":null,"latency":{"median":12.8,"p95":21.1,"mean":15.5},"results":{"1":{"score":0.9097,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":8.3},"2":{"score":0.5713,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":118.2},"3":{"score":0.4213,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":97.9},"4":{"score":0.8868,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":8.5},"5":{"score":0.8953,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":8.6},"6":{"score":0.5612,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":16.5},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":14.5},"10":{"score":0.6802,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":10.4},"11":{"score":0.7815,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":27},"12":{"score":0.3863,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":6.9},"20":{"score":0.5186,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.8},"21":{"score":0.5571,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":6.3},"22":{"score":0.03,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":35.2},"23":{"score":0.4782,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.3},"24":{"score":0.3803,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":6.7},"25":{"score":0.199,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":7.2},"26":{"score":0.702,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6.4},"27":{"score":0.5384,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":6.6},"28":{"score":0.6014,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":6.3},"29":{"score":0.5146,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":7.3},"30":{"score":0.2301,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0838,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":8},"32":{"score":0.5572,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":9.6},"33":{"score":0.0867,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":9.1},"36":{"score":0.2833,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":280.8},"37":{"score":0.4668,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.5},"38":{"score":0.1989,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":61.9},"39":{"score":0.7187,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":7.2},"40":{"score":0.2852,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":6.6},"41":{"score":0.7485,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":7},"42":{"score":0.5698,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":7.7},"43":{"score":0.4474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":6.8},"44":{"score":0.7212,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.9},"45":{"score":0.164,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":7.4},"50":{"score":0.4081,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":8.9},"56":{"score":0.501,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":8},"57":{"score":0.1729,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":7},"58":{"score":0.3684,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":6.9},"59":{"score":0.5315,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":8.4},"61":{"score":0.849,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":7.6},"62":{"score":0.744,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":8.3},"64":{"score":0.4223,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":7}},"benchmarks":{"1":{"raw":0.9097,"skill":0.8781,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7831,"private_eq":0.8404},"2":{"raw":0.5671,"skill":0.5001,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.5888,"private_eq":0.6471},"3":{"raw":0.4213,"skill":0.4102,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2452,"private_eq":0.2669},"4":{"raw":0.8868,"skill":0.8853,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7028,"private_eq":0.7089},"5":{"raw":0.8953,"skill":0.8947,"coverage":1,"random":0.006,"tracks":[],"combined":0.5552,"private_eq":0.5619},"6":{"raw":0.5612,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.6802,"skill":0.4679,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7689,"skill":0.6658,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.3863,"skill":0.0807,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3034,"private_eq":0.5396},"20":{"raw":0.5184,"skill":0.0367,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.5571,"skill":0.1142,"coverage":1,"random":0.5,"tracks":[],"combined":0.1058,"private_eq":0.5575},"22":{"raw":0.0293,"skill":0.0217,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0081},"23":{"raw":0.4632,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3803,"skill":0.1737,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.199,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2519,"combined":0},"26":{"raw":0.702,"skill":0.6026,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5384,"skill":0.3844,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6014,"skill":0.2028,"coverage":1,"random":0.5,"tracks":[],"combined":0.2796,"private_eq":0.6434},"29":{"raw":0.5146,"skill":0.3528,"coverage":1,"random":0.25,"tracks":[],"combined":0.2356,"private_eq":0.4325},"30":{"raw":0.2301,"skill":0.0691,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3207,"random":0.25},{"track":"10-choice","raw":0.1395,"random":0.1}],"combined":0.0317,"private_eq":0.2094},"31":{"raw":0.0838,"skill":0.0021,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0903},"32":{"raw":0.5572,"skill":0.296,"coverage":1,"random":0.371,"tracks":[],"combined":0.0107,"private_eq":0.3842},"33":{"raw":0.0867,"skill":0.0746,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0146,"private_eq":0.0376},"36":{"raw":0.2833,"skill":0.1892,"coverage":1,"random":0.116,"tracks":[],"combined":0.179,"private_eq":0.2816},"37":{"raw":0.4668,"skill":0.3312,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4559,"private_eq":0.5703},"38":{"raw":0.1989,"skill":0.1733,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.7187,"skill":0.5863,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5117,"private_eq":0.6711},"40":{"raw":0.2852,"skill":0.0805,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0539,"headline":false},{"track":"A · English","score":0.2852,"headline":true},{"track":"C · Arabic pairs","score":0.59,"headline":false},{"track":"C · English pairs","score":0.88,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7485,"skill":0.6228,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5698,"skill":0.163,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4474,"skill":0.1232,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0394,"private_eq":0.4008},"44":{"raw":0.7212,"skill":0.4424,"coverage":1,"random":0.5,"tracks":[],"combined":0.3371,"private_eq":0.6718},"45":{"raw":0.1637,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.4081,"skill":0.141,"coverage":1,"random":0.311,"tracks":[],"combined":0.174,"private_eq":0.4367},"56":{"raw":0.501,"skill":0.002,"coverage":1,"random":0.5,"tracks":[],"combined":0.0405,"private_eq":0.5252},"57":{"raw":0.1729,"skill":0.0697,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3684,"skill":0.0845,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3495,"combined":0.0473},"59":{"raw":0.5315,"skill":0.0286,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.849,"skill":0.698,"coverage":1,"random":0.5,"tracks":[],"combined":0.4507,"private_eq":0.728},"62":{"raw":0.744,"skill":0.6587,"coverage":1,"random":0.25,"tracks":[],"combined":0.3919,"private_eq":0.5484},"64":{"raw":0.4223,"skill":0.2779,"coverage":1,"random":0.2,"tracks":[],"combined":0.2446,"private_eq":0.4018}},"coverage":0.9995,"pending":0.0005,"notes":["New in the 2026-10-18 PR batch (PR #187).","Submitted public 0.3 run adopted (kit rescore 31.34, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.63% on 825 rows (bar 98.5%)."],"scores":{"balanced_raw":48.83,"balanced_skill":17.84,"breadth_skill":27.66,"public_skill":31.34},"frozen_scores":{"balanced_raw":48.83,"balanced_skill":31.34,"breadth_skill":27.66},"categories":[{"id":"knowledge","raw":0.3277,"skill":0.1196,"combined":0.0743,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.5216,"skill":0.3022,"combined":0.1317,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.6508,"skill":0.5143,"combined":0.4047,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5386,"skill":0.5009,"combined":0.4081,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.3997,"skill":0.0986,"combined":0.1007,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"mola-150m","pub":31.34,"same_skills":23.55,"new_domains":6.32,"v":17.84,"lab":false,"jev":false,"rank_v":203,"rank_pub":185,"name":"Mola-150M","eq":{"S":22.77,"O":10.45},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":13.66,"new_domains":4.18}},"new_in_edition":true},{"engine":"mola-68m","name":"Mola-68M","long_name":"Mola-68M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":68277249,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/codavidgarcia/mola-68m","code_url":null,"base_model":"cross-encoder/ettin-reranker-68m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-68m-v1","weights_repo":null,"served_checkpoint":null,"pr":187,"new":true},"completed":110201,"counts":{"ok":110195,"unsupported":6,"error":0},"lifted":false,"calibration":{"acc":0.5454,"conf":0.6597,"ece":0.1188,"brier":0.5594,"over95":0.0114,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0349,"conf":0.0262,"acc":0.0897},{"lo":0.1,"w":0.0198,"conf":0.1518,"acc":0.038},{"lo":0.2,"w":0.0248,"conf":0.2594,"acc":0.1437},{"lo":0.3,"w":0.0641,"conf":0.3543,"acc":0.2594},{"lo":0.4,"w":0.0688,"conf":0.4508,"acc":0.3384},{"lo":0.5,"w":0.2007,"conf":0.5463,"acc":0.451},{"lo":0.6,"w":0.146,"conf":0.6471,"acc":0.5153},{"lo":0.7,"w":0.1121,"conf":0.7507,"acc":0.5419},{"lo":0.8,"w":0.101,"conf":0.849,"acc":0.6722},{"lo":0.9,"w":0.2278,"conf":0.9727,"acc":0.8938}],"areas":{"knowledge":{"ece":0.168,"acc":0.4214,"conf":0.5893},"language":{"ece":0.0719,"acc":0.6599,"conf":0.7318},"retrieval":{"ece":0.035,"acc":0.741,"conf":0.776},"tools":{"ece":0.1285,"acc":0.6211,"conf":0.7479},"arts":{"ece":0.168,"acc":0.3851,"conf":0.5247}}},"gaps":null,"latency":{"median":11.4,"p95":18,"mean":13.5},"results":{"1":{"score":0.8937,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":32.2},"2":{"score":0.5285,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":127.9},"3":{"score":0.4606,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":92.3},"4":{"score":0.879,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":15.2},"5":{"score":0.8749,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":18.3},"6":{"score":0.5297,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":41.7},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":42.4},"10":{"score":0.6613,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":31.4},"11":{"score":0.7446,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":41.3},"12":{"score":0.3538,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.3},"20":{"score":0.4892,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.1},"21":{"score":0.5133,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.2},"22":{"score":0.07,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":38.9},"23":{"score":0.495,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.4},"24":{"score":0.3394,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":13.8},"25":{"score":0.2653,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":13.8},"26":{"score":0.6103,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":13},"27":{"score":0.4514,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":13.1},"28":{"score":0.5478,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":12.2},"29":{"score":0.3766,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":13.5},"30":{"score":0.2312,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0686,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":14.4},"32":{"score":0.504,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":16},"33":{"score":0.0358,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":16.3},"36":{"score":0.2051,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":295.4},"37":{"score":0.4544,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.1},"38":{"score":0.1004,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":69.2},"39":{"score":0.7225,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":14.6},"40":{"score":0.284,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":14.1},"41":{"score":0.7159,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":14.3},"42":{"score":0.4954,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":15.2},"43":{"score":0.4404,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":13.9},"44":{"score":0.6414,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.9},"45":{"score":0.15,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":14.3},"50":{"score":0.3896,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":16.2},"56":{"score":0.5245,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":15.5},"57":{"score":0.1405,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":13.5},"58":{"score":0.3503,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":13.5},"59":{"score":0.5181,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":15.4},"61":{"score":0.835,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":14.3},"62":{"score":0.6758,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":15.8},"64":{"score":0.3011,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":13.4}},"benchmarks":{"1":{"raw":0.8937,"skill":0.8566,"coverage":1,"random":0.2592,"tracks":[],"combined":0.749,"private_eq":0.8154},"2":{"raw":0.5246,"skill":0.451,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.5139,"private_eq":0.583},"3":{"raw":0.4606,"skill":0.4502,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2452,"private_eq":0.2669},"4":{"raw":0.879,"skill":0.8774,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6285,"private_eq":0.6364},"5":{"raw":0.8749,"skill":0.8741,"coverage":1,"random":0.006,"tracks":[],"combined":0.6169,"private_eq":0.6225},"6":{"raw":0.5297,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.6613,"skill":0.4364,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7326,"skill":0.6132,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.3538,"skill":0.0321,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2921,"private_eq":0.5321},"20":{"raw":0.4893,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0395,"private_eq":0.5247},"21":{"raw":0.5133,"skill":0.0266,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0656,"skill":0.0583,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.0098,"private_eq":0.0277},"23":{"raw":0.5073,"skill":0.0146,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3394,"skill":0.1192,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2653,"skill":0.0204,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2758,"combined":0.0243},"26":{"raw":0.6103,"skill":0.4803,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.4514,"skill":0.2683,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5478,"skill":0.0956,"coverage":1,"random":0.5,"tracks":[],"combined":0.088,"private_eq":0.5487},"29":{"raw":0.3766,"skill":0.1688,"coverage":1,"random":0.25,"tracks":[],"combined":0.201,"private_eq":0.4068},"30":{"raw":0.2312,"skill":0.0714,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3298,"random":0.25},{"track":"10-choice","raw":0.1327,"random":0.1}],"combined":0.0541,"private_eq":0.2277},"31":{"raw":0.0686,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.504,"skill":0.2114,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.0358,"skill":0.023,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.2051,"skill":0.1008,"coverage":1,"random":0.116,"tracks":[],"combined":0.1953,"private_eq":0.2959},"37":{"raw":0.4544,"skill":0.3157,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4972,"private_eq":0.6029},"38":{"raw":0.1004,"skill":0.0716,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.7225,"skill":0.5919,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4874,"private_eq":0.6548},"40":{"raw":0.284,"skill":0.0789,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0807,"headline":false},{"track":"A · English","score":0.284,"headline":true},{"track":"C · Arabic pairs","score":0.455,"headline":false},{"track":"C · English pairs","score":0.815,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7159,"skill":0.5739,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4954,"skill":0.0183,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4404,"skill":0.1121,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.6414,"skill":0.2828,"coverage":1,"random":0.5,"tracks":[],"combined":0.2567,"private_eq":0.6321},"45":{"raw":0.1497,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.3896,"skill":0.1141,"coverage":1,"random":0.311,"tracks":[],"combined":0.1234,"private_eq":0.4022},"56":{"raw":0.5245,"skill":0.049,"coverage":1,"random":0.5,"tracks":[],"combined":0.1358,"private_eq":0.5723},"57":{"raw":0.1405,"skill":0.0333,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1289,"combined":0.01},"58":{"raw":0.3503,"skill":0.0583,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3153,"combined":0},"59":{"raw":0.5181,"skill":0.0008,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.835,"skill":0.67,"coverage":1,"random":0.5,"tracks":[],"combined":0.5187,"private_eq":0.7616},"62":{"raw":0.6758,"skill":0.5677,"coverage":1,"random":0.25,"tracks":[],"combined":0.3029,"private_eq":0.4824},"64":{"raw":0.3011,"skill":0.1264,"coverage":1,"random":0.2,"tracks":[],"combined":0.1185,"private_eq":0.302}},"coverage":0.9995,"pending":0.0005,"notes":["New in the 2026-10-18 PR batch (PR #187).","Submitted public 0.3 run adopted (kit rescore 27.27, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.58% on 825 rows (bar 98.5%)."],"scores":{"balanced_raw":46.1,"balanced_skill":17.6,"breadth_skill":22.84,"public_skill":27.27},"frozen_scores":{"balanced_raw":46.1,"balanced_skill":27.27,"breadth_skill":22.84},"categories":[{"id":"knowledge","raw":0.3055,"skill":0.0792,"combined":0.0394,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.4729,"skill":0.2329,"combined":0.1222,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.631,"skill":0.4936,"combined":0.4364,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5233,"skill":0.479,"combined":0.3702,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.3777,"skill":0.0567,"combined":0.0561,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"mola-68m","pub":27.27,"same_skills":21.99,"new_domains":7.74,"v":17.6,"lab":false,"jev":false,"rank_v":205,"rank_pub":200,"name":"Mola-68M","eq":{"S":21.2,"O":12.2},"rank_tie":48,"tied":true,"parts":{"public":0,"same_skills":12.72,"new_domains":4.88}},"new_in_edition":true},{"engine":"jev-t5-342m-public-v2","name":"JEV FLAN-T5 Large public v2 (342M)","long_name":"JEV FLAN-T5 Large public v2 (342M)","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":null,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Elfsong/jev-flan-t5-large-20261017-public-v2","code_url":null,"base_model":"google/flan-t5-large","base_url":"https://huggingface.co/google/flan-t5-large","weights_repo":null,"served_checkpoint":null,"pr":218,"new":true},"completed":110201,"counts":{"ok":109786,"unsupported":415,"error":0},"lifted":false,"calibration":{"acc":0.5467,"conf":0.5768,"ece":0.0472,"brier":0.557,"over95":0.0033,"n":34329,"benchmarks":30,"rel":[{"lo":0,"w":0.0626,"conf":0.0351,"acc":0.0246},{"lo":0.1,"w":0.0069,"conf":0.1465,"acc":0.1315},{"lo":0.2,"w":0.0451,"conf":0.2666,"acc":0.2082},{"lo":0.3,"w":0.0993,"conf":0.3531,"acc":0.3346},{"lo":0.4,"w":0.0987,"conf":0.4489,"acc":0.479},{"lo":0.5,"w":0.2182,"conf":0.5486,"acc":0.5742},{"lo":0.6,"w":0.1623,"conf":0.6445,"acc":0.6231},{"lo":0.7,"w":0.1138,"conf":0.7491,"acc":0.703},{"lo":0.8,"w":0.1036,"conf":0.8504,"acc":0.6852},{"lo":0.9,"w":0.0895,"conf":0.9447,"acc":0.8601}],"areas":{"knowledge":{"ece":0.0539,"acc":0.4656,"conf":0.4706},"language":{"ece":0.0384,"acc":0.6942,"conf":0.6665},"retrieval":{"ece":0.0872,"acc":0.6394,"conf":0.7034},"tools":{"ece":0.2325,"acc":0.5527,"conf":0.7697},"arts":{"ece":0.082,"acc":0.3672,"conf":0.4492}}},"gaps":null,"latency":{"median":16.5,"p95":274.2,"mean":56},"results":{"1":{"score":0.3878,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":44.9},"2":{"score":0.3909,"jev":null,"metric":"nDCG@10","requests":685,"answered":679,"unsupported":6,"errors":0,"median_ms":316.6},"3":{"score":null,"jev":null,"metric":null,"requests":508,"answered":0,"unsupported":508,"errors":0,"median_ms":null},"4":{"score":0.7539,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":20.3},"5":{"score":0.7679,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":22},"6":{"score":0.6251,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":158.5},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":594},"10":{"score":0.4635,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":54.8},"11":{"score":0.555,"jev":null,"metric":"macro-F1","requests":123,"answered":121,"unsupported":2,"errors":0,"median_ms":1306.7},"12":{"score":0.4924,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":15.8},"20":{"score":0.3944,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.5},"21":{"score":0.567,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":15.6},"22":{"score":0.0035,"jev":null,"metric":"accuracy","requests":2000,"answered":1991,"unsupported":9,"errors":0,"median_ms":552.8},"23":{"score":0.5248,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17},"24":{"score":0.4007,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":16.3},"25":{"score":0.2653,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":16},"26":{"score":0.7487,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":15.7},"27":{"score":0.5836,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":16.5},"28":{"score":0.704,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":15.7},"29":{"score":0.7727,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":16.3},"30":{"score":0.1873,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0794,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":25.2},"32":{"score":0.5691,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":32.6},"33":{"score":0.1703,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":73.8},"36":{"score":0.2577,"jev":null,"metric":"nDCG@10","requests":220,"answered":217,"unsupported":3,"errors":0,"median_ms":526.3},"37":{"score":0.2711,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17},"38":{"score":0.1023,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":509.5},"39":{"score":0.8102,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":16.9},"40":{"score":0.2995,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":16.3},"41":{"score":0.5187,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":16.6},"42":{"score":0.5709,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":17.3},"43":{"score":0.4421,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":16.5},"44":{"score":0.5648,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.5},"45":{"score":0.1457,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":17.5},"50":{"score":0.324,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":22.2},"56":{"score":0.5025,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":83.1},"57":{"score":0.1661,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":16.1},"58":{"score":0.4038,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":16.4},"59":{"score":0.5158,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":22.1},"61":{"score":0.6222,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":17.2},"62":{"score":0.5868,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":26.4},"64":{"score":0.4545,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":16.7}},"benchmarks":{"1":{"raw":0.3878,"skill":0.1736,"coverage":1,"random":0.2592,"tracks":[],"combined":0.17,"private_eq":0.3914},"2":{"raw":0.3874,"skill":0.2926,"coverage":0.9912,"random":0.1341,"tracks":[],"combined":0.1958,"private_eq":0.3107},"3":{"raw":0,"skill":0,"coverage":0,"random":0.0189,"tracks":[],"combined":0.0247,"private_eq":0.0531},"4":{"raw":0.7539,"skill":0.7507,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7336,"private_eq":0.739},"5":{"raw":0.7679,"skill":0.7665,"coverage":1,"random":0.006,"tracks":[],"combined":0.5222,"private_eq":0.5295},"6":{"raw":0.6251,"skill":0.121,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.4635,"skill":0.1073,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5378,"skill":0.3316,"coverage":0.9837,"random":0.3085,"tracks":[]},"12":{"raw":0.4924,"skill":0.2397,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3793,"private_eq":0.5897},"20":{"raw":0.4077,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.567,"skill":0.1339,"coverage":1,"random":0.5,"tracks":[],"combined":0.0856,"private_eq":0.5475},"22":{"raw":0.0038,"skill":0,"coverage":0.9955,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5085,"skill":0.0171,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4007,"skill":0.2009,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2653,"skill":0.0204,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2716,"combined":0.0186},"26":{"raw":0.7487,"skill":0.6648,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5836,"skill":0.4447,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.704,"skill":0.408,"coverage":1,"random":0.5,"tracks":[],"combined":0.3553,"private_eq":0.6808},"29":{"raw":0.7727,"skill":0.6969,"coverage":1,"random":0.25,"tracks":[],"combined":0.4138,"private_eq":0.5646},"30":{"raw":0.1873,"skill":0.0155,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2669,"random":0.25},{"track":"10-choice","raw":0.1077,"random":0.1}],"combined":0.1025,"private_eq":0.2672},"31":{"raw":0.0794,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0892},"32":{"raw":0.5691,"skill":0.3149,"coverage":1,"random":0.371,"tracks":[],"combined":0.0262,"private_eq":0.3938},"33":{"raw":0.1703,"skill":0.1593,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0225},"36":{"raw":0.2542,"skill":0.1563,"coverage":0.9864,"random":0.116,"tracks":[],"combined":0.2632,"private_eq":0.3552},"37":{"raw":0.2711,"skill":0.0858,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3582,"private_eq":0.4933},"38":{"raw":0.1023,"skill":0.0736,"coverage":1,"random":0.031,"tracks":[],"combined":0.1012,"private_eq":0.138},"39":{"raw":0.8102,"skill":0.7208,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4311,"private_eq":0.6169},"40":{"raw":0.2995,"skill":0.0988,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.2995,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.47,"headline":false}],"combined":0.0095,"private_eq":0.2381},"41":{"raw":0.5187,"skill":0.2781,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3054,"private_eq":0.5415},"42":{"raw":0.5709,"skill":0.1652,"coverage":1,"random":0.486,"tracks":[],"combined":0.1985,"private_eq":0.5922},"43":{"raw":0.4421,"skill":0.1148,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.5648,"skill":0.1296,"coverage":1,"random":0.5,"tracks":[],"combined":0.1657,"private_eq":0.5871},"45":{"raw":0.1457,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.324,"skill":0.0189,"coverage":1,"random":0.311,"tracks":[],"combined":0.1451,"private_eq":0.417},"56":{"raw":0.5025,"skill":0.005,"coverage":1,"random":0.5,"tracks":[],"combined":0.0577,"private_eq":0.5337},"57":{"raw":0.1661,"skill":0.0621,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1329,"combined":0.0145},"58":{"raw":0.4038,"skill":0.1358,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3277,"combined":0.0153},"59":{"raw":0.5158,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.6222,"skill":0.2444,"coverage":1,"random":0.5,"tracks":[],"combined":0.1669,"private_eq":0.5877},"62":{"raw":0.5868,"skill":0.4491,"coverage":1,"random":0.25,"tracks":[],"combined":0.2618,"private_eq":0.4519},"64":{"raw":0.4545,"skill":0.3181,"coverage":1,"random":0.2,"tracks":[],"combined":0.3319,"private_eq":0.4708}},"coverage":0.9542,"pending":0.0458,"notes":["New in the 2026-10-18 PR batch (PR #218).","Submitted public 0.3 run adopted (kit rescore 22.02, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.69% on 821 rows (bar 98.5%)."],"scores":{"balanced_raw":41.44,"balanced_skill":17.31,"breadth_skill":20.49,"public_skill":22.02},"frozen_scores":{"balanced_raw":41.44,"balanced_skill":22.02,"breadth_skill":20.49},"categories":[{"id":"knowledge","raw":0.3362,"skill":0.1238,"combined":0.0708,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5184,"skill":0.297,"combined":0.2365,"coverage":0.9983,"pending":0,"n":9},{"id":"retrieval","raw":0.5344,"skill":0.3551,"combined":0.3645,"coverage":0.9975,"pending":0,"n":6},{"id":"tools","raw":0.2666,"skill":0.1759,"combined":0.1261,"coverage":0.7761,"pending":0,"n":5},{"id":"arts","raw":0.3776,"skill":0.0813,"combined":0.1083,"coverage":0.9992,"pending":0,"n":6}],"v03":{"engine":"jev-t5-342m-public-v2","pub":22.02,"same_skills":19.77,"new_domains":9.9,"v":17.31,"lab":false,"jev":false,"rank_v":206,"rank_pub":209,"name":"JEV FLAN-T5 Large public v2 (342M)","eq":{"S":18.94,"O":14.86},"rank_tie":48,"tied":true,"parts":{"public":0,"same_skills":11.37,"new_domains":5.95}},"new_in_edition":true},{"engine":"prumo","name":"Prumo","long_name":"Prumo","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":null,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tardellirs/prumo","code_url":null,"base_model":null,"base_url":null,"weights_repo":null,"served_checkpoint":null,"pr":213,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.4386,"conf":0.4684,"ece":0.0474,"brier":0.6212,"over95":0.0204,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0186,"conf":0.061,"acc":0.1913},{"lo":0.1,"w":0.0295,"conf":0.1426,"acc":0.2602},{"lo":0.2,"w":0.2089,"conf":0.2443,"acc":0.2294},{"lo":0.3,"w":0.1845,"conf":0.3341,"acc":0.3292},{"lo":0.4,"w":0.0206,"conf":0.4608,"acc":0.3142},{"lo":0.5,"w":0.3424,"conf":0.5016,"acc":0.51},{"lo":0.6,"w":0.0347,"conf":0.617,"acc":0.5845},{"lo":0.7,"w":0.0125,"conf":0.7452,"acc":0.4547},{"lo":0.8,"w":0.0505,"conf":0.8546,"acc":0.7889},{"lo":0.9,"w":0.0979,"conf":0.9739,"acc":0.7339}],"areas":{"knowledge":{"ece":0.0307,"acc":0.3718,"conf":0.3748},"language":{"ece":0.0183,"acc":0.4103,"conf":0.3981},"retrieval":{"ece":0.1047,"acc":0.4731,"conf":0.5215},"tools":{"ece":0.1545,"acc":0.5871,"conf":0.7309},"arts":{"ece":0.0934,"acc":0.5066,"conf":0.5824}}},"gaps":null,"latency":{"median":0.3,"p95":11.5,"mean":8.2},"results":{"1":{"score":0.7586,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":0.3},"2":{"score":0.4751,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3.2},"3":{"score":0.4626,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3.8},"4":{"score":0.4381,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":0.7},"5":{"score":0.4503,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":0.9},"6":{"score":0.5785,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":0.4},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1.6},"10":{"score":0.232,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":0.2},"11":{"score":0.1752,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":35.5},"12":{"score":0.1671,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":0.2},"20":{"score":0.5026,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.1},"21":{"score":0.5095,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":0.1},"22":{"score":0.53,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":0.7},"23":{"score":0.5022,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.1},"24":{"score":0.2467,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":0.1},"25":{"score":0.2398,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":0.1},"26":{"score":0.2471,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":0.1},"27":{"score":0.2654,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":0.1},"28":{"score":0.5043,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":0.1},"29":{"score":0.2475,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":0.2},"30":{"score":0.2669,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.3872,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.7},"32":{"score":0.3604,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":1.1},"33":{"score":0.0291,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":2.4},"36":{"score":0.2964,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5.2},"37":{"score":0.2538,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.1},"38":{"score":0.039,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":6.5},"39":{"score":0.324,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":0.2},"40":{"score":0.25,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":0.1},"41":{"score":0.1687,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":0.2},"42":{"score":0.3995,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":0.1},"43":{"score":0.9982,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":40.6},"44":{"score":0.501,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.1},"45":{"score":0.1417,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":0.2},"50":{"score":0.4803,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":1.4},"56":{"score":0.5,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":0.5},"57":{"score":0.1108,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":0.2},"58":{"score":0.6357,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":0.1},"59":{"score":0.6352,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":0.3},"61":{"score":0.6965,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":0.2},"62":{"score":0.7755,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":0.3},"64":{"score":0.1837,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":0.1}},"benchmarks":{"1":{"raw":0.7586,"skill":0.6741,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5107,"private_eq":0.6409},"2":{"raw":0.4751,"skill":0.3938,"coverage":1,"random":0.1341,"tracks":[],"combined":0.504,"private_eq":0.5745},"3":{"raw":0.4626,"skill":0.4522,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3397,"private_eq":0.3586},"4":{"raw":0.4381,"skill":0.4309,"coverage":1,"random":0.0127,"tracks":[],"combined":0.4987,"private_eq":0.5097},"5":{"raw":0.4503,"skill":0.447,"coverage":1,"random":0.006,"tracks":[],"combined":0.1929,"private_eq":0.2059},"6":{"raw":0.5785,"skill":0.0117,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.232,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.1752,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.1671,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0,"private_eq":0.3324},"20":{"raw":0.501,"skill":0.0019,"coverage":1,"random":0.5,"tracks":[],"combined":0.0467,"private_eq":0.5283},"21":{"raw":0.5095,"skill":0.019,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.5239,"skill":0.5202,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4003,"private_eq":0.4108},"23":{"raw":0.5088,"skill":0.0176,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2467,"skill":0,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2398,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.25,"combined":0},"26":{"raw":0.2471,"skill":0,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.2654,"skill":0.0203,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5043,"skill":0.0086,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5013},"29":{"raw":0.2475,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0.2306,"private_eq":0.4288},"30":{"raw":0.2669,"skill":0.1141,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3578,"random":0.25},{"track":"10-choice","raw":0.1759,"random":0.1}],"combined":0.1076,"private_eq":0.2713},"31":{"raw":0.3872,"skill":0.3326,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1418,"private_eq":0.2201},"32":{"raw":0.3604,"skill":0,"coverage":1,"random":0.371,"tracks":[],"combined":0.0493,"private_eq":0.4082},"33":{"raw":0.0291,"skill":0.0162,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0042,"private_eq":0.0275},"36":{"raw":0.2964,"skill":0.204,"coverage":1,"random":0.116,"tracks":[],"combined":0.126,"private_eq":0.2353},"37":{"raw":0.2538,"skill":0.064,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.039,"skill":0.0083,"coverage":1,"random":0.031,"tracks":[],"combined":0.0116,"private_eq":0.0522},"39":{"raw":0.324,"skill":0.0057,"coverage":1,"random":0.3201,"tracks":[],"combined":0.0057,"private_eq":0.331},"40":{"raw":0.25,"skill":0.0351,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.25,"headline":false},{"track":"A · English","score":0.25,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.465,"headline":false}],"combined":0.0279,"private_eq":0.2522},"41":{"raw":0.1687,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.3995,"skill":0,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.9982,"skill":0.9972,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2735,"private_eq":0.5467},"44":{"raw":0.501,"skill":0.002,"coverage":1,"random":0.5,"tracks":[],"combined":0.1228,"private_eq":0.5659},"45":{"raw":0.1417,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4803,"skill":0.2458,"coverage":1,"random":0.311,"tracks":[],"combined":0.0726,"private_eq":0.3676},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1108,"skill":0,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.6357,"skill":0.472,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3554,"combined":0.0559},"59":{"raw":0.6352,"skill":0.2436,"coverage":1,"random":0.5177,"tracks":[],"combined":0.0096,"private_eq":0.5273},"61":{"raw":0.6965,"skill":0.393,"coverage":1,"random":0.5,"tracks":[],"combined":0.3826,"private_eq":0.6943},"62":{"raw":0.7755,"skill":0.7007,"coverage":1,"random":0.25,"tracks":[],"combined":0.4261,"private_eq":0.5737},"64":{"raw":0.1837,"skill":0,"coverage":1,"random":0.2,"tracks":[],"combined":0,"private_eq":0.2}},"coverage":1,"pending":0,"notes":["New in the 2026-10-18 PR batch (PR #213).","Submitted public 0.3 run adopted (kit rescore 20.14). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 100.00% on 831 rows (bar 98.5%).","Rule-based engine, no trained weights."],"scores":{"balanced_raw":38.96,"balanced_skill":13.99,"breadth_skill":16.9,"public_skill":20.14},"frozen_scores":{"balanced_raw":38.96,"balanced_skill":20.14,"breadth_skill":16.9},"categories":[{"id":"knowledge","raw":0.3751,"skill":0.1699,"combined":0.0677,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2648,"skill":0.0311,"combined":0.0355,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.4352,"skill":0.266,"combined":0.2034,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.503,"skill":0.453,"combined":0.3592,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4512,"skill":0.1341,"combined":0.0997,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"prumo","pub":20.14,"same_skills":15.78,"new_domains":8.08,"v":13.99,"lab":false,"jev":false,"rank_v":218,"rank_pub":214,"name":"Prumo","eq":{"S":14.91,"O":12.62},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":8.95,"new_domains":5.05}},"new_in_edition":true},{"engine":"dinah-1","name":"Dinah-1","long_name":"Dinah-1","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":149614978,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Lukitaduarte/dinah-1","code_url":null,"base_model":"Lukitaduarte/dinah-0-apache","base_url":"https://huggingface.co/Lukitaduarte/dinah-0-apache","weights_repo":null,"served_checkpoint":null,"pr":176,"replaces":"dinah-0","new":true},"completed":110201,"counts":{"ok":110195,"unsupported":6,"error":0},"lifted":false,"calibration":{"acc":0.5525,"conf":0.7699,"ece":0.2179,"brier":0.6323,"over95":0.0652,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0358,"conf":0.0452,"acc":0.031},{"lo":0.1,"w":0.0159,"conf":0.1436,"acc":0.1577},{"lo":0.2,"w":0.0101,"conf":0.2479,"acc":0.2502},{"lo":0.3,"w":0.0275,"conf":0.3527,"acc":0.3327},{"lo":0.4,"w":0.0419,"conf":0.4551,"acc":0.3016},{"lo":0.5,"w":0.1002,"conf":0.5503,"acc":0.421},{"lo":0.6,"w":0.1026,"conf":0.649,"acc":0.4324},{"lo":0.7,"w":0.0936,"conf":0.7498,"acc":0.4238},{"lo":0.8,"w":0.1238,"conf":0.8523,"acc":0.4558},{"lo":0.9,"w":0.4484,"conf":0.9749,"acc":0.7623}],"areas":{"knowledge":{"ece":0.2273,"acc":0.4678,"conf":0.6932},"language":{"ece":0.2783,"acc":0.5934,"conf":0.8717},"retrieval":{"ece":0.1384,"acc":0.7392,"conf":0.8754},"tools":{"ece":0.1805,"acc":0.7458,"conf":0.9179},"arts":{"ece":0.1838,"acc":0.3828,"conf":0.5666}}},"gaps":null,"latency":{"median":14.2,"p95":57.6,"mean":29.6},"results":{"1":{"score":0.8477,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":24.3},"2":{"score":0.4963,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":155.4},"3":{"score":0.3681,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":118.5},"4":{"score":0.8807,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":22.8},"5":{"score":0.8987,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":24.3},"6":{"score":0.6117,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":37.9},"9":{"score":0.875,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":139.3},"10":{"score":0.4599,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":22.4},"11":{"score":0.6986,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":404.1},"12":{"score":0.3033,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":20},"20":{"score":0.5196,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.2},"21":{"score":0.5068,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":19.9},"22":{"score":0.009,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":61.3},"23":{"score":0.5002,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.7},"24":{"score":0.3173,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":20.1},"25":{"score":0.2755,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":20.3},"26":{"score":0.5034,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":20},"27":{"score":0.4044,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":19.9},"28":{"score":0.6038,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":19.5},"29":{"score":0.5275,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":20.6},"30":{"score":0.4697,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.251,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.1},"32":{"score":0.4614,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":22.9},"33":{"score":0.0103,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":34.4},"36":{"score":0.205,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":296},"37":{"score":0.479,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.1},"38":{"score":0.026,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":181.8},"39":{"score":0.8443,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":20.9},"40":{"score":0.274,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":20.1},"41":{"score":0.5539,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":20.4},"42":{"score":0.4906,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":21.1},"43":{"score":1,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":35.6},"44":{"score":0.8286,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.2},"45":{"score":0.148,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":21},"50":{"score":0.3914,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":22.4},"56":{"score":0.5515,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":46.7},"57":{"score":0.1336,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":20.9},"58":{"score":0.3724,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":20.5},"59":{"score":0.4453,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":22.5},"61":{"score":0.8223,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":21},"62":{"score":0.7673,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":21.6},"64":{"score":0.4754,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":20.7}},"benchmarks":{"1":{"raw":0.8477,"skill":0.7944,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6672,"private_eq":0.7555},"2":{"raw":0.4927,"skill":0.4142,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.5187,"private_eq":0.5871},"3":{"raw":0.3681,"skill":0.3559,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2348,"private_eq":0.2568},"4":{"raw":0.8807,"skill":0.8792,"coverage":1,"random":0.0127,"tracks":[],"combined":0.5802,"private_eq":0.5893},"5":{"raw":0.8987,"skill":0.8981,"coverage":1,"random":0.006,"tracks":[],"combined":0.4702,"private_eq":0.4784},"6":{"raw":0.6117,"skill":0.0896,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.875,"skill":0.875,"coverage":1,"random":0,"tracks":[],"combined":0.432,"private_eq":0.4375},"10":{"raw":0.4599,"skill":0.1013,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6863,"skill":0.5463,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.3033,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2965,"private_eq":0.535},"20":{"raw":0.5174,"skill":0.0347,"coverage":1,"random":0.5,"tracks":[],"combined":0.0684,"private_eq":0.539},"21":{"raw":0.5068,"skill":0.0137,"coverage":1,"random":0.5,"tracks":[],"combined":0.0453,"private_eq":0.5276},"22":{"raw":0.0091,"skill":0.0013,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5025,"skill":0.0051,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3173,"skill":0.0897,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2755,"skill":0.034,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.25,"combined":0},"26":{"raw":0.5034,"skill":0.3377,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.4044,"skill":0.2057,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6038,"skill":0.2076,"coverage":1,"random":0.5,"tracks":[],"combined":0.088,"private_eq":0.5487},"29":{"raw":0.5275,"skill":0.37,"coverage":1,"random":0.25,"tracks":[],"combined":0.1763,"private_eq":0.3885},"30":{"raw":0.4697,"skill":0.3583,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5277,"random":0.25},{"track":"10-choice","raw":0.4117,"random":0.1}],"combined":0.058,"private_eq":0.2309},"31":{"raw":0.251,"skill":0.1842,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0778,"private_eq":0.1621},"32":{"raw":0.4614,"skill":0.1437,"coverage":1,"random":0.371,"tracks":[],"combined":0.0418,"private_eq":0.4035},"33":{"raw":0.0103,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.205,"skill":0.1006,"coverage":1,"random":0.116,"tracks":[],"combined":0.137,"private_eq":0.2449},"37":{"raw":0.479,"skill":0.3466,"coverage":1,"random":0.2027,"tracks":[],"combined":0.0925,"private_eq":0.2839},"38":{"raw":0.026,"skill":0,"coverage":1,"random":0.031,"tracks":[],"combined":0,"private_eq":0.031},"39":{"raw":0.8443,"skill":0.771,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5144,"private_eq":0.6729},"40":{"raw":0.274,"skill":0.066,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0093,"headline":false},{"track":"A · English","score":0.274,"headline":true},{"track":"C · Arabic pairs","score":0.55,"headline":false},{"track":"C · English pairs","score":0.765,"headline":false}],"combined":0.0335,"private_eq":0.2565},"41":{"raw":0.5539,"skill":0.3309,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.4906,"skill":0.0089,"coverage":1,"random":0.486,"tracks":[],"combined":0.1261,"private_eq":0.5554},"43":{"raw":1,"skill":1,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3111,"private_eq":0.5701},"44":{"raw":0.8286,"skill":0.6572,"coverage":1,"random":0.5,"tracks":[],"combined":0.5887,"private_eq":0.7962},"45":{"raw":0.1477,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.3914,"skill":0.1167,"coverage":1,"random":0.311,"tracks":[],"combined":0.1523,"private_eq":0.4219},"56":{"raw":0.5515,"skill":0.103,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1336,"skill":0.0255,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3724,"skill":0.0903,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3101,"combined":0},"59":{"raw":0.4453,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8223,"skill":0.6446,"coverage":1,"random":0.5,"tracks":[],"combined":0.2917,"private_eq":0.6494},"62":{"raw":0.7673,"skill":0.6897,"coverage":1,"random":0.25,"tracks":[],"combined":0.2344,"private_eq":0.4316},"64":{"raw":0.4754,"skill":0.3442,"coverage":1,"random":0.2,"tracks":[],"combined":0.3998,"private_eq":0.5245}},"coverage":0.9995,"pending":0.0005,"notes":["New in Decision Index 0.4 (PR #176).","Replaces the earlier board entry dinah-0 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 34.32, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.75% on 827 rows (bar 98.5%).","Training targets index blocks: home-appliance cases built from the kit simulator's logic and train splits of several index benchmarks (author-disclosed).","A code expert runs request-supplied Python in a sandboxed child process, and a calculator expert scores numeric options."],"scores":{"balanced_raw":51.36,"balanced_skill":16.36,"breadth_skill":30.77,"public_skill":34.32},"frozen_scores":{"balanced_raw":51.36,"balanced_skill":34.32,"breadth_skill":30.77},"categories":[{"id":"knowledge","raw":0.405,"skill":0.231,"combined":0.1051,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.4593,"skill":0.2305,"combined":0.144,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.6415,"skill":0.5072,"combined":0.2725,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6655,"skill":0.6221,"combined":0.4199,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.4004,"skill":0.0859,"combined":0.1311,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"dinah-1","pub":34.32,"same_skills":22.29,"new_domains":4.85,"v":16.36,"lab":false,"jev":false,"rank_v":211,"rank_pub":175,"name":"Dinah-1","eq":{"S":21.5,"O":8.64},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":12.9,"new_domains":3.46}},"new_in_edition":true},{"engine":"crivo-400m","name":"Crivo-400M","long_name":"Crivo-400M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":395838465,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tardellirs/crivo-400m","code_url":null,"base_model":"cross-encoder/ettin-reranker-400m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-400m-v1","weights_repo":null,"served_checkpoint":null,"pr":167,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6117,"conf":0.7327,"ece":0.121,"brier":0.5035,"over95":0.0324,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0023,"conf":0.092,"acc":0.0174},{"lo":0.1,"w":0.0227,"conf":0.1598,"acc":0.0557},{"lo":0.2,"w":0.0402,"conf":0.2587,"acc":0.1226},{"lo":0.3,"w":0.0713,"conf":0.3449,"acc":0.1834},{"lo":0.4,"w":0.0599,"conf":0.4509,"acc":0.3568},{"lo":0.5,"w":0.1224,"conf":0.5482,"acc":0.473},{"lo":0.6,"w":0.1022,"conf":0.6488,"acc":0.5346},{"lo":0.7,"w":0.0971,"conf":0.7494,"acc":0.6161},{"lo":0.8,"w":0.0923,"conf":0.8518,"acc":0.6444},{"lo":0.9,"w":0.3896,"conf":0.9806,"acc":0.8706}],"areas":{"knowledge":{"ece":0.1408,"acc":0.4376,"conf":0.5784},"language":{"ece":0.0565,"acc":0.7573,"conf":0.8138},"retrieval":{"ece":0.1569,"acc":0.7164,"conf":0.8734},"tools":{"ece":0.1052,"acc":0.8819,"conf":0.9787},"arts":{"ece":0.1831,"acc":0.452,"conf":0.6352}}},"gaps":null,"latency":{"median":16.7,"p95":39.9,"mean":21.2},"results":{"1":{"score":0.8713,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":24.5},"2":{"score":0.6425,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":228.6},"3":{"score":0.6929,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":152.8},"4":{"score":0.8596,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":25.1},"5":{"score":0.8872,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":26.4},"6":{"score":0.6607,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":55.8},"9":{"score":0.7159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":171.6},"10":{"score":0.4325,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":25.1},"11":{"score":0.8229,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":431},"12":{"score":0.4387,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.9},"20":{"score":0.9162,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.9},"21":{"score":0.5164,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":22.7},"22":{"score":0.0235,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":99.1},"23":{"score":0.4836,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.4},"24":{"score":0.4088,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":22.9},"25":{"score":0.2347,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":22.9},"26":{"score":0.7715,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":22.8},"27":{"score":0.5776,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":22.8},"28":{"score":0.6322,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":22.7},"29":{"score":0.727,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":23.2},"30":{"score":0.1922,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0824,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":24.6},"32":{"score":0.5479,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":25.5},"33":{"score":0.0267,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":37.9},"36":{"score":0.3553,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":448.6},"37":{"score":0.4093,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":23.6},"38":{"score":0.3203,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":98.6},"39":{"score":0.871,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":23.4},"40":{"score":0.3567,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":22.8},"41":{"score":0.7534,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":22.9},"42":{"score":0.6194,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":23.8},"43":{"score":0.3877,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":22.9},"44":{"score":0.7364,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.9},"45":{"score":0.1397,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":23.2},"50":{"score":0.3807,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":25.1},"56":{"score":0.526,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":27.4},"57":{"score":0.1692,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":23.2},"58":{"score":0.402,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":22.9},"59":{"score":0.6606,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":24.8},"61":{"score":0.8193,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":23.6},"62":{"score":0.707,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":24.5},"64":{"score":0.6231,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":23.2}},"benchmarks":{"1":{"raw":0.8713,"skill":0.8263,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7218,"private_eq":0.7955},"2":{"raw":0.6425,"skill":0.5872,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6321,"private_eq":0.6842},"3":{"raw":0.6929,"skill":0.687,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6389,"private_eq":0.6488},"4":{"raw":0.8596,"skill":0.8578,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7708,"private_eq":0.7753},"5":{"raw":0.8872,"skill":0.8865,"coverage":1,"random":0.006,"tracks":[],"combined":0.6421,"private_eq":0.6473},"6":{"raw":0.6607,"skill":0.2045,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7159,"skill":0.7159,"coverage":1,"random":0,"tracks":[],"combined":0.3461,"private_eq":0.3525},"10":{"raw":0.4325,"skill":0.0557,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8229,"skill":0.7438,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4387,"skill":0.1592,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3266,"private_eq":0.5549},"20":{"raw":0.913,"skill":0.826,"coverage":1,"random":0.5,"tracks":[],"combined":0.7625,"private_eq":0.8821},"21":{"raw":0.5164,"skill":0.0327,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0246,"skill":0.0169,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0081},"23":{"raw":0.5082,"skill":0.0163,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4088,"skill":0.2117,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2347,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.25,"combined":0},"26":{"raw":0.7715,"skill":0.6953,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5776,"skill":0.4366,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6322,"skill":0.2644,"coverage":1,"random":0.5,"tracks":[],"combined":0.2697,"private_eq":0.6385},"29":{"raw":0.727,"skill":0.636,"coverage":1,"random":0.25,"tracks":[],"combined":0.4039,"private_eq":0.5573},"30":{"raw":0.1922,"skill":0.0234,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2851,"random":0.25},{"track":"10-choice","raw":0.0993,"random":0.1}],"combined":0.0715,"private_eq":0.2419},"31":{"raw":0.0824,"skill":0.0006,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0834},"32":{"raw":0.5479,"skill":0.2812,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0267,"skill":0.0138,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.3553,"skill":0.2707,"coverage":1,"random":0.116,"tracks":[],"combined":0.256,"private_eq":0.3489},"37":{"raw":0.4093,"skill":0.2591,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1061,"private_eq":0.2946},"38":{"raw":0.3203,"skill":0.2986,"coverage":1,"random":0.031,"tracks":[],"combined":0.4682,"private_eq":0.4896},"39":{"raw":0.871,"skill":0.8103,"coverage":1,"random":0.3201,"tracks":[],"combined":0.2542,"private_eq":0.498},"40":{"raw":0.3567,"skill":0.1725,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.195,"headline":false},{"track":"A · English","score":0.3567,"headline":true},{"track":"C · Arabic pairs","score":0.53,"headline":false},{"track":"C · English pairs","score":0.5,"headline":false}],"combined":0.1371,"private_eq":0.3361},"41":{"raw":0.7534,"skill":0.6301,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.6194,"skill":0.2595,"coverage":1,"random":0.486,"tracks":[],"combined":0.0723,"private_eq":0.5281},"43":{"raw":0.3877,"skill":0.0286,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3722},"44":{"raw":0.7364,"skill":0.4728,"coverage":1,"random":0.5,"tracks":[],"combined":0.4603,"private_eq":0.7327},"45":{"raw":0.1397,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3807,"skill":0.1011,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.526,"skill":0.052,"coverage":1,"random":0.5,"tracks":[],"combined":0.0751,"private_eq":0.5423},"57":{"raw":0.1692,"skill":0.0656,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2217,"combined":0.1155},"58":{"raw":0.402,"skill":0.1332,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3352,"combined":0.0263},"59":{"raw":0.6606,"skill":0.2963,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8193,"skill":0.6386,"coverage":1,"random":0.5,"tracks":[],"combined":0.5244,"private_eq":0.7644},"62":{"raw":0.707,"skill":0.6093,"coverage":1,"random":0.25,"tracks":[],"combined":0.4124,"private_eq":0.5636},"64":{"raw":0.6231,"skill":0.5289,"coverage":1,"random":0.2,"tracks":[],"combined":0.5161,"private_eq":0.6165}},"coverage":1,"pending":0,"notes":["New in Decision Index 0.4 (PR #167).","Submitted public 0.3 run adopted (kit rescore 39.79, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.91% on 831 rows (bar 98.5%).","Trains on train/dev splits of benchmark source datasets and on smart-home cases built from the suite's templates (author-disclosed).","License CC BY-NC 4.0."],"scores":{"balanced_raw":55.55,"balanced_skill":23.74,"breadth_skill":35.02,"public_skill":39.79},"frozen_scores":{"balanced_raw":55.55,"balanced_skill":39.79,"breadth_skill":35.02},"categories":[{"id":"knowledge","raw":0.3222,"skill":0.1147,"combined":0.0904,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6174,"skill":0.4431,"combined":0.2128,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.648,"skill":0.5103,"combined":0.4104,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7301,"skill":0.6904,"combined":0.5599,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4943,"skill":0.2536,"combined":0.2922,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"crivo-400m","pub":39.79,"same_skills":30.7,"new_domains":9.48,"v":23.74,"lab":false,"jev":false,"rank_v":184,"rank_pub":156,"name":"Crivo-400M","eq":{"S":30,"O":14.35},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":18,"new_domains":5.74}},"new_in_edition":true},{"engine":"jev-ettin-400m","name":"JEV Ettin 400M","long_name":"JEV Ettin 400M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":400000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Elfsong/jev-ettin-400m-20261013-v0","code_url":null,"base_model":"jhu-clsp/ettin-encoder-400m","base_url":"https://huggingface.co/jhu-clsp/ettin-encoder-400m","weights_repo":null,"served_checkpoint":null,"pr":163,"new":true},"completed":110201,"counts":{"ok":110192,"unsupported":9,"error":0},"lifted":false,"calibration":{"acc":0.4699,"conf":0.5601,"ece":0.0902,"brier":0.6351,"over95":0.0109,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0453,"conf":0.0398,"acc":0.034},{"lo":0.1,"w":0.035,"conf":0.1415,"acc":0.1011},{"lo":0.2,"w":0.0421,"conf":0.2633,"acc":0.2357},{"lo":0.3,"w":0.1061,"conf":0.3503,"acc":0.2643},{"lo":0.4,"w":0.1127,"conf":0.4496,"acc":0.4122},{"lo":0.5,"w":0.2284,"conf":0.5473,"acc":0.5026},{"lo":0.6,"w":0.1551,"conf":0.6469,"acc":0.5527},{"lo":0.7,"w":0.1172,"conf":0.7476,"acc":0.6292},{"lo":0.8,"w":0.084,"conf":0.8477,"acc":0.6427},{"lo":0.9,"w":0.0741,"conf":0.9483,"acc":0.7039}],"areas":{"knowledge":{"ece":0.0586,"acc":0.4432,"conf":0.4932},"language":{"ece":0.0866,"acc":0.5792,"conf":0.6657},"retrieval":{"ece":0.0993,"acc":0.5672,"conf":0.5633},"tools":{"ece":0.3789,"acc":0.303,"conf":0.6642},"arts":{"ece":0.0913,"acc":0.3491,"conf":0.4388}}},"gaps":null,"latency":{"median":16.2,"p95":79.9,"mean":30.4},"results":{"1":{"score":0.389,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":30.4},"2":{"score":0.3997,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":306},"3":{"score":0.253,"jev":null,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":350.1},"4":{"score":0.5615,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":22.9},"5":{"score":0.513,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":25.8},"6":{"score":0.6177,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":40},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":231.1},"10":{"score":0.4323,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":25.9},"11":{"score":0.33,"jev":null,"metric":"macro-F1","requests":123,"answered":121,"unsupported":2,"errors":0,"median_ms":410.7},"12":{"score":0.3817,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":17.4},"20":{"score":0.5098,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.6},"21":{"score":0.5126,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":17.2},"22":{"score":0.024,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":177.3},"23":{"score":0.4788,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.6},"24":{"score":0.4144,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":17.4},"25":{"score":0.2296,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":18},"26":{"score":0.7601,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":17.2},"27":{"score":0.5597,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":17.2},"28":{"score":0.5959,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":17.5},"29":{"score":0.557,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":18},"30":{"score":0.2005,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0664,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.8},"32":{"score":0.5346,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":22.6},"33":{"score":0.0624,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":76.9},"36":{"score":0.2368,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":351.4},"37":{"score":0.2963,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.9},"38":{"score":0.0849,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":540.6},"39":{"score":0.4415,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":18.9},"40":{"score":0.2614,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":18.3},"41":{"score":0.4652,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":17.9},"42":{"score":0.5502,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":19.6},"43":{"score":0.4018,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":18.2},"44":{"score":0.5744,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.5},"45":{"score":0.156,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":18.7},"50":{"score":0.2822,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":22.1},"56":{"score":0.5215,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":95.4},"57":{"score":0.1863,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":18.5},"58":{"score":0.4055,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":18.5},"59":{"score":0.5177,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":22.9},"61":{"score":0.566,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":19.8},"62":{"score":0.1698,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":22.5},"64":{"score":0.4205,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":19.1}},"benchmarks":{"1":{"raw":0.389,"skill":0.1752,"coverage":1,"random":0.2592,"tracks":[],"combined":0.204,"private_eq":0.4163},"2":{"raw":0.3968,"skill":0.3034,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.2926,"private_eq":0.3936},"3":{"raw":0.252,"skill":0.2376,"coverage":0.9961,"random":0.0189,"tracks":[],"combined":0.0826,"private_eq":0.1092},"4":{"raw":0.5615,"skill":0.5559,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6807,"private_eq":0.6874},"5":{"raw":0.513,"skill":0.5101,"coverage":1,"random":0.006,"tracks":[],"combined":0.4399,"private_eq":0.4486},"6":{"raw":0.6177,"skill":0.1036,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.4323,"skill":0.0554,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.3232,"skill":0.0212,"coverage":0.9837,"random":0.3085,"tracks":[]},"12":{"raw":0.3817,"skill":0.0738,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2642,"private_eq":0.5137},"20":{"raw":0.516,"skill":0.0319,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.5126,"skill":0.0251,"coverage":1,"random":0.5,"tracks":[],"combined":0.0453,"private_eq":0.5276},"22":{"raw":0.0229,"skill":0.0153,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0142},"23":{"raw":0.494,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4144,"skill":0.2192,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2296,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.283,"combined":0.034},"26":{"raw":0.7601,"skill":0.68,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5597,"skill":0.4128,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5959,"skill":0.1918,"coverage":1,"random":0.5,"tracks":[],"combined":0.1687,"private_eq":0.5886},"29":{"raw":0.557,"skill":0.4093,"coverage":1,"random":0.25,"tracks":[],"combined":0.4238,"private_eq":0.572},"30":{"raw":0.2005,"skill":0.0323,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2851,"random":0.25},{"track":"10-choice","raw":0.116,"random":0.1}],"combined":0.0597,"private_eq":0.2323},"31":{"raw":0.0664,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0819},"32":{"raw":0.5346,"skill":0.2601,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0624,"skill":0.05,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.2368,"skill":0.1366,"coverage":1,"random":0.116,"tracks":[],"combined":0.218,"private_eq":0.3157},"37":{"raw":0.2963,"skill":0.1174,"coverage":1,"random":0.2027,"tracks":[],"combined":0,"private_eq":0.2027},"38":{"raw":0.0849,"skill":0.0556,"coverage":1,"random":0.031,"tracks":[],"combined":0.0925,"private_eq":0.1297},"39":{"raw":0.4415,"skill":0.1786,"coverage":1,"random":0.3201,"tracks":[],"combined":0.0509,"private_eq":0.3614},"40":{"raw":0.2614,"skill":0.0498,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0578,"headline":false},{"track":"A · English","score":0.2614,"headline":true},{"track":"C · Arabic pairs","score":0.4,"headline":false},{"track":"C · English pairs","score":0.195,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4652,"skill":0.1978,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5502,"skill":0.1249,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4018,"skill":0.0508,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0688,"private_eq":0.4191},"44":{"raw":0.5744,"skill":0.1488,"coverage":1,"random":0.5,"tracks":[],"combined":0.278,"private_eq":0.6426},"45":{"raw":0.1557,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.2822,"skill":0,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.5215,"skill":0.043,"coverage":1,"random":0.5,"tracks":[],"combined":0.179,"private_eq":0.5937},"57":{"raw":0.1863,"skill":0.0848,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.252,"combined":0.15},"58":{"raw":0.4055,"skill":0.1383,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3402,"combined":0.0336},"59":{"raw":0.5177,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.566,"skill":0.132,"coverage":1,"random":0.5,"tracks":[],"combined":0.0649,"private_eq":0.5373},"62":{"raw":0.1698,"skill":0,"coverage":1,"random":0.25,"tracks":[],"combined":0,"private_eq":0.254},"64":{"raw":0.4205,"skill":0.2756,"coverage":1,"random":0.2,"tracks":[],"combined":0.3222,"private_eq":0.4631}},"coverage":0.9992,"pending":0.0008,"notes":["New in Decision Index 0.4 (PR #163).","Submitted public 0.3 run adopted (kit rescore 14.11, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.44% on 824 rows (bar 98.5%).","Training data undisclosed."],"scores":{"balanced_raw":35.64,"balanced_skill":15.24,"breadth_skill":13.31,"public_skill":14.11},"frozen_scores":{"balanced_raw":35.64,"balanced_skill":14.11,"breadth_skill":13.31},"categories":[{"id":"knowledge","raw":0.3106,"skill":0.0867,"combined":0.0776,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.4011,"skill":0.1285,"combined":0.1104,"coverage":0.9983,"pending":0,"n":9},{"id":"retrieval","raw":0.4481,"skill":0.263,"combined":0.2787,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.2474,"skill":0.1479,"combined":0.115,"coverage":0.9978,"pending":0,"n":5},{"id":"arts","raw":0.3747,"skill":0.058,"combined":0.1112,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"jev-ettin-400m","pub":14.11,"same_skills":14.82,"new_domains":11.77,"v":15.24,"lab":false,"jev":false,"rank_v":215,"rank_pub":225,"name":"JEV Ettin 400M","eq":{"S":13.94,"O":17.19},"rank_tie":50,"tied":true,"parts":{"public":0,"same_skills":8.36,"new_domains":6.87}},"new_in_edition":true},{"engine":"mogan-decision-31b-it","name":"mogan-decision-31B-it","long_name":"mogan-decision-31B-it","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/moganai/mogan-decision-31B-it","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"moganai","name":"Mogan AI","logo":"data/orgs/moganai.png"},"pr":21,"new":true,"show_notes":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7903,"conf":0.8443,"ece":0.0544,"brier":0.2942,"over95":0.0201,"n":35093,"benchmarks":32,"rel":[{"lo":0,"w":0.0024,"conf":0.0865,"acc":0.1613},{"lo":0.1,"w":0.0132,"conf":0.1476,"acc":0.1355},{"lo":0.2,"w":0.0087,"conf":0.2514,"acc":0.2463},{"lo":0.3,"w":0.0155,"conf":0.3546,"acc":0.3423},{"lo":0.4,"w":0.0371,"conf":0.4556,"acc":0.409},{"lo":0.5,"w":0.0744,"conf":0.548,"acc":0.4529},{"lo":0.6,"w":0.0811,"conf":0.6511,"acc":0.5677},{"lo":0.7,"w":0.0759,"conf":0.7538,"acc":0.6677},{"lo":0.8,"w":0.0993,"conf":0.8548,"acc":0.757},{"lo":0.9,"w":0.5925,"conf":0.9821,"acc":0.945}],"areas":{"knowledge":{"ece":0.0866,"acc":0.7278,"conf":0.813},"language":{"ece":0.0194,"acc":0.8789,"conf":0.8912},"retrieval":{"ece":0.0639,"acc":0.7658,"conf":0.826},"tools":{"ece":0.0429,"acc":0.9372,"conf":0.9532},"arts":{"ece":0.0804,"acc":0.6996,"conf":0.778}}},"gaps":null,"latency":{"median":109,"p95":1690.1,"mean":382.3},"results":{"1":{"score":0.9687,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":334.9},"2":{"score":0.6792,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3926.3},"3":{"score":0.8465,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1876.7},"4":{"score":0.9196,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":138.3},"5":{"score":0.9584,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":150.9},"6":{"score":0.7985,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":984.8},"9":{"score":0.6023,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":3600.6},"10":{"score":0.5368,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":269.7},"11":{"score":0.831,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":6699.7},"12":{"score":0.7957,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":112},"20":{"score":0.9326,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":79.5},"21":{"score":0.6956,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":82.5},"22":{"score":0.782,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1411.5},"23":{"score":0.6484,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":107.8},"24":{"score":0.8577,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":80.3},"25":{"score":0.4949,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":82.5},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":78.9},"27":{"score":0.977,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":78.7},"28":{"score":0.9384,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":78.5},"29":{"score":0.969,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":80.3},"30":{"score":0.6509,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2986,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":172.1},"32":{"score":0.6596,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":210.2},"33":{"score":0.3188,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":696.7},"36":{"score":0.4706,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5630.4},"37":{"score":0.6053,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":107.4},"38":{"score":0.4736,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2143.5},"39":{"score":0.9178,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":123.2},"40":{"score":0.6786,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":81.1},"41":{"score":0.8052,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":79.2},"42":{"score":0.7617,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":122.2},"43":{"score":0.7474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":79.2},"44":{"score":0.8758,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":78.7},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":82.4},"50":{"score":0.4385,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":173.9},"56":{"score":0.787,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":535.3},"57":{"score":0.7041,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":80.4},"58":{"score":0.8914,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":79.2},"59":{"score":0.8451,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":160.9},"61":{"score":0.8905,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":98.6},"62":{"score":0.9307,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":157.9},"64":{"score":0.8788,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":79.5}},"benchmarks":{"1":{"raw":0.9687,"skill":0.9578,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6792,"skill":0.6296,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7104,"private_eq":0.7512},"3":{"raw":0.8465,"skill":0.8435,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.9196,"skill":0.9186,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8974,"private_eq":0.8989},"5":{"raw":0.9584,"skill":0.9581,"coverage":1,"random":0.006,"tracks":[],"combined":0.7307,"private_eq":0.7344},"6":{"raw":0.7985,"skill":0.5275,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6023,"skill":0.6023,"coverage":1,"random":0,"tracks":[],"combined":0.5433,"private_eq":0.5475},"10":{"raw":0.5368,"skill":0.2293,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.831,"skill":0.7555,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7957,"skill":0.694,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3877,"private_eq":0.5952},"20":{"raw":0.9349,"skill":0.8698,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6956,"skill":0.3912,"coverage":1,"random":0.5,"tracks":[],"combined":0.3371,"private_eq":0.6718},"22":{"raw":0.7806,"skill":0.7789,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5512,"private_eq":0.5588},"23":{"raw":0.6452,"skill":0.2905,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8577,"skill":0.8103,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4949,"skill":0.3265,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5034,"combined":0.3313},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.977,"skill":0.9693,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9384,"skill":0.8768,"coverage":1,"random":0.5,"tracks":[],"combined":0.6227,"private_eq":0.813},"29":{"raw":0.969,"skill":0.9587,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6509,"skill":0.5849,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7551,"random":0.25},{"track":"10-choice","raw":0.5466,"random":0.1}],"combined":0.6386,"private_eq":0.7044},"31":{"raw":0.2986,"skill":0.2361,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0886,"private_eq":0.1719},"32":{"raw":0.6596,"skill":0.4588,"coverage":1,"random":0.371,"tracks":[],"combined":0.2273,"private_eq":0.5189},"33":{"raw":0.3188,"skill":0.3098,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3896,"private_eq":0.4035},"36":{"raw":0.4706,"skill":0.4011,"coverage":1,"random":0.116,"tracks":[],"combined":0.402,"private_eq":0.4765},"37":{"raw":0.6053,"skill":0.5049,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4911,"private_eq":0.5981},"38":{"raw":0.4736,"skill":0.4568,"coverage":1,"random":0.031,"tracks":[],"combined":0.5511,"private_eq":0.569},"39":{"raw":0.9178,"skill":0.8791,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4461,"private_eq":0.627},"40":{"raw":0.6786,"skill":0.5865,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4881,"headline":false},{"track":"A · English","score":0.6786,"headline":true},{"track":"C · Arabic pairs","score":0.9,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.5297,"private_eq":0.6378},"41":{"raw":0.8052,"skill":0.7078,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.7617,"skill":0.5364,"coverage":1,"random":0.486,"tracks":[],"combined":0.8965,"private_eq":0.9469},"43":{"raw":0.7474,"skill":0.5992,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6205,"private_eq":0.7629},"44":{"raw":0.8758,"skill":0.7516,"coverage":1,"random":0.5,"tracks":[],"combined":0.6531,"private_eq":0.828},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4385,"skill":0.1851,"coverage":1,"random":0.311,"tracks":[],"combined":0.2104,"private_eq":0.4615},"56":{"raw":0.787,"skill":0.574,"coverage":1,"random":0.5,"tracks":[],"combined":0.5944,"private_eq":0.799},"57":{"raw":0.7041,"skill":0.6672,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6086,"combined":0.5557},"58":{"raw":0.8914,"skill":0.8426,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5029,"combined":0.2722},"59":{"raw":0.8451,"skill":0.6788,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3414,"private_eq":0.6855},"61":{"raw":0.8905,"skill":0.781,"coverage":1,"random":0.5,"tracks":[],"combined":0.6777,"private_eq":0.8402},"62":{"raw":0.9307,"skill":0.9076,"coverage":1,"random":0.25,"tracks":[],"combined":0.5356,"private_eq":0.6549},"64":{"raw":0.8788,"skill":0.8485,"coverage":1,"random":0.2,"tracks":[],"combined":0.8361,"private_eq":0.8696}},"coverage":1,"pending":0,"notes":["New in Decision Index 0.4 (PR #21).","Submitted public 0.2.1 run adopted and rescored by the lab on the 0.3 public suite, with the rebuilt GSM8K run by the lab. Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.63% on 760 rows (bar 98.5%).","Training includes public train/dev splits of several index benchmark families and teacher soft labels (author-disclosed).","Some private test items overlapped its published training data; they are not counted for this model.","License CC BY-NC 4.0."],"scores":{"balanced_raw":73.83,"balanced_skill":59.82,"breadth_skill":64.86,"public_skill":65.55},"frozen_scores":{"balanced_raw":73.83,"balanced_skill":65.55,"breadth_skill":64.86},"categories":[{"id":"knowledge","raw":0.6102,"skill":0.5163,"combined":0.4404,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7905,"skill":0.7004,"combined":0.5263,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7729,"skill":0.6959,"combined":0.6362,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.813,"skill":0.7965,"combined":0.7346,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7289,"skill":0.5607,"combined":0.564,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"mogan-decision-31b-it","pub":65.55,"same_skills":57.67,"new_domains":49.39,"v":59.82,"lab":false,"jev":false,"rank_v":52,"rank_pub":31,"name":"mogan-decision-31B-it","eq":{"S":57.28,"O":63.64},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":34.37,"new_domains":25.46}},"new_in_edition":true},{"engine":"jeffan-31b-1710","name":"Jeffan-31b-1710","long_name":"Jeffan-31b-1710","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":31273086512,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tattrongvu/Jeffan-31b-1710","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":241,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8241,"conf":0.8706,"ece":0.0465,"brier":0.2485,"over95":0.0164,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0003,"conf":0.0896,"acc":0.125},{"lo":0.1,"w":0.0075,"conf":0.1541,"acc":0.1575},{"lo":0.2,"w":0.0109,"conf":0.2535,"acc":0.235},{"lo":0.3,"w":0.0217,"conf":0.3548,"acc":0.2925},{"lo":0.4,"w":0.0335,"conf":0.4542,"acc":0.3446},{"lo":0.5,"w":0.0583,"conf":0.552,"acc":0.4654},{"lo":0.6,"w":0.0477,"conf":0.653,"acc":0.5762},{"lo":0.7,"w":0.0638,"conf":0.747,"acc":0.7116},{"lo":0.8,"w":0.0724,"conf":0.8549,"acc":0.7329},{"lo":0.9,"w":0.6839,"conf":0.9809,"acc":0.9495}],"areas":{"knowledge":{"ece":0.0687,"acc":0.7447,"conf":0.8132},"language":{"ece":0.0117,"acc":0.9128,"conf":0.9184},"retrieval":{"ece":0.0883,"acc":0.8169,"conf":0.9048},"tools":{"ece":0.0397,"acc":0.9566,"conf":0.9907},"arts":{"ece":0.0589,"acc":0.7495,"conf":0.7996}}},"gaps":null,"latency":{"median":121,"p95":1514,"mean":336.2},"results":{"1":{"score":0.9752,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":54.9},"2":{"score":0.7565,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":537.4},"3":{"score":0.8799,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":426.3},"4":{"score":0.9193,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":69.2},"5":{"score":0.9638,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":96.1},"6":{"score":0.7996,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":410.1},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":151},"10":{"score":0.6612,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":123},"11":{"score":0.8408,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":818.6},"12":{"score":0.8217,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":41.5},"20":{"score":0.951,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.2},"21":{"score":0.7264,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":39.3},"22":{"score":0.822,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":380.6},"23":{"score":0.6674,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":61.6},"24":{"score":0.8954,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":22.1},"25":{"score":0.5306,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":48.8},"26":{"score":0.9941,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":19.8},"27":{"score":0.9804,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":20},"28":{"score":0.9534,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":19.6},"29":{"score":0.9766,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":25.6},"30":{"score":0.923,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.297,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":128.9},"32":{"score":0.8245,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":84.7},"33":{"score":0.44,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":146.3},"36":{"score":0.479,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":903.1},"37":{"score":0.636,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.9},"38":{"score":0.6336,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":344.7},"39":{"score":0.9422,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":27.7},"40":{"score":0.6875,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":38.7},"41":{"score":0.8418,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":45.4},"42":{"score":0.8782,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":41.5},"43":{"score":0.9018,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":23.7},"44":{"score":0.9552,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":20.1},"45":{"score":0.1597,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":52.3},"50":{"score":0.6301,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":121.7},"56":{"score":0.849,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":102.9},"57":{"score":0.7901,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":46.1},"58":{"score":0.9301,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":23.3},"59":{"score":0.8415,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":57.1},"61":{"score":0.8858,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":40.6},"62":{"score":0.9206,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":33.9},"64":{"score":0.8826,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":23.3}},"benchmarks":{"1":{"raw":0.9752,"skill":0.9665,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8581,"private_eq":0.8953},"2":{"raw":0.7565,"skill":0.7188,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6974,"private_eq":0.7401},"3":{"raw":0.8799,"skill":0.8776,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9276,"private_eq":0.9288},"4":{"raw":0.9193,"skill":0.9183,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9046,"private_eq":0.9059},"5":{"raw":0.9638,"skill":0.9636,"coverage":1,"random":0.006,"tracks":[],"combined":0.8685,"private_eq":0.8698},"6":{"raw":0.7996,"skill":0.5301,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9125,"private_eq":0.9125},"10":{"raw":0.6612,"skill":0.4363,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8408,"skill":0.7698,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.8217,"skill":0.7329,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5024,"private_eq":0.6709},"20":{"raw":0.9519,"skill":0.9039,"coverage":1,"random":0.5,"tracks":[],"combined":0.9432,"private_eq":0.9714},"21":{"raw":0.7264,"skill":0.4528,"coverage":1,"random":0.5,"tracks":[],"combined":0.4679,"private_eq":0.7365},"22":{"raw":0.8241,"skill":0.8227,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6926,"private_eq":0.6975},"23":{"raw":0.6601,"skill":0.3202,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8954,"skill":0.8605,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5306,"skill":0.3741,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6789,"combined":0.568},"26":{"raw":0.9941,"skill":0.9921,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9804,"skill":0.9739,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9534,"skill":0.9068,"coverage":1,"random":0.5,"tracks":[],"combined":0.6883,"private_eq":0.8454},"29":{"raw":0.9766,"skill":0.9688,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.923,"skill":0.9084,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9454,"random":0.25},{"track":"10-choice","raw":0.9007,"random":0.1}],"combined":0.8561,"private_eq":0.8818},"31":{"raw":0.297,"skill":0.2343,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1167,"private_eq":0.1974},"32":{"raw":0.8245,"skill":0.721,"coverage":1,"random":0.371,"tracks":[],"combined":0.3975,"private_eq":0.6247},"33":{"raw":0.44,"skill":0.4326,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4307,"private_eq":0.4436},"36":{"raw":0.479,"skill":0.4106,"coverage":1,"random":0.116,"tracks":[],"combined":0.4291,"private_eq":0.5002},"37":{"raw":0.636,"skill":0.5434,"coverage":1,"random":0.2027,"tracks":[],"combined":0.624,"private_eq":0.7028},"38":{"raw":0.6336,"skill":0.6219,"coverage":1,"random":0.031,"tracks":[],"combined":0.7351,"private_eq":0.7453},"39":{"raw":0.9422,"skill":0.915,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5974,"private_eq":0.7287},"40":{"raw":0.6875,"skill":0.598,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.6223,"headline":false},{"track":"A · English","score":0.6875,"headline":true},{"track":"C · Arabic pairs","score":0.99,"headline":false},{"track":"C · English pairs","score":0.995,"headline":false}],"combined":0.5798,"private_eq":0.6763},"41":{"raw":0.8418,"skill":0.7627,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1013,"private_eq":0.407},"42":{"raw":0.8782,"skill":0.763,"coverage":1,"random":0.486,"tracks":[],"combined":0.9459,"private_eq":0.972},"43":{"raw":0.9018,"skill":0.8441,"coverage":1,"random":0.3697,"tracks":[],"combined":0.7,"private_eq":0.8124},"44":{"raw":0.9552,"skill":0.9104,"coverage":1,"random":0.5,"tracks":[],"combined":0.7868,"private_eq":0.8941},"45":{"raw":0.1597,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6301,"skill":0.4631,"coverage":1,"random":0.311,"tracks":[],"combined":0.3047,"private_eq":0.5257},"56":{"raw":0.849,"skill":0.698,"coverage":1,"random":0.5,"tracks":[],"combined":0.6116,"private_eq":0.8075},"57":{"raw":0.7901,"skill":0.7639,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.7022,"combined":0.6622},"58":{"raw":0.9301,"skill":0.8987,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5731,"combined":0.3751},"59":{"raw":0.8415,"skill":0.6714,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3467,"private_eq":0.688},"61":{"raw":0.8858,"skill":0.7716,"coverage":1,"random":0.5,"tracks":[],"combined":0.7174,"private_eq":0.8598},"62":{"raw":0.9206,"skill":0.8941,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.8826,"skill":0.8533,"coverage":1,"random":0.2,"tracks":[],"combined":0.8361,"private_eq":0.8696}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #241).","Submitted public 0.3 run adopted (kit rescore 73.13, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.65% on 831 rows (bar 98.5%).","Training data undisclosed.","License CC BY-NC 4.0."],"scores":{"balanced_raw":79.74,"balanced_skill":69.75,"breadth_skill":72.64,"public_skill":73.13},"frozen_scores":{"balanced_raw":79.74,"balanced_skill":73.13,"breadth_skill":72.64},"categories":[{"id":"knowledge","raw":0.6982,"skill":0.6319,"combined":0.5593,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8323,"skill":0.76,"combined":0.6084,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7887,"skill":0.7218,"combined":0.6963,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.908,"skill":0.8937,"combined":0.8118,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7792,"skill":0.636,"combined":0.6489,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jeffan-31b-1710","pub":73.13,"same_skills":66.15,"new_domains":59.06,"v":69.75,"lab":false,"jev":false,"rank_v":6,"rank_pub":2,"name":"Jeffan-31b-1710","eq":{"S":65.86,"O":75.58},"rank_tie":2,"tied":true,"parts":{"public":0,"same_skills":39.52,"new_domains":30.23}},"new_in_edition":true},{"engine":"autotrust-jev-27b-v11","name":"AutoTrust-JEV-27B v1.1","long_name":"AutoTrust-JEV-27B v1.1","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27356728560,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/autotrust/AutoTrust-JEV-27B-v1.1","code_url":null,"base_model":"vllm-sr/d3","base_url":"https://huggingface.co/vllm-sr/d3","weights_repo":null,"served_checkpoint":null,"org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"},"pr":231,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7817,"conf":0.8488,"ece":0.0672,"brier":0.3084,"over95":0.0189,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0006,"conf":0.0721,"acc":0.1795},{"lo":0.1,"w":0.0076,"conf":0.1609,"acc":0.1339},{"lo":0.2,"w":0.0108,"conf":0.2543,"acc":0.1462},{"lo":0.3,"w":0.0236,"conf":0.3555,"acc":0.273},{"lo":0.4,"w":0.0361,"conf":0.4554,"acc":0.3619},{"lo":0.5,"w":0.0642,"conf":0.5507,"acc":0.4794},{"lo":0.6,"w":0.0701,"conf":0.6512,"acc":0.5274},{"lo":0.7,"w":0.0792,"conf":0.7521,"acc":0.6327},{"lo":0.8,"w":0.114,"conf":0.8555,"acc":0.7309},{"lo":0.9,"w":0.5939,"conf":0.9798,"acc":0.9401}],"areas":{"knowledge":{"ece":0.1172,"acc":0.6985,"conf":0.8157},"language":{"ece":0.0149,"acc":0.9029,"conf":0.8884},"retrieval":{"ece":0.1473,"acc":0.7295,"conf":0.8764},"tools":{"ece":0.0344,"acc":0.9557,"conf":0.985},"arts":{"ece":0.0823,"acc":0.6673,"conf":0.74}}},"gaps":null,"latency":{"median":79.9,"p95":861.2,"mean":224.7},"results":{"1":{"score":0.9723,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6871,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.876,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.9365,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9709,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7981,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.8977,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.4652,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8258,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7978,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9642,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6301,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.627,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6488,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8868,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5459,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.971,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.9534,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9819,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.8503,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2114,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.7048,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3788,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4919,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5208,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.564,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9447,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.66,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7947,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8754,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8526,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8102,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1697,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4278,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.72,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7364,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.8616,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8375,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8915,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8954,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8295,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9723,"skill":0.9625,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9739,"private_eq":0.9801},"2":{"raw":0.6871,"skill":0.6386,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7156,"private_eq":0.7557},"3":{"raw":0.876,"skill":0.8736,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9171,"private_eq":0.9186},"4":{"raw":0.9365,"skill":0.9357,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8909,"private_eq":0.8925},"5":{"raw":0.9709,"skill":0.9707,"coverage":1,"random":0.006,"tracks":[],"combined":0.869,"private_eq":0.8703},"6":{"raw":0.7981,"skill":0.5266,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8977,"skill":0.8977,"coverage":1,"random":0,"tracks":[],"combined":0.7861,"private_eq":0.7875},"10":{"raw":0.4652,"skill":0.1101,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8258,"skill":0.748,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7978,"skill":0.6971,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4937,"private_eq":0.6652},"20":{"raw":0.9656,"skill":0.9312,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.6301,"skill":0.2603,"coverage":1,"random":0.5,"tracks":[],"combined":0.3069,"private_eq":0.6569},"22":{"raw":0.6198,"skill":0.6168,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5954,"private_eq":0.6021},"23":{"raw":0.6472,"skill":0.2944,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8868,"skill":0.8491,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5459,"skill":0.3946,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5262,"combined":0.362},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.971,"skill":0.9613,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9534,"skill":0.9068,"coverage":1,"random":0.5,"tracks":[],"combined":0.6126,"private_eq":0.808},"29":{"raw":0.9819,"skill":0.9759,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8503,"skill":0.8212,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8878,"random":0.25},{"track":"10-choice","raw":0.8127,"random":0.1}],"combined":0.7706,"private_eq":0.8121},"31":{"raw":0.2114,"skill":0.1411,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1178,"private_eq":0.1984},"32":{"raw":0.7048,"skill":0.5307,"coverage":1,"random":0.371,"tracks":[],"combined":0.3589,"private_eq":0.6007},"33":{"raw":0.3788,"skill":0.3706,"coverage":1,"random":0.0131,"tracks":[],"combined":0.477,"private_eq":0.4887},"36":{"raw":0.4919,"skill":0.4251,"coverage":1,"random":0.116,"tracks":[],"combined":0.4268,"private_eq":0.4982},"37":{"raw":0.5208,"skill":0.3989,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5921,"private_eq":0.6777},"38":{"raw":0.564,"skill":0.5501,"coverage":1,"random":0.031,"tracks":[],"combined":0.7577,"private_eq":0.7669},"39":{"raw":0.9447,"skill":0.9187,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5257,"private_eq":0.6805},"40":{"raw":0.66,"skill":0.5626,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5957,"headline":false},{"track":"A · English","score":0.66,"headline":true},{"track":"C · Arabic pairs","score":0.935,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.5711,"private_eq":0.6696},"41":{"raw":0.7947,"skill":0.6921,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2633,"private_eq":0.5138},"42":{"raw":0.8754,"skill":0.7576,"coverage":1,"random":0.486,"tracks":[],"combined":0.9485,"private_eq":0.9733},"43":{"raw":0.8526,"skill":0.7662,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6332,"private_eq":0.7708},"44":{"raw":0.8102,"skill":0.6204,"coverage":1,"random":0.5,"tracks":[],"combined":0.5351,"private_eq":0.7697},"45":{"raw":0.1697,"skill":0.0067,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4278,"skill":0.1696,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.72,"skill":0.44,"coverage":1,"random":0.5,"tracks":[],"combined":0.9059,"private_eq":0.953},"57":{"raw":0.7364,"skill":0.7035,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6543,"combined":0.6077},"58":{"raw":0.8616,"skill":0.7994,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5474,"combined":0.3374},"59":{"raw":0.8375,"skill":0.6631,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2959,"private_eq":0.6638},"61":{"raw":0.8915,"skill":0.783,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.8954,"skill":0.8605,"coverage":1,"random":0.25,"tracks":[],"combined":0.5562,"private_eq":0.6702},"64":{"raw":0.8295,"skill":0.7869,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #231).","Submitted public 0.3 run adopted (kit rescore 67.32). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.65% on 831 rows (bar 98.5%).","Fine-tuned from vLLM-SR d3 (27B); training includes public train splits of suite datasets (author-declared).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":75.62,"balanced_skill":67.95,"breadth_skill":66.47,"public_skill":67.32},"frozen_scores":{"balanced_raw":75.62,"balanced_skill":67.32,"breadth_skill":66.47},"categories":[{"id":"knowledge","raw":0.644,"skill":0.5519,"combined":0.4811,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8125,"skill":0.734,"combined":0.6138,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7593,"skill":0.6697,"combined":0.7221,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.87,"skill":0.8519,"combined":0.8013,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6867,"skill":0.5099,"combined":0.5799,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"autotrust-jev-27b-v11","pub":67.32,"same_skills":64.08,"new_domains":57.96,"v":67.95,"lab":false,"jev":false,"rank_v":12,"rank_pub":24,"name":"AutoTrust-JEV-27B v1.1","eq":{"S":63.77,"O":74.22},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.26,"new_domains":29.69}},"new_in_edition":true},{"engine":"flexdm-gemma-4-31b","name":"FlexDM · Gemma-4-31B","long_name":"FlexDM · Gemma-4-31B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/1000luca/FlexDM/tree/v1.0","base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":246,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7534,"conf":0.7879,"ece":0.0439,"brier":0.355,"over95":0.0025,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0011,"conf":0.0801,"acc":0.0364},{"lo":0.1,"w":0.0057,"conf":0.1644,"acc":0.0697},{"lo":0.2,"w":0.0135,"conf":0.2551,"acc":0.1355},{"lo":0.3,"w":0.022,"conf":0.3551,"acc":0.2533},{"lo":0.4,"w":0.0416,"conf":0.4558,"acc":0.309},{"lo":0.5,"w":0.0596,"conf":0.5507,"acc":0.4691},{"lo":0.6,"w":0.0691,"conf":0.6551,"acc":0.4924},{"lo":0.7,"w":0.1475,"conf":0.7597,"acc":0.675},{"lo":0.8,"w":0.3211,"conf":0.8428,"acc":0.8495},{"lo":0.9,"w":0.3187,"conf":0.9281,"acc":0.9362}],"areas":{"knowledge":{"ece":0.0845,"acc":0.7141,"conf":0.7094},"language":{"ece":0.0431,"acc":0.8666,"conf":0.8372},"retrieval":{"ece":0.222,"acc":0.6215,"conf":0.8435},"tools":{"ece":0.0807,"acc":0.9402,"conf":0.8595},"arts":{"ece":0.1466,"acc":0.6217,"conf":0.7684}}},"gaps":null,"latency":{"median":108.6,"p95":1106.1,"mean":300.9},"results":{"1":{"score":0.9776,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":528.3},"2":{"score":0.6626,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3799.3},"3":{"score":0.8524,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1781.9},"4":{"score":0.7996,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":472.8},"5":{"score":0.9146,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":749.8},"6":{"score":0.7997,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1473.9},"9":{"score":0.7045,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":688.9},"10":{"score":0.0803,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":594},"11":{"score":0.7878,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":796},"12":{"score":0.7988,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":121.6},"20":{"score":0.9356,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":76.1},"21":{"score":0.645,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":46.1},"22":{"score":0.383,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1575.5},"23":{"score":0.6526,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":160.6},"24":{"score":0.8799,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":72.7},"25":{"score":0.4898,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":141.4},"26":{"score":0.9916,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":58.6},"27":{"score":0.9761,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":60.9},"28":{"score":0.8966,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":57.7},"29":{"score":0.9257,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":142.7},"30":{"score":0.7263,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2376,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":397.3},"32":{"score":0.6875,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":675},"33":{"score":0.3412,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1285.8},"36":{"score":0.486,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":6110.2},"37":{"score":0.5876,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":147.6},"38":{"score":0.4089,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2179.7},"39":{"score":0.9314,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":166.2},"40":{"score":0.5981,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":47.1},"41":{"score":0.8032,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":125.8},"42":{"score":0.7972,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":282.7},"43":{"score":0.8632,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":77.9},"44":{"score":0.8394,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99},"45":{"score":0.1657,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":145.6},"50":{"score":0.5024,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":365.6},"56":{"score":0.9595,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1012.3},"57":{"score":0.7199,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":96.5},"58":{"score":0.7874,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":76.2},"59":{"score":0.7809,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":247.3},"61":{"score":0.8115,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":172.3},"62":{"score":0.853,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":270.2},"64":{"score":0.7936,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":92.7}},"benchmarks":{"1":{"raw":0.9776,"skill":0.9697,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6626,"skill":0.6103,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7162,"private_eq":0.7562},"3":{"raw":0.8524,"skill":0.8496,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.7996,"skill":0.797,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8899,"private_eq":0.8915},"5":{"raw":0.9146,"skill":0.9141,"coverage":1,"random":0.006,"tracks":[],"combined":0.8178,"private_eq":0.82},"6":{"raw":0.7997,"skill":0.5304,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7045,"skill":0.7045,"coverage":1,"random":0,"tracks":[],"combined":0.604,"private_eq":0.6075},"10":{"raw":0.0803,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7878,"skill":0.6931,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7988,"skill":0.6986,"coverage":1,"random":0.3324,"tracks":[],"combined":0.531,"private_eq":0.6898},"20":{"raw":0.9377,"skill":0.8754,"coverage":1,"random":0.5,"tracks":[],"combined":0.9142,"private_eq":0.9571},"21":{"raw":0.645,"skill":0.29,"coverage":1,"random":0.5,"tracks":[],"combined":0.3069,"private_eq":0.6569},"22":{"raw":0.3838,"skill":0.379,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3776,"private_eq":0.3885},"23":{"raw":0.6592,"skill":0.3184,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8799,"skill":0.8399,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4898,"skill":0.3197,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5283,"combined":0.3648},"26":{"raw":0.9916,"skill":0.9888,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9761,"skill":0.9681,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8966,"skill":0.7932,"coverage":1,"random":0.5,"tracks":[],"combined":0.7437,"private_eq":0.8728},"29":{"raw":0.9257,"skill":0.9009,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7263,"skill":0.6745,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8074,"random":0.25},{"track":"10-choice","raw":0.6452,"random":0.1}],"combined":0.7863,"private_eq":0.8249},"31":{"raw":0.2376,"skill":0.1696,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0962,"private_eq":0.1788},"32":{"raw":0.6875,"skill":0.5032,"coverage":1,"random":0.371,"tracks":[],"combined":0.4518,"private_eq":0.6585},"33":{"raw":0.3412,"skill":0.3325,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4512,"private_eq":0.4636},"36":{"raw":0.486,"skill":0.4185,"coverage":1,"random":0.116,"tracks":[],"combined":0.4014,"private_eq":0.476},"37":{"raw":0.5876,"skill":0.4828,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6051,"private_eq":0.6879},"38":{"raw":0.4089,"skill":0.39,"coverage":1,"random":0.031,"tracks":[],"combined":0.601,"private_eq":0.6168},"39":{"raw":0.9314,"skill":0.8991,"coverage":1,"random":0.3201,"tracks":[],"combined":0.591,"private_eq":0.7244},"40":{"raw":0.5981,"skill":0.4829,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5594,"headline":false},{"track":"A · English","score":0.5981,"headline":true},{"track":"C · Arabic pairs","score":0.94,"headline":false},{"track":"C · English pairs","score":0.99,"headline":false}],"combined":0.4943,"private_eq":0.6106},"41":{"raw":0.8032,"skill":0.7047,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5352,"private_eq":0.693},"42":{"raw":0.7972,"skill":0.6054,"coverage":1,"random":0.486,"tracks":[],"combined":0.7023,"private_eq":0.8482},"43":{"raw":0.8632,"skill":0.7829,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6372,"private_eq":0.7733},"44":{"raw":0.8394,"skill":0.6788,"coverage":1,"random":0.5,"tracks":[],"combined":0.6531,"private_eq":0.828},"45":{"raw":0.1657,"skill":0.0019,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5024,"skill":0.2778,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.9595,"skill":0.919,"coverage":1,"random":0.5,"tracks":[],"combined":0.8452,"private_eq":0.923},"57":{"raw":0.7199,"skill":0.685,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5733,"combined":0.5156},"58":{"raw":0.7874,"skill":0.6918,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5544,"combined":0.3477},"59":{"raw":0.7809,"skill":0.5457,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4054,"private_eq":0.716},"61":{"raw":0.8115,"skill":0.623,"coverage":1,"random":0.5,"tracks":[],"combined":0.6209,"private_eq":0.8121},"62":{"raw":0.853,"skill":0.804,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.7936,"skill":0.742,"coverage":1,"random":0.2,"tracks":[],"combined":0.778,"private_eq":0.8236}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #246).","Submitted public 0.3 run adopted (kit rescore 63.63, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8 at load); decision agreement with the submitted rows 100.00% on 831 rows (bar 96%).","Inference technique on frozen Gemma 4 31B weights; no weights published.","Served in FP8 (vLLM quantization at load), so the quantized agreement bar applies.","Prompt layout and input cut chosen on a public-suite sample (author-declared)."],"scores":{"balanced_raw":72.25,"balanced_skill":67.56,"breadth_skill":62.81,"public_skill":63.63},"frozen_scores":{"balanced_raw":72.25,"balanced_skill":63.63,"breadth_skill":62.81},"categories":[{"id":"knowledge","raw":0.6139,"skill":0.511,"combined":0.5039,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7635,"skill":0.6639,"combined":0.6152,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7574,"skill":0.694,"combined":0.6973,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8178,"skill":0.7967,"combined":0.7647,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6536,"skill":0.4804,"combined":0.5276,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"flexdm-gemma-4-31b","pub":63.63,"same_skills":62.98,"new_domains":58.52,"v":67.56,"lab":false,"jev":false,"rank_v":16,"rank_pub":43,"name":"FlexDM · Gemma-4-31B","eq":{"S":62.66,"O":74.92},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.59,"new_domains":29.97}},"new_in_edition":true},{"engine":"visionjev-1.0.0","name":"VisionJEV 1.0.0 (27B)","long_name":"VisionJEV 1.0.0 (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/taoszhang/VisionJEV","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"pr":239,"new":true},"completed":110201,"counts":{"ok":110177,"unsupported":24,"error":0},"lifted":false,"calibration":{"acc":0.7755,"conf":0.8376,"ece":0.0742,"brier":0.3118,"over95":0.0203,"n":34407,"benchmarks":31,"rel":[{"lo":0,"w":0.004,"conf":0.0841,"acc":0.1303},{"lo":0.1,"w":0.0195,"conf":0.1518,"acc":0.2275},{"lo":0.2,"w":0.0175,"conf":0.2499,"acc":0.3678},{"lo":0.3,"w":0.0266,"conf":0.3543,"acc":0.4431},{"lo":0.4,"w":0.0357,"conf":0.4514,"acc":0.2825},{"lo":0.5,"w":0.0705,"conf":0.5523,"acc":0.5054},{"lo":0.6,"w":0.0609,"conf":0.6506,"acc":0.5556},{"lo":0.7,"w":0.0687,"conf":0.7512,"acc":0.5667},{"lo":0.8,"w":0.0891,"conf":0.8524,"acc":0.6956},{"lo":0.9,"w":0.6076,"conf":0.9846,"acc":0.9412}],"areas":{"knowledge":{"ece":0.0967,"acc":0.7206,"conf":0.8132},"language":{"ece":0.0075,"acc":0.8878,"conf":0.8953},"retrieval":{"ece":0.18,"acc":0.679,"conf":0.859},"tools":{"ece":0.0424,"acc":0.9529,"conf":0.9917},"arts":{"ece":0.0935,"acc":0.654,"conf":0.6729}}},"gaps":null,"latency":{"median":89.9,"p95":1099.7,"mean":324.8},"results":{"1":{"score":0.9734,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":595.2},"2":{"score":0.6628,"jev":null,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":5240.5},"3":{"score":0.8755,"jev":null,"metric":"accuracy","requests":508,"answered":506,"unsupported":2,"errors":0,"median_ms":2439.3},"4":{"score":0.8096,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":304.8},"5":{"score":0.9642,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":511.6},"6":{"score":0.7981,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1446.7},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":6223},"10":{"score":0.2708,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":390.8},"11":{"score":0.7868,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":10683.8},"12":{"score":0.7321,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":65.9},"20":{"score":0.9588,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":68.5},"21":{"score":0.6149,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":40.5},"22":{"score":0.6304,"jev":null,"metric":"accuracy","requests":2000,"answered":1975,"unsupported":25,"errors":0,"median_ms":2082.4},"23":{"score":0.6446,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":177.2},"24":{"score":0.8745,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":51.5},"25":{"score":0.5255,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":90.2},"26":{"score":0.992,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":43.8},"27":{"score":0.9727,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":46.7},"28":{"score":0.9321,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":36},"29":{"score":0.9477,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":96.2},"30":{"score":0.7559,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2866,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":337},"32":{"score":0.7008,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":339.9},"33":{"score":0.3424,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1235.9},"36":{"score":0.5026,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5846},"37":{"score":0.6205,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":132.4},"38":{"score":0.5163,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3696.5},"39":{"score":0.9487,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":129.6},"40":{"score":0.6901,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":36.5},"41":{"score":0.8058,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":72.1},"42":{"score":0.8621,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":194.1},"43":{"score":0.8474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":69.3},"44":{"score":0.9672,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":64.7},"45":{"score":0.172,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":96.4},"50":{"score":0.429,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":305.9},"56":{"score":0.7525,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":781.7},"57":{"score":0.7162,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":86.9},"58":{"score":0.8315,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":76.3},"59":{"score":0.8621,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":279.6},"61":{"score":0.9028,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":164},"62":{"score":0.7429,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":279},"64":{"score":0.7519,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":87.4}},"benchmarks":{"1":{"raw":0.9734,"skill":0.9641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9669,"private_eq":0.975},"2":{"raw":0.6609,"skill":0.6084,"coverage":0.9971,"random":0.1341,"tracks":[],"combined":0.684,"private_eq":0.7286},"3":{"raw":0.8721,"skill":0.8696,"coverage":0.9961,"random":0.0189,"tracks":[],"combined":0.9328,"private_eq":0.9338},"4":{"raw":0.8096,"skill":0.8072,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8812,"private_eq":0.883},"5":{"raw":0.9642,"skill":0.964,"coverage":1,"random":0.006,"tracks":[],"combined":0.8394,"private_eq":0.8412},"6":{"raw":0.7981,"skill":0.5266,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9327,"private_eq":0.9325},"10":{"raw":0.2708,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7868,"skill":0.6917,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7321,"skill":0.5987,"coverage":1,"random":0.3324,"tracks":[],"combined":0.545,"private_eq":0.699},"20":{"raw":0.9599,"skill":0.9198,"coverage":1,"random":0.5,"tracks":[],"combined":0.9432,"private_eq":0.9714},"21":{"raw":0.6149,"skill":0.2298,"coverage":1,"random":0.5,"tracks":[],"combined":0.2768,"private_eq":0.642},"22":{"raw":0.6146,"skill":0.6116,"coverage":0.9875,"random":0.0078,"tracks":[],"combined":0.5887,"private_eq":0.5956},"23":{"raw":0.6505,"skill":0.3009,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8745,"skill":0.8327,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5255,"skill":0.3673,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4971,"combined":0.3228},"26":{"raw":0.992,"skill":0.9893,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9727,"skill":0.9636,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9321,"skill":0.8642,"coverage":1,"random":0.5,"tracks":[],"combined":0.6933,"private_eq":0.8479},"29":{"raw":0.9477,"skill":0.9303,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7559,"skill":0.7075,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8089,"random":0.25},{"track":"10-choice","raw":0.7028,"random":0.1}],"combined":0.7274,"private_eq":0.7768},"31":{"raw":0.2866,"skill":0.223,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0842,"private_eq":0.1679},"32":{"raw":0.7008,"skill":0.5243,"coverage":1,"random":0.371,"tracks":[],"combined":0.3975,"private_eq":0.6247},"33":{"raw":0.3424,"skill":0.3337,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3845,"private_eq":0.3985},"36":{"raw":0.5026,"skill":0.4373,"coverage":1,"random":0.116,"tracks":[],"combined":0.3965,"private_eq":0.4717},"37":{"raw":0.6205,"skill":0.5239,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5366,"private_eq":0.6339},"38":{"raw":0.5163,"skill":0.5008,"coverage":1,"random":0.031,"tracks":[],"combined":0.5997,"private_eq":0.6156},"39":{"raw":0.9487,"skill":0.9245,"coverage":1,"random":0.3201,"tracks":[],"combined":0.508,"private_eq":0.6686},"40":{"raw":0.6901,"skill":0.6013,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5362,"headline":false},{"track":"A · English","score":0.6901,"headline":true},{"track":"C · Arabic pairs","score":0.975,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.5605,"private_eq":0.6615},"41":{"raw":0.8058,"skill":0.7087,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0808,"private_eq":0.3935},"42":{"raw":0.8621,"skill":0.7317,"coverage":1,"random":0.486,"tracks":[],"combined":0.9426,"private_eq":0.9703},"43":{"raw":0.8474,"skill":0.7578,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6457,"private_eq":0.7786},"44":{"raw":0.9672,"skill":0.9344,"coverage":1,"random":0.5,"tracks":[],"combined":0.8511,"private_eq":0.9259},"45":{"raw":0.1717,"skill":0.009,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.429,"skill":0.1713,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.7525,"skill":0.505,"coverage":1,"random":0.5,"tracks":[],"combined":0.6203,"private_eq":0.8118},"57":{"raw":0.7162,"skill":0.6808,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6328,"combined":0.5833},"58":{"raw":0.8315,"skill":0.7558,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5587,"combined":0.354},"59":{"raw":0.8621,"skill":0.7141,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4031,"private_eq":0.7149},"61":{"raw":0.9028,"skill":0.8056,"coverage":1,"random":0.5,"tracks":[],"combined":0.6492,"private_eq":0.8261},"62":{"raw":0.7429,"skill":0.6572,"coverage":1,"random":0.25,"tracks":[],"combined":0.5562,"private_eq":0.6702},"64":{"raw":0.7519,"skill":0.6899,"coverage":1,"random":0.2,"tracks":[],"combined":0.681,"private_eq":0.7469}},"coverage":0.9993,"pending":0.0007,"notes":["New in the 2026-10-20 PR batch (PR #239).","Submitted public 0.3 run adopted (kit rescore 66.46). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.70% on 825 rows (bar 98.5%).","The submitted runs used Cambricon MLU hardware with private code; the lab ran the repository's CUDA path.","Questions over its declared 8,192-token limit count as unsupported."],"scores":{"balanced_raw":74.73,"balanced_skill":66.88,"breadth_skill":65.7,"public_skill":66.46},"frozen_scores":{"balanced_raw":74.73,"balanced_skill":66.46,"breadth_skill":65.7},"categories":[{"id":"knowledge","raw":0.6427,"skill":0.5578,"combined":0.5032,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.7966,"skill":0.7136,"combined":0.5893,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7587,"skill":0.6795,"combined":0.6586,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8553,"skill":0.827,"combined":0.8245,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.6701,"skill":0.4872,"combined":0.5429,"coverage":0.9979,"pending":0,"n":6}],"v03":{"engine":"visionjev-1.0.0","pub":66.46,"same_skills":62.8,"new_domains":57.37,"v":66.88,"lab":false,"jev":false,"rank_v":20,"rank_pub":26,"name":"VisionJEV 1.0.0 (27B)","eq":{"S":62.47,"O":73.5},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.48,"new_domains":29.4}},"new_in_edition":true},{"engine":"nimble-v4-max","name":"Bespoke Nimble v4 Max (27B)","long_name":"Bespoke Nimble v4 Max (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27356728560,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/bespokelabs/Nimble-v4-max","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"bespokelabs","name":"Bespoke Labs","logo":"data/orgs/bespokelabs.png"},"pr":226,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7938,"conf":0.8545,"ece":0.0635,"brier":0.2875,"over95":0.0231,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0061,"conf":0.0805,"acc":0.1097},{"lo":0.1,"w":0.0142,"conf":0.1448,"acc":0.189},{"lo":0.2,"w":0.0095,"conf":0.2516,"acc":0.3157},{"lo":0.3,"w":0.0138,"conf":0.3555,"acc":0.3373},{"lo":0.4,"w":0.0414,"conf":0.4557,"acc":0.3413},{"lo":0.5,"w":0.0677,"conf":0.5459,"acc":0.4905},{"lo":0.6,"w":0.0599,"conf":0.6497,"acc":0.5221},{"lo":0.7,"w":0.0611,"conf":0.749,"acc":0.6287},{"lo":0.8,"w":0.0899,"conf":0.8542,"acc":0.6948},{"lo":0.9,"w":0.6363,"conf":0.9858,"acc":0.9481}],"areas":{"knowledge":{"ece":0.1101,"acc":0.7059,"conf":0.8046},"language":{"ece":0.0148,"acc":0.8935,"conf":0.9083},"retrieval":{"ece":0.0202,"acc":0.7982,"conf":0.7983},"tools":{"ece":0.0391,"acc":0.9496,"conf":0.9882},"arts":{"ece":0.1289,"acc":0.6933,"conf":0.8223}}},"gaps":null,"latency":{"median":107.1,"p95":632.5,"mean":253},"results":{"1":{"score":0.9799,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":261.1},"2":{"score":0.7288,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1419.7},"3":{"score":0.8799,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":932.6},"4":{"score":0.7993,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":121.4},"5":{"score":0.9161,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":195.6},"6":{"score":0.7964,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":409.1},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":443},"10":{"score":0.7328,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":152.3},"11":{"score":0.8587,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":528.4},"12":{"score":0.7551,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":116.1},"20":{"score":0.9658,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":108.5},"21":{"score":0.6339,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":112.7},"22":{"score":0.808,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":814.8},"23":{"score":0.6666,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111.7},"24":{"score":0.8786,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":109.1},"25":{"score":0.5255,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":110.9},"26":{"score":0.9907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":107.1},"27":{"score":0.9701,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":106.7},"28":{"score":0.9416,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":106.6},"29":{"score":0.9788,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":109},"30":{"score":0.7589,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2732,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":134.8},"32":{"score":0.6995,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":140},"33":{"score":0.2358,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":276.5},"36":{"score":0.4967,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2114.6},"37":{"score":0.6291,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":113.9},"38":{"score":0.3947,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1657.5},"39":{"score":0.9427,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":147.8},"40":{"score":0.6122,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":112.9},"41":{"score":0.7774,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":107.4},"42":{"score":0.8613,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":116.2},"43":{"score":0.8667,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":107.2},"44":{"score":0.8948,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":107.1},"45":{"score":0.1238,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":110.6},"50":{"score":0.4302,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":121.9},"56":{"score":0.914,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":245.2},"57":{"score":0.7102,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":108.1},"58":{"score":0.9352,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":107.7},"59":{"score":0.8596,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":121.4},"61":{"score":0.8942,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":110.7},"62":{"score":0.9074,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":122.8},"64":{"score":0.8144,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":106.8}},"benchmarks":{"1":{"raw":0.9799,"skill":0.9729,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9874,"private_eq":0.99},"2":{"raw":0.7288,"skill":0.6868,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7127,"private_eq":0.7532},"3":{"raw":0.8799,"skill":0.8776,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8961,"private_eq":0.8982},"4":{"raw":0.7993,"skill":0.7967,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8611,"private_eq":0.8634},"5":{"raw":0.9161,"skill":0.9156,"coverage":1,"random":0.006,"tracks":[],"combined":0.8669,"private_eq":0.8682},"6":{"raw":0.7964,"skill":0.5226,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9403,"private_eq":0.94},"10":{"raw":0.7328,"skill":0.5554,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8587,"skill":0.7957,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7551,"skill":0.6332,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5195,"private_eq":0.6822},"20":{"raw":0.967,"skill":0.934,"coverage":1,"random":0.5,"tracks":[],"combined":0.965,"private_eq":0.9822},"21":{"raw":0.6339,"skill":0.2679,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.8041,"skill":0.8025,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5482,"private_eq":0.5558},"23":{"raw":0.655,"skill":0.31,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8786,"skill":0.8381,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5255,"skill":0.3673,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5127,"combined":0.3438},"26":{"raw":0.9907,"skill":0.9876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9416,"skill":0.8832,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"29":{"raw":0.9788,"skill":0.9717,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7589,"skill":0.7117,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8158,"random":0.25},{"track":"10-choice","raw":0.702,"random":0.1}],"combined":0.7431,"private_eq":0.7896},"31":{"raw":0.2732,"skill":0.2084,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0853,"private_eq":0.1689},"32":{"raw":0.6995,"skill":0.5223,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.2358,"skill":0.2257,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1275,"private_eq":0.1478},"36":{"raw":0.4967,"skill":0.4307,"coverage":1,"random":0.116,"tracks":[],"combined":0.4192,"private_eq":0.4915},"37":{"raw":0.6291,"skill":0.5348,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6124,"private_eq":0.6937},"38":{"raw":0.3947,"skill":0.3753,"coverage":1,"random":0.031,"tracks":[],"combined":0.605,"private_eq":0.6206},"39":{"raw":0.9427,"skill":0.9157,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5772,"private_eq":0.7151},"40":{"raw":0.6122,"skill":0.5011,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4785,"headline":false},{"track":"A · English","score":0.6122,"headline":true},{"track":"C · Arabic pairs","score":0.96,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.4655,"private_eq":0.5885},"41":{"raw":0.7774,"skill":0.6662,"coverage":1,"random":0.3333,"tracks":[],"combined":0.6329,"private_eq":0.7574},"42":{"raw":0.8613,"skill":0.7302,"coverage":1,"random":0.486,"tracks":[],"combined":0.9278,"private_eq":0.9628},"43":{"raw":0.8667,"skill":0.7885,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6499,"private_eq":0.7812},"44":{"raw":0.8948,"skill":0.7896,"coverage":1,"random":0.5,"tracks":[],"combined":0.7441,"private_eq":0.873},"45":{"raw":0.1238,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4302,"skill":0.173,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.914,"skill":0.828,"coverage":1,"random":0.5,"tracks":[],"combined":0.2395,"private_eq":0.6236},"57":{"raw":0.7102,"skill":0.6741,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6192,"combined":0.5678},"58":{"raw":0.9352,"skill":0.9061,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5428,"combined":0.3307},"59":{"raw":0.8596,"skill":0.7089,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3313,"private_eq":0.6807},"61":{"raw":0.8942,"skill":0.7884,"coverage":1,"random":0.5,"tracks":[],"combined":0.6322,"private_eq":0.8177},"62":{"raw":0.9074,"skill":0.8765,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.8144,"skill":0.768,"coverage":1,"random":0.2,"tracks":[],"combined":0.7877,"private_eq":0.8313}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #226).","Submitted public 0.3 run adopted (kit rescore 68.46, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.85% on 891 rows (bar 98.5%).","A separate model from Bespoke Nimble 9B v3, which stays on the board.","License CC BY-NC 4.0."],"scores":{"balanced_raw":75.81,"balanced_skill":66.56,"breadth_skill":67.59,"public_skill":68.46},"frozen_scores":{"balanced_raw":75.81,"balanced_skill":68.46,"breadth_skill":67.59},"categories":[{"id":"knowledge","raw":0.6344,"skill":0.5536,"combined":0.4618,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7859,"skill":0.7041,"combined":0.6386,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7715,"skill":0.7156,"combined":0.6153,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.9015,"skill":0.8859,"combined":0.8391,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7174,"skill":0.5426,"combined":0.5672,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"nimble-v4-max","pub":68.46,"same_skills":62.56,"new_domains":57.01,"v":66.56,"lab":false,"jev":false,"rank_v":23,"rank_pub":15,"name":"Bespoke Nimble v4 Max (27B)","eq":{"S":62.24,"O":73.05},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.34,"new_domains":29.22}},"new_in_edition":true},{"engine":"aleria-judge-1.1.1","name":"Aleria Judge 1.1.1 (Gemma 4 31B)","long_name":"Aleria Judge 1.1.1 (Gemma 4 31B)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/ALERIA-ML/aleria-judge/tree/v1.1.1","base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":237,"replaces":"morj-1.0.1","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7547,"conf":0.8414,"ece":0.0887,"brier":0.3643,"over95":0.0218,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0008,"conf":0.0863,"acc":0.1256},{"lo":0.1,"w":0.0149,"conf":0.1545,"acc":0.206},{"lo":0.2,"w":0.0193,"conf":0.2476,"acc":0.2567},{"lo":0.3,"w":0.022,"conf":0.3561,"acc":0.2431},{"lo":0.4,"w":0.0353,"conf":0.4485,"acc":0.3988},{"lo":0.5,"w":0.0504,"conf":0.5408,"acc":0.4874},{"lo":0.6,"w":0.0406,"conf":0.6465,"acc":0.5045},{"lo":0.7,"w":0.0711,"conf":0.7564,"acc":0.5402},{"lo":0.8,"w":0.1382,"conf":0.8595,"acc":0.6702},{"lo":0.9,"w":0.6074,"conf":0.9623,"acc":0.9071}],"areas":{"knowledge":{"ece":0.0822,"acc":0.7044,"conf":0.7865},"language":{"ece":0.0545,"acc":0.8737,"conf":0.9241},"retrieval":{"ece":0.2588,"acc":0.611,"conf":0.8674},"tools":{"ece":0.0174,"acc":0.9385,"conf":0.9419},"arts":{"ece":0.1199,"acc":0.6456,"conf":0.7212}}},"gaps":null,"latency":{"median":117.3,"p95":698,"mean":336.5},"results":{"1":{"score":0.9782,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":264.5},"2":{"score":0.697,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3857.3},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2026.9},"4":{"score":0.794,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":401.6},"5":{"score":0.9187,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":584.4},"6":{"score":0.7992,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":584},"9":{"score":0.6591,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1069.1},"10":{"score":0.0175,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":327.7},"11":{"score":0.7963,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":818.9},"12":{"score":0.7862,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":69},"20":{"score":0.9772,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":90.7},"21":{"score":0.664,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":63.6},"22":{"score":0.428,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3538.9},"23":{"score":0.6546,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":184.7},"24":{"score":0.8817,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":59.1},"25":{"score":0.5255,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":92.1},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":54.9},"27":{"score":0.9795,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":57.9},"28":{"score":0.8832,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":61.2},"29":{"score":0.9347,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":94.2},"30":{"score":0.7835,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2472,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":398.5},"32":{"score":0.6981,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":350.1},"33":{"score":0.3188,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":349.6},"36":{"score":0.5105,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5445.7},"37":{"score":0.5731,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":94},"38":{"score":0.4189,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3611.3},"39":{"score":0.939,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":80.9},"40":{"score":0.6265,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":60.8},"41":{"score":0.7966,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":70.7},"42":{"score":0.8194,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":139},"43":{"score":0.8211,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":71.2},"44":{"score":0.8342,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":90.1},"45":{"score":0.1537,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":120},"50":{"score":0.4857,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":264.1},"56":{"score":0.899,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":406.6},"57":{"score":0.7297,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":88.7},"58":{"score":0.7619,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":63.9},"59":{"score":0.8039,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":189.6},"61":{"score":0.7602,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":134.4},"62":{"score":0.8264,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":220.7},"64":{"score":0.7955,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":70.3}},"benchmarks":{"1":{"raw":0.9782,"skill":0.9705,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9329,"private_eq":0.9501},"2":{"raw":0.697,"skill":0.65,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7098,"private_eq":0.7507},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8699,"private_eq":0.8728},"4":{"raw":0.794,"skill":0.7914,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8914,"private_eq":0.893},"5":{"raw":0.9187,"skill":0.9182,"coverage":1,"random":0.006,"tracks":[],"combined":0.7784,"private_eq":0.7812},"6":{"raw":0.7992,"skill":0.5292,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6591,"skill":0.6591,"coverage":1,"random":0,"tracks":[],"combined":0.6141,"private_eq":0.6175},"10":{"raw":0.0175,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7963,"skill":0.7054,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7862,"skill":0.6797,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5362,"private_eq":0.6932},"20":{"raw":0.9778,"skill":0.9557,"coverage":1,"random":0.5,"tracks":[],"combined":0.965,"private_eq":0.9822},"21":{"raw":0.664,"skill":0.328,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.4354,"skill":0.4309,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4418,"private_eq":0.4515},"23":{"raw":0.6544,"skill":0.3087,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8817,"skill":0.8423,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5255,"skill":0.3673,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5522,"combined":0.3971},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9795,"skill":0.9727,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8832,"skill":0.7664,"coverage":1,"random":0.5,"tracks":[],"combined":0.7235,"private_eq":0.8628},"29":{"raw":0.9347,"skill":0.9129,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7835,"skill":0.7418,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8408,"random":0.25},{"track":"10-choice","raw":0.7263,"random":0.1}],"combined":0.7554,"private_eq":0.7997},"31":{"raw":0.2472,"skill":0.1801,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0939,"private_eq":0.1767},"32":{"raw":0.6981,"skill":0.52,"coverage":1,"random":0.371,"tracks":[],"combined":0.4671,"private_eq":0.668},"33":{"raw":0.3188,"skill":0.3098,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3794,"private_eq":0.3935},"36":{"raw":0.5105,"skill":0.4463,"coverage":1,"random":0.116,"tracks":[],"combined":0.4099,"private_eq":0.4834},"37":{"raw":0.5731,"skill":0.4646,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5788,"private_eq":0.6672},"38":{"raw":0.4189,"skill":0.4003,"coverage":1,"random":0.031,"tracks":[],"combined":0.6116,"private_eq":0.627},"39":{"raw":0.939,"skill":0.9103,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5941,"private_eq":0.7265},"40":{"raw":0.6265,"skill":0.5195,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5855,"headline":false},{"track":"A · English","score":0.6265,"headline":true},{"track":"C · Arabic pairs","score":0.885,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.5048,"private_eq":0.6187},"41":{"raw":0.7966,"skill":0.695,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5396,"private_eq":0.6959},"42":{"raw":0.8194,"skill":0.6486,"coverage":1,"random":0.486,"tracks":[],"combined":0.7466,"private_eq":0.8707},"43":{"raw":0.8211,"skill":0.7161,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6122,"private_eq":0.7577},"44":{"raw":0.8342,"skill":0.6684,"coverage":1,"random":0.5,"tracks":[],"combined":0.6316,"private_eq":0.8174},"45":{"raw":0.1537,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4857,"skill":0.2536,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.899,"skill":0.798,"coverage":1,"random":0.5,"tracks":[],"combined":0.7674,"private_eq":0.8845},"57":{"raw":0.7297,"skill":0.696,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5869,"combined":0.531},"58":{"raw":0.7619,"skill":0.6549,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5462,"combined":0.3357},"59":{"raw":0.8039,"skill":0.5934,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4094,"private_eq":0.7179},"61":{"raw":0.7602,"skill":0.5204,"coverage":1,"random":0.5,"tracks":[],"combined":0.5357,"private_eq":0.77},"62":{"raw":0.8264,"skill":0.7685,"coverage":1,"random":0.25,"tracks":[],"combined":0.6108,"private_eq":0.7107},"64":{"raw":0.7955,"skill":0.7444,"coverage":1,"random":0.2,"tracks":[],"combined":0.8264,"private_eq":0.8619}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #237).","Replaces the earlier board entry morj-1.0.1 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 63.46, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.90% on 831 rows (bar 98.5%).","Inference technique on frozen Gemma 4 31B weights; the project was formerly named MORJ."],"scores":{"balanced_raw":72.24,"balanced_skill":66.67,"breadth_skill":62.73,"public_skill":63.46},"frozen_scores":{"balanced_raw":72.24,"balanced_skill":63.46,"breadth_skill":62.73},"categories":[{"id":"knowledge","raw":0.614,"skill":0.5098,"combined":0.493,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7729,"skill":0.6791,"combined":0.6251,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7424,"skill":0.6621,"combined":0.6633,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8114,"skill":0.7896,"combined":0.7589,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6688,"skill":0.5035,"combined":0.5582,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"aleria-judge-1.1.1","pub":63.46,"same_skills":62.39,"new_domains":57.44,"v":66.67,"lab":false,"jev":false,"rank_v":21,"rank_pub":45,"name":"Aleria Judge 1.1.1 (Gemma 4 31B)","eq":{"S":62.06,"O":73.59},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.23,"new_domains":29.43}},"new_in_edition":true},{"engine":"anyjev-tacitus-31b","name":"AnyJev Tacitus 31B","long_name":"AnyJev Tacitus 31B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":31273086512,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/morriszjm/AnyJev-Tacitus-31B","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":229,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7836,"conf":0.8068,"ece":0.0232,"brier":0.2966,"over95":0.0136,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0007,"conf":0.0861,"acc":0},{"lo":0.1,"w":0.0149,"conf":0.1585,"acc":0.1509},{"lo":0.2,"w":0.0186,"conf":0.2508,"acc":0.1408},{"lo":0.3,"w":0.0473,"conf":0.3534,"acc":0.3387},{"lo":0.4,"w":0.041,"conf":0.4497,"acc":0.3437},{"lo":0.5,"w":0.0751,"conf":0.5485,"acc":0.5386},{"lo":0.6,"w":0.0914,"conf":0.651,"acc":0.651},{"lo":0.7,"w":0.086,"conf":0.7511,"acc":0.7174},{"lo":0.8,"w":0.0959,"conf":0.8563,"acc":0.8289},{"lo":0.9,"w":0.5291,"conf":0.9774,"acc":0.9591}],"areas":{"knowledge":{"ece":0.0696,"acc":0.711,"conf":0.779},"language":{"ece":0.01,"acc":0.887,"conf":0.8786},"retrieval":{"ece":0.0752,"acc":0.7804,"conf":0.7861},"tools":{"ece":0.0438,"acc":0.9402,"conf":0.9781},"arts":{"ece":0.0263,"acc":0.6512,"conf":0.6472}}},"gaps":null,"latency":{"median":95.2,"p95":999.6,"mean":264.8},"results":{"1":{"score":0.9782,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6618,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8622,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.817,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9202,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7983,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.75,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6471,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8522,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.776,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9886,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6781,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.2635,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6944,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8858,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5306,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9924,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9795,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8848,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.957,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.876,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2738,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6676,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3467,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4703,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5925,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.3642,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9486,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6817,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.8115,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8274,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8614,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.807,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.6086,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.872,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.739,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7954,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8334,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8317,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8765,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8371,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9782,"skill":0.9705,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9602,"private_eq":0.9701},"2":{"raw":0.6618,"skill":0.6094,"coverage":1,"random":0.1341,"tracks":[],"combined":0.698,"private_eq":0.7406},"3":{"raw":0.8622,"skill":0.8595,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9013,"private_eq":0.9033},"4":{"raw":0.817,"skill":0.8146,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8885,"private_eq":0.8902},"5":{"raw":0.9202,"skill":0.9197,"coverage":1,"random":0.006,"tracks":[],"combined":0.7684,"private_eq":0.7714},"6":{"raw":0.7983,"skill":0.5271,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.75,"skill":0.75,"coverage":1,"random":0,"tracks":[],"combined":0.6824,"private_eq":0.685},"10":{"raw":0.6471,"skill":0.4128,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8522,"skill":0.7863,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.776,"skill":0.6645,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5069,"private_eq":0.6739},"20":{"raw":0.9889,"skill":0.9779,"coverage":1,"random":0.5,"tracks":[],"combined":0.9721,"private_eq":0.9857},"21":{"raw":0.6781,"skill":0.3562,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.2511,"skill":0.2453,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2042,"private_eq":0.2184},"23":{"raw":0.6923,"skill":0.3846,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8858,"skill":0.8477,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5306,"skill":0.3741,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5324,"combined":0.3704},"26":{"raw":0.9924,"skill":0.9899,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9795,"skill":0.9727,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8848,"skill":0.7696,"coverage":1,"random":0.5,"tracks":[],"combined":0.6984,"private_eq":0.8504},"29":{"raw":0.957,"skill":0.9427,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.876,"skill":0.8528,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9143,"random":0.25},{"track":"10-choice","raw":0.8378,"random":0.1}],"combined":0.8167,"private_eq":0.8497},"31":{"raw":0.2738,"skill":0.2091,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0929,"private_eq":0.1758},"32":{"raw":0.6676,"skill":0.4715,"coverage":1,"random":0.371,"tracks":[],"combined":0.3899,"private_eq":0.62},"33":{"raw":0.3467,"skill":0.338,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4256,"private_eq":0.4386},"36":{"raw":0.4703,"skill":0.4007,"coverage":1,"random":0.116,"tracks":[],"combined":0.4045,"private_eq":0.4787},"37":{"raw":0.5925,"skill":0.4888,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5944,"private_eq":0.6795},"38":{"raw":0.3642,"skill":0.3439,"coverage":1,"random":0.031,"tracks":[],"combined":0.5526,"private_eq":0.5704},"39":{"raw":0.9486,"skill":0.9244,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5342,"private_eq":0.6862},"40":{"raw":0.6817,"skill":0.5905,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.6077,"headline":false},{"track":"A · English","score":0.6817,"headline":true},{"track":"C · Arabic pairs","score":0.925,"headline":false},{"track":"C · English pairs","score":0.995,"headline":false}],"combined":0.6255,"private_eq":0.7114},"41":{"raw":0.8115,"skill":0.7173,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4597,"private_eq":0.6432},"42":{"raw":0.8274,"skill":0.6642,"coverage":1,"random":0.486,"tracks":[],"combined":0.963,"private_eq":0.9807},"43":{"raw":0.8614,"skill":0.7801,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6666,"private_eq":0.7916},"44":{"raw":0.807,"skill":0.614,"coverage":1,"random":0.5,"tracks":[],"combined":0.5727,"private_eq":0.7883},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6086,"skill":0.4319,"coverage":1,"random":0.311,"tracks":[],"combined":0.2322,"private_eq":0.4763},"56":{"raw":0.872,"skill":0.744,"coverage":1,"random":0.5,"tracks":[],"combined":0.6116,"private_eq":0.8075},"57":{"raw":0.739,"skill":0.7064,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5508,"combined":0.49},"58":{"raw":0.7954,"skill":0.7034,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5175,"combined":0.2936},"59":{"raw":0.8334,"skill":0.6546,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4108,"private_eq":0.7186},"61":{"raw":0.8317,"skill":0.6634,"coverage":1,"random":0.5,"tracks":[],"combined":0.6721,"private_eq":0.8374},"62":{"raw":0.8765,"skill":0.8353,"coverage":1,"random":0.25,"tracks":[],"combined":0.652,"private_eq":0.7412},"64":{"raw":0.8371,"skill":0.7964,"coverage":1,"random":0.2,"tracks":[],"combined":0.8167,"private_eq":0.8542}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #229).","Submitted public 0.3 run adopted (kit rescore 65.52, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8 at load); decision agreement with the submitted rows 98.29% on 827 rows (bar 96%).","A separate model from AnyJev L0 · Gemma-4-31B, which stays on the board.","Served in FP8 at load as in the submitted run, so the quantized agreement bar applies.","Run with a 32,768-token context window on one card; the author used 114,688 across four GPUs.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":73.5,"balanced_skill":66.47,"breadth_skill":64.8,"public_skill":65.52},"frozen_scores":{"balanced_raw":73.5,"balanced_skill":65.52,"breadth_skill":64.8},"categories":[{"id":"knowledge","raw":0.6257,"skill":0.5275,"combined":0.4803,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7871,"skill":0.7032,"combined":0.6375,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7493,"skill":0.6755,"combined":0.6593,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8327,"skill":0.8131,"combined":0.79,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.676,"skill":0.532,"combined":0.5104,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"anyjev-tacitus-31b","pub":65.52,"same_skills":62.48,"new_domains":56.93,"v":66.47,"lab":false,"jev":false,"rank_v":24,"rank_pub":32,"name":"AnyJev Tacitus 31B","eq":{"S":62.15,"O":72.95},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.29,"new_domains":29.18}},"new_in_edition":true},{"engine":"pick-150m-v5","name":"PICK-150M v5","long_name":"PICK-150M v5","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":149015808,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/lCREEl/pick-150m","code_url":null,"base_model":"cross-encoder/ettin-reranker-150m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-150m-v1","weights_repo":null,"served_checkpoint":null,"pr":233,"new":true},"completed":110201,"counts":{"ok":110192,"unsupported":9,"error":0},"lifted":false,"calibration":{"acc":0.5637,"conf":0.6886,"ece":0.1249,"brier":0.5526,"over95":0.0316,"n":34410,"benchmarks":31,"rel":[{"lo":0,"w":0.0529,"conf":0.0648,"acc":0.0311},{"lo":0.1,"w":0.0113,"conf":0.132,"acc":0.0632},{"lo":0.2,"w":0.0174,"conf":0.273,"acc":0.2601},{"lo":0.3,"w":0.0566,"conf":0.3546,"acc":0.2394},{"lo":0.4,"w":0.0812,"conf":0.451,"acc":0.3551},{"lo":0.5,"w":0.1668,"conf":0.5479,"acc":0.478},{"lo":0.6,"w":0.1127,"conf":0.6468,"acc":0.5053},{"lo":0.7,"w":0.0911,"conf":0.7469,"acc":0.5598},{"lo":0.8,"w":0.0884,"conf":0.8514,"acc":0.6242},{"lo":0.9,"w":0.3217,"conf":0.9783,"acc":0.8443}],"areas":{"knowledge":{"ece":0.179,"acc":0.3915,"conf":0.5704},"language":{"ece":0.1074,"acc":0.6971,"conf":0.8045},"retrieval":{"ece":0.0719,"acc":0.7425,"conf":0.8087},"tools":{"ece":0.1889,"acc":0.7518,"conf":0.931},"arts":{"ece":0.0672,"acc":0.4119,"conf":0.4749}}},"gaps":null,"latency":{"median":15.7,"p95":178.4,"mean":29.8},"results":{"1":{"score":0.7367,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":74.3},"2":{"score":0.5462,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":1750.6},"3":{"score":0.4134,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1872.1},"4":{"score":0.9172,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":184.8},"5":{"score":0.9109,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":433.6},"6":{"score":0.5203,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":325},"9":{"score":0.9432,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":365},"10":{"score":0.5286,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":154},"11":{"score":0.7785,"jev":null,"metric":"macro-F1","requests":123,"answered":121,"unsupported":2,"errors":0,"median_ms":698.6},"12":{"score":0.4569,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":32.6},"20":{"score":0.5352,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.8},"21":{"score":0.5213,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":31.8},"22":{"score":0.014,"jev":null,"metric":"accuracy","requests":2000,"answered":1996,"unsupported":4,"errors":0,"median_ms":1516.5},"23":{"score":0.4562,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.2},"24":{"score":0.3411,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":32.5},"25":{"score":0.2398,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":33.4},"26":{"score":0.6418,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":32.2},"27":{"score":0.442,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":33.3},"28":{"score":0.5627,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":31.9},"29":{"score":0.4453,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":34.3},"30":{"score":0.1789,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0762,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.6},"32":{"score":0.5505,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":137.5},"33":{"score":0.0018,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":109.9},"36":{"score":0.2719,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3113},"37":{"score":0.3257,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.7},"38":{"score":0.2266,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1605.3},"39":{"score":0.8992,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":28.4},"40":{"score":0.1408,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":27.7},"41":{"score":0.7402,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":28.6},"42":{"score":0.5895,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":45.4},"43":{"score":0.414,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":27.1},"44":{"score":0.675,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.2},"45":{"score":0.13,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":30},"50":{"score":0.494,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":100.3},"56":{"score":0.491,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":66.6},"57":{"score":0.1367,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":29.9},"58":{"score":0.3492,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":27.8},"59":{"score":0.6569,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":84.7},"61":{"score":0.713,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":44.9},"62":{"score":0.6725,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":76.4},"64":{"score":0.2973,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":28}},"benchmarks":{"1":{"raw":0.7367,"skill":0.6446,"coverage":1,"random":0.2592,"tracks":[],"combined":0.5174,"private_eq":0.6458},"2":{"raw":0.5422,"skill":0.4713,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.5461,"private_eq":0.6106},"3":{"raw":0.4134,"skill":0.4021,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3345,"private_eq":0.3535},"4":{"raw":0.9172,"skill":0.9161,"coverage":1,"random":0.0127,"tracks":[],"combined":0.6691,"private_eq":0.676},"5":{"raw":0.9109,"skill":0.9104,"coverage":1,"random":0.006,"tracks":[],"combined":0.4984,"private_eq":0.5061},"6":{"raw":0.5203,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9432,"skill":0.9432,"coverage":1,"random":0,"tracks":[],"combined":0.2196,"private_eq":0.2275},"10":{"raw":0.5286,"skill":0.2156,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7542,"skill":0.6446,"coverage":0.9837,"random":0.3085,"tracks":[]},"12":{"raw":0.4569,"skill":0.1865,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2143,"private_eq":0.4808},"20":{"raw":0.5341,"skill":0.0681,"coverage":1,"random":0.5,"tracks":[],"combined":0.0322,"private_eq":0.5211},"21":{"raw":0.5213,"skill":0.0426,"coverage":1,"random":0.5,"tracks":[],"combined":0.0352,"private_eq":0.5226},"22":{"raw":0.0136,"skill":0.0059,"coverage":0.998,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.4661,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3411,"skill":0.1215,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2398,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2529,"combined":0},"26":{"raw":0.6418,"skill":0.5223,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.442,"skill":0.2558,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5627,"skill":0.1254,"coverage":1,"random":0.5,"tracks":[],"combined":0.2444,"private_eq":0.626},"29":{"raw":0.4453,"skill":0.2604,"coverage":1,"random":0.25,"tracks":[],"combined":0.2108,"private_eq":0.4141},"30":{"raw":0.1789,"skill":0.0059,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2472,"random":0.25},{"track":"10-choice","raw":0.1107,"random":0.1}],"combined":0.0726,"private_eq":0.2428},"31":{"raw":0.0762,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0,"private_eq":0.0824},"32":{"raw":0.5505,"skill":0.2854,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0018,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.2719,"skill":0.1763,"coverage":1,"random":0.116,"tracks":[],"combined":0.1517,"private_eq":0.2578},"37":{"raw":0.3257,"skill":0.1543,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1148,"private_eq":0.3015},"38":{"raw":0.2266,"skill":0.2019,"coverage":1,"random":0.031,"tracks":[],"combined":0.0781,"private_eq":0.1159},"39":{"raw":0.8992,"skill":0.8517,"coverage":1,"random":0.3201,"tracks":[],"combined":0.44,"private_eq":0.6229},"40":{"raw":0.1408,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0178,"headline":false},{"track":"A · English","score":0.1408,"headline":true},{"track":"C · Arabic pairs","score":0.43,"headline":false},{"track":"C · English pairs","score":0.31,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7402,"skill":0.6104,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5895,"skill":0.2014,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.414,"skill":0.0703,"coverage":1,"random":0.3697,"tracks":[],"combined":0,"private_eq":0.3697},"44":{"raw":0.675,"skill":0.35,"coverage":1,"random":0.5,"tracks":[],"combined":0.3211,"private_eq":0.6639},"45":{"raw":0.1297,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.494,"skill":0.2657,"coverage":1,"random":0.311,"tracks":[],"combined":0.1162,"private_eq":0.3973},"56":{"raw":0.491,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0.0146,"private_eq":0.5124},"57":{"raw":0.1367,"skill":0.029,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3492,"skill":0.0567,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3101,"combined":0},"59":{"raw":0.6569,"skill":0.2886,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.713,"skill":0.426,"coverage":1,"random":0.5,"tracks":[],"combined":0.3316,"private_eq":0.6691},"62":{"raw":0.6725,"skill":0.5633,"coverage":1,"random":0.25,"tracks":[],"combined":0.2823,"private_eq":0.4671},"64":{"raw":0.2973,"skill":0.1216,"coverage":1,"random":0.2,"tracks":[],"combined":0.0506,"private_eq":0.2483}},"coverage":0.9993,"pending":0.0007,"notes":["New in the 2026-10-20 PR batch (PR #233).","Submitted public 0.3 run adopted (kit rescore 32.09, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.90% on 825 rows (bar 98.5%).","Questions over its 7,999-token limit count as unsupported.","License CC BY-NC 4.0."],"scores":{"balanced_raw":49.89,"balanced_skill":15.62,"breadth_skill":26.96,"public_skill":32.09},"frozen_scores":{"balanced_raw":49.89,"balanced_skill":32.09,"breadth_skill":26.96},"categories":[{"id":"knowledge","raw":0.2998,"skill":0.0804,"combined":0.0567,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.5415,"skill":0.3548,"combined":0.1174,"coverage":0.9983,"pending":0,"n":9},{"id":"retrieval","raw":0.6136,"skill":0.4521,"combined":0.3097,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6552,"skill":0.5989,"combined":0.3834,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.3877,"skill":0.084,"combined":0.0447,"coverage":0.9997,"pending":0,"n":6}],"v03":{"engine":"pick-150m-v5","pub":32.09,"same_skills":19.76,"new_domains":6.48,"v":15.62,"lab":false,"jev":false,"rank_v":213,"rank_pub":184,"name":"PICK-150M v5","eq":{"S":18.94,"O":10.65},"rank_tie":50,"tied":true,"parts":{"public":0,"same_skills":11.36,"new_domains":4.26}},"new_in_edition":true},{"engine":"vllm-sr-d3-turbo-v11","name":"vLLM-SR d3-turbo v1.1 (27B)","long_name":"vLLM-SR d3-turbo v1.1 (27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3-turbo","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pr":242,"replaces":"vllm-sr-d3-turbo","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7983,"conf":0.8506,"ece":0.0523,"brier":0.279,"over95":0.0141,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0012,"conf":0.0869,"acc":0.0667},{"lo":0.1,"w":0.0112,"conf":0.1546,"acc":0.1056},{"lo":0.2,"w":0.0093,"conf":0.2488,"acc":0.2292},{"lo":0.3,"w":0.0164,"conf":0.3594,"acc":0.2804},{"lo":0.4,"w":0.0297,"conf":0.4563,"acc":0.4481},{"lo":0.5,"w":0.0713,"conf":0.5534,"acc":0.4851},{"lo":0.6,"w":0.0753,"conf":0.6495,"acc":0.5267},{"lo":0.7,"w":0.0819,"conf":0.7519,"acc":0.6233},{"lo":0.8,"w":0.1054,"conf":0.8512,"acc":0.7211},{"lo":0.9,"w":0.5984,"conf":0.9816,"acc":0.9621}],"areas":{"knowledge":{"ece":0.0791,"acc":0.7342,"conf":0.8124},"language":{"ece":0.0118,"acc":0.9107,"conf":0.8989},"retrieval":{"ece":0.1623,"acc":0.6816,"conf":0.8433},"tools":{"ece":0.0409,"acc":0.9483,"conf":0.985},"arts":{"ece":0.0377,"acc":0.7275,"conf":0.7652}}},"gaps":null,"latency":{"median":190.5,"p95":586.6,"mean":341.9},"results":{"1":{"score":0.977,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":310.6},"2":{"score":0.6883,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3246.6},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1211.8},"4":{"score":0.9381,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":254.3},"5":{"score":0.9685,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":377.7},"6":{"score":0.7955,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":561.9},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":809.8},"10":{"score":0.1611,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":287.9},"11":{"score":0.8557,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":670.9},"12":{"score":0.8022,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":103.8},"20":{"score":0.951,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":104.3},"21":{"score":0.6686,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":84.2},"22":{"score":0.7385,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1259.6},"23":{"score":0.7088,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":169.3},"24":{"score":0.8809,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":88.3},"25":{"score":0.5918,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":104.8},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":80.9},"27":{"score":0.9701,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":82.2},"28":{"score":0.9534,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":79.8},"29":{"score":0.9816,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":109},"30":{"score":0.845,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2434,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":232},"32":{"score":0.7061,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":250.7},"33":{"score":0.4382,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":330.1},"36":{"score":0.4831,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4370.1},"37":{"score":0.6177,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":167.1},"38":{"score":0.6464,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1129.2},"39":{"score":0.9526,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":133},"40":{"score":0.6503,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":94.6},"41":{"score":0.8036,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":108.6},"42":{"score":0.8662,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":156.5},"43":{"score":0.8596,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":107.8},"44":{"score":0.9282,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":103.2},"45":{"score":0.1617,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":119},"50":{"score":0.6163,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":187.9},"56":{"score":0.962,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":236.8},"57":{"score":0.7295,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":112.6},"58":{"score":0.8923,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":102.8},"59":{"score":0.8234,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":208.4},"61":{"score":0.8872,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":160.1},"62":{"score":0.8992,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":209.1},"64":{"score":0.8314,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":137.5}},"benchmarks":{"1":{"raw":0.977,"skill":0.9689,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9669,"private_eq":0.975},"2":{"raw":0.6883,"skill":0.64,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7162,"private_eq":0.7562},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9433,"private_eq":0.944},"4":{"raw":0.9381,"skill":0.9373,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9112,"private_eq":0.9123},"5":{"raw":0.9685,"skill":0.9683,"coverage":1,"random":0.006,"tracks":[],"combined":0.8725,"private_eq":0.8737},"6":{"raw":0.7955,"skill":0.5205,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9302,"private_eq":0.93},"10":{"raw":0.1611,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8557,"skill":0.7912,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.8022,"skill":0.7037,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5357,"private_eq":0.6929},"20":{"raw":0.9513,"skill":0.9026,"coverage":1,"random":0.5,"tracks":[],"combined":0.9577,"private_eq":0.9786},"21":{"raw":0.6686,"skill":0.3371,"coverage":1,"random":0.5,"tracks":[],"combined":0.3069,"private_eq":0.6569},"22":{"raw":0.7336,"skill":0.7316,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6003,"private_eq":0.6069},"23":{"raw":0.7243,"skill":0.4486,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8809,"skill":0.8412,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5918,"skill":0.4558,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5179,"combined":0.3508},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9534,"skill":0.9068,"coverage":1,"random":0.5,"tracks":[],"combined":0.5824,"private_eq":0.7931},"29":{"raw":0.9816,"skill":0.9755,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.845,"skill":0.8143,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8787,"random":0.25},{"track":"10-choice","raw":0.8112,"random":0.1}],"combined":0.7904,"private_eq":0.8282},"31":{"raw":0.2434,"skill":0.1759,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0853,"private_eq":0.1689},"32":{"raw":0.7061,"skill":0.5328,"coverage":1,"random":0.371,"tracks":[],"combined":0.3743,"private_eq":0.6103},"33":{"raw":0.4382,"skill":0.4307,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4565,"private_eq":0.4687},"36":{"raw":0.4831,"skill":0.4153,"coverage":1,"random":0.116,"tracks":[],"combined":0.4129,"private_eq":0.486},"37":{"raw":0.6177,"skill":0.5205,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5912,"private_eq":0.677},"38":{"raw":0.6464,"skill":0.6351,"coverage":1,"random":0.031,"tracks":[],"combined":0.7943,"private_eq":0.802},"39":{"raw":0.9526,"skill":0.9303,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4638,"private_eq":0.6389},"40":{"raw":0.6503,"skill":0.5501,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5795,"headline":false},{"track":"A · English","score":0.6503,"headline":true},{"track":"C · Arabic pairs","score":0.93,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.5803,"private_eq":0.6767},"41":{"raw":0.8036,"skill":0.7055,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2306,"private_eq":0.4922},"42":{"raw":0.8662,"skill":0.7397,"coverage":1,"random":0.486,"tracks":[],"combined":0.9367,"private_eq":0.9673},"43":{"raw":0.8596,"skill":0.7773,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6247,"private_eq":0.7655},"44":{"raw":0.9282,"skill":0.8564,"coverage":1,"random":0.5,"tracks":[],"combined":0.8297,"private_eq":0.9153},"45":{"raw":0.1617,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6163,"skill":0.4432,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.962,"skill":0.924,"coverage":1,"random":0.5,"tracks":[],"combined":0.6288,"private_eq":0.816},"57":{"raw":0.7295,"skill":0.6958,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6279,"combined":0.5777},"58":{"raw":0.8923,"skill":0.8439,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5303,"combined":0.3123},"59":{"raw":0.8234,"skill":0.6338,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2837,"private_eq":0.658},"61":{"raw":0.8872,"skill":0.7744,"coverage":1,"random":0.5,"tracks":[],"combined":0.6721,"private_eq":0.8374},"62":{"raw":0.8992,"skill":0.8656,"coverage":1,"random":0.25,"tracks":[],"combined":0.5903,"private_eq":0.6955},"64":{"raw":0.8314,"skill":0.7892,"coverage":1,"random":0.2,"tracks":[],"combined":0.7877,"private_eq":0.8313}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #242).","Replaces the earlier board entry vllm-sr-d3-turbo (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 71.62). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.90% on 827 rows (bar 98.5%).","A separate model from vLLM-SR d3 (27B), which stays on the board.","Served with the model repo's own engine on an RTX PRO 6000, where the submitted run used an AMD MI325X."],"scores":{"balanced_raw":78.47,"balanced_skill":67.31,"breadth_skill":71.01,"public_skill":71.62},"frozen_scores":{"balanced_raw":78.47,"balanced_skill":71.62,"breadth_skill":71.01},"categories":[{"id":"knowledge","raw":0.6679,"skill":0.5892,"combined":0.4953,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8233,"skill":0.7448,"combined":0.6111,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.8083,"skill":0.7582,"combined":0.6861,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8861,"skill":0.8685,"combined":0.8387,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7542,"skill":0.6087,"combined":0.5755,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-turbo-v11","pub":71.62,"same_skills":64.29,"new_domains":56.41,"v":67.31,"lab":false,"jev":false,"rank_v":18,"rank_pub":8,"name":"vLLM-SR d3-turbo v1.1 (27B)","eq":{"S":63.98,"O":72.31},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":38.39,"new_domains":28.93}},"new_in_edition":true},{"engine":"cerno-9b-v11","name":"Cerno 9B v1.1","long_name":"Cerno 9B v1.1","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":8953803264,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/apertixai/cerno-9b","code_url":null,"base_model":"Qwen/Qwen3.5-9B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B-Base","weights_repo":null,"served_checkpoint":null,"pr":234,"replaces":"cerno-9b","new":true},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.7296,"conf":0.7769,"ece":0.048,"brier":0.3552,"over95":0.0098,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0293,"conf":0.0747,"acc":0.0859},{"lo":0.1,"w":0.0279,"conf":0.1379,"acc":0.1309},{"lo":0.2,"w":0.0116,"conf":0.2483,"acc":0.2122},{"lo":0.3,"w":0.0235,"conf":0.3546,"acc":0.3408},{"lo":0.4,"w":0.0446,"conf":0.4485,"acc":0.327},{"lo":0.5,"w":0.0785,"conf":0.5483,"acc":0.4505},{"lo":0.6,"w":0.0868,"conf":0.65,"acc":0.6051},{"lo":0.7,"w":0.1004,"conf":0.7511,"acc":0.6432},{"lo":0.8,"w":0.1341,"conf":0.8542,"acc":0.7754},{"lo":0.9,"w":0.4634,"conf":0.9718,"acc":0.9537}],"areas":{"knowledge":{"ece":0.106,"acc":0.6161,"conf":0.7221},"language":{"ece":0.018,"acc":0.862,"conf":0.8785},"retrieval":{"ece":0.0555,"acc":0.7816,"conf":0.833},"tools":{"ece":0.033,"acc":0.8901,"conf":0.8835},"arts":{"ece":0.0293,"acc":0.5801,"conf":0.5951}}},"gaps":null,"latency":{"median":27.4,"p95":167.1,"mean":58.2},"results":{"1":{"score":0.9498,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":94.2},"2":{"score":0.664,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":277.7},"3":{"score":0.748,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":164.6},"4":{"score":0.9248,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":59.2},"5":{"score":0.958,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":59.8},"6":{"score":0.7994,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":126.7},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":219.3},"10":{"score":0.6061,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":64.6},"11":{"score":0.815,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":367.2},"12":{"score":0.7129,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":27.9},"20":{"score":0.898,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.1},"21":{"score":0.6442,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":27.7},"22":{"score":0.1401,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":134.7},"23":{"score":0.6218,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.4},"24":{"score":0.7938,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":28},"25":{"score":0.4337,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":28.1},"26":{"score":0.9848,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":27.7},"27":{"score":0.9556,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.8},"28":{"score":0.8785,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":27.6},"29":{"score":0.9551,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":28.4},"30":{"score":0.6531,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1278,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60},"32":{"score":0.5851,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":62.1},"33":{"score":0.2606,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":132.5},"36":{"score":0.4625,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":400.3},"37":{"score":0.5866,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.6},"38":{"score":0.2551,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":291.5},"39":{"score":0.868,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":55.9},"40":{"score":0.5903,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":28.6},"41":{"score":0.7966,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":28.6},"42":{"score":0.8101,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32.7},"43":{"score":0.4912,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":28.4},"44":{"score":0.7892,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.3},"45":{"score":0.0958,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":29.1},"50":{"score":0.6241,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":63.9},"56":{"score":0.6675,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":106.4},"57":{"score":0.5952,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":28.7},"58":{"score":0.8033,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":28},"59":{"score":0.8388,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":61.8},"61":{"score":0.876,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":29.9},"62":{"score":0.8883,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":58.9},"64":{"score":0.7424,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":27.9}},"benchmarks":{"1":{"raw":0.9498,"skill":0.9323,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8921,"private_eq":0.9202},"2":{"raw":0.663,"skill":0.6108,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.6957,"private_eq":0.7386},"3":{"raw":0.748,"skill":0.7431,"coverage":1,"random":0.0189,"tracks":[],"combined":0.597,"private_eq":0.6081},"4":{"raw":0.9248,"skill":0.9238,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8184,"private_eq":0.8218},"5":{"raw":0.958,"skill":0.9577,"coverage":1,"random":0.006,"tracks":[],"combined":0.7278,"private_eq":0.7315},"6":{"raw":0.7994,"skill":0.5297,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.6445,"private_eq":0.6475},"10":{"raw":0.6061,"skill":0.3446,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.815,"skill":0.7325,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7129,"skill":0.57,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4268,"private_eq":0.621},"20":{"raw":0.8993,"skill":0.7986,"coverage":1,"random":0.5,"tracks":[],"combined":0.7842,"private_eq":0.8928},"21":{"raw":0.6442,"skill":0.2884,"coverage":1,"random":0.5,"tracks":[],"combined":0.4277,"private_eq":0.7166},"22":{"raw":0.1318,"skill":0.125,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.1519,"private_eq":0.1671},"23":{"raw":0.6174,"skill":0.2348,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7938,"skill":0.7251,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4337,"skill":0.2449,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3828,"combined":0.1686},"26":{"raw":0.9848,"skill":0.9797,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9556,"skill":0.9408,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8785,"skill":0.757,"coverage":1,"random":0.5,"tracks":[],"combined":0.5521,"private_eq":0.7781},"29":{"raw":0.9551,"skill":0.9401,"coverage":1,"random":0.25,"tracks":[],"combined":0.9879,"private_eq":0.9903},"30":{"raw":0.6531,"skill":0.5845,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7293,"random":0.25},{"track":"10-choice","raw":0.577,"random":0.1}],"combined":0.5947,"private_eq":0.6686},"31":{"raw":0.1278,"skill":0.05,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0409,"private_eq":0.1286},"32":{"raw":0.5851,"skill":0.3404,"coverage":1,"random":0.371,"tracks":[],"combined":0.2196,"private_eq":0.5141},"33":{"raw":0.2606,"skill":0.2508,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0608,"private_eq":0.0827},"36":{"raw":0.4625,"skill":0.392,"coverage":1,"random":0.116,"tracks":[],"combined":0.3843,"private_eq":0.461},"37":{"raw":0.5866,"skill":0.4815,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5078,"private_eq":0.6112},"38":{"raw":0.2551,"skill":0.2313,"coverage":1,"random":0.031,"tracks":[],"combined":0.2979,"private_eq":0.3264},"39":{"raw":0.868,"skill":0.8059,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5459,"private_eq":0.6941},"40":{"raw":0.5903,"skill":0.473,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1441,"headline":false},{"track":"A · English","score":0.5903,"headline":true},{"track":"C · Arabic pairs","score":0.605,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}],"combined":0.3963,"private_eq":0.5353},"41":{"raw":0.7966,"skill":0.695,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4136,"private_eq":0.6128},"42":{"raw":0.8101,"skill":0.6305,"coverage":1,"random":0.486,"tracks":[],"combined":0.892,"private_eq":0.9446},"43":{"raw":0.4912,"skill":0.1928,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4534,"private_eq":0.6588},"44":{"raw":0.7892,"skill":0.5784,"coverage":1,"random":0.5,"tracks":[],"combined":0.487,"private_eq":0.7459},"45":{"raw":0.0958,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6241,"skill":0.4545,"coverage":1,"random":0.311,"tracks":[],"combined":0.2176,"private_eq":0.4664},"56":{"raw":0.6675,"skill":0.335,"coverage":1,"random":0.5,"tracks":[],"combined":0.3261,"private_eq":0.6664},"57":{"raw":0.5952,"skill":0.5447,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5566,"combined":0.4966},"58":{"raw":0.8033,"skill":0.7149,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4308,"combined":0.1665},"59":{"raw":0.8388,"skill":0.6658,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2896,"private_eq":0.6608},"61":{"raw":0.876,"skill":0.752,"coverage":1,"random":0.5,"tracks":[],"combined":0.6492,"private_eq":0.8261},"62":{"raw":0.8883,"skill":0.8511,"coverage":1,"random":0.25,"tracks":[],"combined":0.6861,"private_eq":0.7665},"64":{"raw":0.7424,"skill":0.678,"coverage":1,"random":0.2,"tracks":[],"combined":0.7003,"private_eq":0.7622}},"coverage":0.9999,"pending":0.0001,"notes":["New in the 2026-10-20 PR batch (PR #234).","Replaces the earlier board entry cerno-9b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 58.59, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8); decision agreement with the submitted rows 98.44% on 824 rows (bar 96%).","Served in FP8 as published, so the quantized agreement bar applies.","The pinned GitHub inference commit is no longer available; the lab used the inference code bundled in the model repo at the pinned revision."],"scores":{"balanced_raw":68.68,"balanced_skill":47.34,"breadth_skill":56.97,"public_skill":58.59},"frozen_scores":{"balanced_raw":68.68,"balanced_skill":58.59,"breadth_skill":56.97},"categories":[{"id":"knowledge","raw":0.5229,"skill":0.3926,"combined":0.3256,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7421,"skill":0.6432,"combined":0.5397,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7492,"skill":0.651,"combined":0.5757,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.814,"skill":0.7924,"combined":0.7061,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.6099,"skill":0.4299,"combined":0.4564,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"cerno-9b-v11","pub":58.59,"same_skills":52.36,"new_domains":30.63,"v":47.34,"lab":false,"jev":false,"rank_v":100,"rank_pub":71,"name":"Cerno 9B v1.1","eq":{"S":51.91,"O":40.48},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":31.15,"new_domains":16.19}},"new_in_edition":true},{"engine":"cerno-4b-v11","name":"Cerno 4B v1.1","long_name":"Cerno 4B v1.1","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4205751296,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/apertixai/cerno-4b","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null,"pr":234,"replaces":"cerno-4b","new":true},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.7154,"conf":0.7656,"ece":0.0502,"brier":0.3666,"over95":0.0113,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0332,"conf":0.0727,"acc":0.0606},{"lo":0.1,"w":0.0251,"conf":0.1383,"acc":0.0671},{"lo":0.2,"w":0.0116,"conf":0.2612,"acc":0.203},{"lo":0.3,"w":0.0323,"conf":0.3589,"acc":0.2088},{"lo":0.4,"w":0.0449,"conf":0.4564,"acc":0.3502},{"lo":0.5,"w":0.0868,"conf":0.5496,"acc":0.5165},{"lo":0.6,"w":0.0913,"conf":0.6491,"acc":0.5602},{"lo":0.7,"w":0.1008,"conf":0.7525,"acc":0.6737},{"lo":0.8,"w":0.129,"conf":0.8529,"acc":0.7799},{"lo":0.9,"w":0.4451,"conf":0.9702,"acc":0.949}],"areas":{"knowledge":{"ece":0.1153,"acc":0.5826,"conf":0.6974},"language":{"ece":0.0159,"acc":0.8574,"conf":0.869},"retrieval":{"ece":0.0596,"acc":0.7555,"conf":0.8058},"tools":{"ece":0.038,"acc":0.9254,"conf":0.9192},"arts":{"ece":0.0397,"acc":0.5674,"conf":0.5918}}},"gaps":null,"latency":{"median":26.7,"p95":145.6,"mean":51.9},"results":{"1":{"score":0.9203,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":95.5},"2":{"score":0.6659,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":220.9},"3":{"score":0.8248,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":114.9},"4":{"score":0.9219,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":59.8},"5":{"score":0.955,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":60.8},"6":{"score":0.7988,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":127.9},"9":{"score":0.9432,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":213.9},"10":{"score":0.4891,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":65.2},"11":{"score":0.8255,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":336.5},"12":{"score":0.6938,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":28.1},"20":{"score":0.8556,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.2},"21":{"score":0.6423,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":27.7},"22":{"score":0.05,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":91.5},"23":{"score":0.6306,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.4},"24":{"score":0.7379,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":28},"25":{"score":0.3878,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":28.1},"26":{"score":0.9844,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":27.8},"27":{"score":0.9403,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.8},"28":{"score":0.8564,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":27.6},"29":{"score":0.9386,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":28.4},"30":{"score":0.5868,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0976,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":60.4},"32":{"score":0.5439,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":63},"33":{"score":0.1533,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":131},"36":{"score":0.4707,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":312.9},"37":{"score":0.5733,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.5},"38":{"score":0.2621,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":243.9},"39":{"score":0.8627,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":56.4},"40":{"score":0.5786,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":28.3},"41":{"score":0.7916,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":28.4},"42":{"score":0.7832,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32.3},"43":{"score":0.4456,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":28.7},"44":{"score":0.791,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.5},"45":{"score":0.0978,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":29},"50":{"score":0.6229,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":63.2},"56":{"score":0.6945,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":106.9},"57":{"score":0.5286,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":28.7},"58":{"score":0.801,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":28.8},"59":{"score":0.8388,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":62.4},"61":{"score":0.881,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":30},"62":{"score":0.8965,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":61.1},"64":{"score":0.7481,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":28.1}},"benchmarks":{"1":{"raw":0.9203,"skill":0.8924,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7968,"private_eq":0.8504},"2":{"raw":0.665,"skill":0.6131,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.6938,"private_eq":0.737},"3":{"raw":0.8248,"skill":0.8214,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6967,"private_eq":0.7048},"4":{"raw":0.9219,"skill":0.9209,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8543,"private_eq":0.8568},"5":{"raw":0.955,"skill":0.9547,"coverage":1,"random":0.006,"tracks":[],"combined":0.7034,"private_eq":0.7075},"6":{"raw":0.7988,"skill":0.5283,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9432,"skill":0.9432,"coverage":1,"random":0,"tracks":[],"combined":0.5989,"private_eq":0.6025},"10":{"raw":0.4891,"skill":0.1499,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8255,"skill":0.7476,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6938,"skill":0.5413,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4012,"private_eq":0.6041},"20":{"raw":0.8574,"skill":0.7148,"coverage":1,"random":0.5,"tracks":[],"combined":0.7769,"private_eq":0.8892},"21":{"raw":0.6423,"skill":0.2846,"coverage":1,"random":0.5,"tracks":[],"combined":0.3573,"private_eq":0.6818},"22":{"raw":0.0435,"skill":0.036,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.075,"private_eq":0.0917},"23":{"raw":0.6321,"skill":0.2642,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7379,"skill":0.6505,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3547,"combined":0.1307},"26":{"raw":0.9844,"skill":0.9792,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9403,"skill":0.9204,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8564,"skill":0.7128,"coverage":1,"random":0.5,"tracks":[],"combined":0.5116,"private_eq":0.7581},"29":{"raw":0.9386,"skill":0.9181,"coverage":1,"random":0.25,"tracks":[],"combined":0.8791,"private_eq":0.9096},"30":{"raw":0.5868,"skill":0.5048,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6755,"random":0.25},{"track":"10-choice","raw":0.4981,"random":0.1}],"combined":0.6054,"private_eq":0.6773},"31":{"raw":0.0976,"skill":0.0171,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0377,"private_eq":0.1257},"32":{"raw":0.5439,"skill":0.2749,"coverage":1,"random":0.371,"tracks":[],"combined":0.1423,"private_eq":0.466},"33":{"raw":0.1533,"skill":0.1421,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0131},"36":{"raw":0.4707,"skill":0.4012,"coverage":1,"random":0.116,"tracks":[],"combined":0.3486,"private_eq":0.4298},"37":{"raw":0.5733,"skill":0.4648,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4755,"private_eq":0.5858},"38":{"raw":0.2621,"skill":0.2385,"coverage":1,"random":0.031,"tracks":[],"combined":0.2318,"private_eq":0.2631},"39":{"raw":0.8627,"skill":0.7981,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4855,"private_eq":0.6535},"40":{"raw":0.5786,"skill":0.4578,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1233,"headline":false},{"track":"A · English","score":0.5786,"headline":true},{"track":"C · Arabic pairs","score":0.555,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"combined":0.3181,"private_eq":0.4752},"41":{"raw":0.7916,"skill":0.6873,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2863,"private_eq":0.5289},"42":{"raw":0.7832,"skill":0.5782,"coverage":1,"random":0.486,"tracks":[],"combined":0.7824,"private_eq":0.8889},"43":{"raw":0.4456,"skill":0.1204,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4241,"private_eq":0.6405},"44":{"raw":0.791,"skill":0.582,"coverage":1,"random":0.5,"tracks":[],"combined":0.5298,"private_eq":0.7671},"45":{"raw":0.0978,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6229,"skill":0.4527,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.6945,"skill":0.389,"coverage":1,"random":0.5,"tracks":[],"combined":0.0666,"private_eq":0.5381},"57":{"raw":0.5286,"skill":0.4698,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.501,"combined":0.4333},"58":{"raw":0.801,"skill":0.7116,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4102,"combined":0.1363},"59":{"raw":0.8388,"skill":0.6658,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2177,"private_eq":0.6265},"61":{"raw":0.881,"skill":0.762,"coverage":1,"random":0.5,"tracks":[],"combined":0.5699,"private_eq":0.7869},"62":{"raw":0.8965,"skill":0.862,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.7481,"skill":0.6851,"coverage":1,"random":0.2,"tracks":[],"combined":0.6131,"private_eq":0.6932}},"coverage":0.9999,"pending":0.0001,"notes":["New in the 2026-10-20 PR batch (PR #234).","Replaces the earlier board entry cerno-4b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 57.60, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8); decision agreement with the submitted rows 98.44% on 824 rows (bar 96%).","Served in FP8 as published, so the quantized agreement bar applies.","The pinned GitHub inference commit is no longer available; the lab used the inference code bundled in the model repo at the pinned revision."],"scores":{"balanced_raw":68.07,"balanced_skill":41.15,"breadth_skill":55.24,"public_skill":57.6},"frozen_scores":{"balanced_raw":68.07,"balanced_skill":57.6,"breadth_skill":55.24},"categories":[{"id":"knowledge","raw":0.4853,"skill":0.3446,"combined":0.2947,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7342,"skill":0.6303,"combined":0.4593,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7524,"skill":0.6588,"combined":0.515,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8516,"skill":0.8287,"combined":0.6819,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.591,"skill":0.4062,"combined":0.4051,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"cerno-4b-v11","pub":57.6,"same_skills":47.41,"new_domains":24.2,"v":41.15,"lab":false,"jev":false,"rank_v":127,"rank_pub":85,"name":"Cerno 4B v1.1","eq":{"S":46.9,"O":32.53},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":28.14,"new_domains":13.01}},"new_in_edition":true},{"engine":"shisa-de-2.1-repeat","name":"Shisa DE-2.1 (repeat)","long_name":"Shisa DE-2.1 (repeat)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":25805933872,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/shisa-ai/shisa-de-2.1","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"shisa-ai","name":"Shisa.AI","logo":"data/orgs/shisa-ai.png"},"pr":236,"replaces":"shisa-de-2-repeat","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7221,"conf":0.8868,"ece":0.1646,"brier":0.4262,"over95":0.0709,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0005,"conf":0.0804,"acc":0},{"lo":0.1,"w":0.0057,"conf":0.1553,"acc":0.1362},{"lo":0.2,"w":0.0106,"conf":0.2513,"acc":0.1428},{"lo":0.3,"w":0.0139,"conf":0.3579,"acc":0.2361},{"lo":0.4,"w":0.0246,"conf":0.4517,"acc":0.2075},{"lo":0.5,"w":0.0518,"conf":0.552,"acc":0.3441},{"lo":0.6,"w":0.043,"conf":0.6481,"acc":0.3952},{"lo":0.7,"w":0.0602,"conf":0.7495,"acc":0.4944},{"lo":0.8,"w":0.091,"conf":0.8552,"acc":0.5492},{"lo":0.9,"w":0.6987,"conf":0.9842,"acc":0.8543}],"areas":{"knowledge":{"ece":0.1909,"acc":0.6633,"conf":0.8542},"language":{"ece":0.0742,"acc":0.8431,"conf":0.9166},"retrieval":{"ece":0.3244,"acc":0.6141,"conf":0.9384},"tools":{"ece":0.055,"acc":0.9381,"conf":0.992},"arts":{"ece":0.2157,"acc":0.5789,"conf":0.7935}}},"gaps":null,"latency":{"median":54.5,"p95":512.6,"mean":159.3},"results":{"1":{"score":0.9604,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6499,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8524,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8334,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.8869,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7978,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.5455,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0867,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.737,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7696,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9008,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6317,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.269,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6372,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8484,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4439,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9676,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8311,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.8897,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.7942,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2152,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6662,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3248,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4578,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5272,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.2931,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9143,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6581,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7252,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.7822,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8474,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.7708,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.4499,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.8635,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6802,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.779,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.7704,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8087,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8666,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7121,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9604,"skill":0.9466,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8306,"private_eq":0.8752},"2":{"raw":0.6499,"skill":0.5957,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6694,"private_eq":0.7161},"3":{"raw":0.8524,"skill":0.8496,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8804,"private_eq":0.883},"4":{"raw":0.8334,"skill":0.8313,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9217,"private_eq":0.9226},"5":{"raw":0.8869,"skill":0.8862,"coverage":1,"random":0.006,"tracks":[],"combined":0.7376,"private_eq":0.7411},"6":{"raw":0.7978,"skill":0.5259,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5455,"skill":0.5455,"coverage":1,"random":0,"tracks":[],"combined":0.5635,"private_eq":0.5675},"10":{"raw":0.0867,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.737,"skill":0.6197,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7696,"skill":0.6549,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5516,"private_eq":0.7034},"20":{"raw":0.9035,"skill":0.8069,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6317,"skill":0.2633,"coverage":1,"random":0.5,"tracks":[],"combined":0.2466,"private_eq":0.6271},"22":{"raw":0.263,"skill":0.2572,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2561,"private_eq":0.2693},"23":{"raw":0.6412,"skill":0.2824,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8484,"skill":0.7979,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4439,"skill":0.2585,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4731,"combined":0.2904},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9676,"skill":0.9568,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8311,"skill":0.6622,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"29":{"raw":0.8897,"skill":0.8529,"coverage":1,"random":0.25,"tracks":[],"combined":0.9038,"private_eq":0.9279},"30":{"raw":0.7942,"skill":0.7534,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8393,"random":0.25},{"track":"10-choice","raw":0.7491,"random":0.1}],"combined":0.7807,"private_eq":0.8203},"31":{"raw":0.2152,"skill":0.1452,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.6662,"skill":0.4693,"coverage":1,"random":0.371,"tracks":[],"combined":0.3666,"private_eq":0.6055},"33":{"raw":0.3248,"skill":0.3158,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4411,"private_eq":0.4537},"36":{"raw":0.4578,"skill":0.3866,"coverage":1,"random":0.116,"tracks":[],"combined":0.3686,"private_eq":0.4473},"37":{"raw":0.5272,"skill":0.4069,"coverage":1,"random":0.2027,"tracks":[],"combined":0.396,"private_eq":0.5231},"38":{"raw":0.2931,"skill":0.2705,"coverage":1,"random":0.031,"tracks":[],"combined":0.4538,"private_eq":0.4758},"39":{"raw":0.9143,"skill":0.874,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5565,"private_eq":0.7012},"40":{"raw":0.6581,"skill":0.5601,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4971,"headline":false},{"track":"A · English","score":0.6581,"headline":true},{"track":"C · Arabic pairs","score":0.795,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.6018,"private_eq":0.6932},"41":{"raw":0.7252,"skill":0.5878,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5742,"private_eq":0.7187},"42":{"raw":0.7822,"skill":0.5763,"coverage":1,"random":0.486,"tracks":[],"combined":0.3807,"private_eq":0.6848},"43":{"raw":0.8474,"skill":0.7578,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6457,"private_eq":0.7786},"44":{"raw":0.7708,"skill":0.5416,"coverage":1,"random":0.5,"tracks":[],"combined":0.5511,"private_eq":0.7776},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4499,"skill":0.2016,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.8635,"skill":0.727,"coverage":1,"random":0.5,"tracks":[],"combined":0.6982,"private_eq":0.8503},"57":{"raw":0.6802,"skill":0.6403,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4502,"combined":0.3755},"58":{"raw":0.779,"skill":0.6797,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5167,"combined":0.2924},"59":{"raw":0.7704,"skill":0.5239,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2777,"private_eq":0.6551},"61":{"raw":0.8087,"skill":0.6174,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.8666,"skill":0.8221,"coverage":1,"random":0.25,"tracks":[],"combined":0.5356,"private_eq":0.6549},"64":{"raw":0.7121,"skill":0.6401,"coverage":1,"random":0.2,"tracks":[],"combined":0.7003,"private_eq":0.7622}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #236).","Replaces the earlier board entry shisa-de-2-repeat (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 59.23, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.76% on 831 rows (bar 98.5%).","Repeat (no thinking) run; the adaptive repeat-think entry exceeded the latency limit and is not listed.","Training mix includes BANKING77-train-derived examples (author disclosure).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":69.01,"balanced_skill":58.84,"breadth_skill":58.26,"public_skill":59.23},"frozen_scores":{"balanced_raw":69.01,"balanced_skill":59.23,"breadth_skill":58.26},"categories":[{"id":"knowledge","raw":0.5867,"skill":0.4727,"combined":0.4429,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.731,"skill":0.6193,"combined":0.5465,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7293,"skill":0.6479,"combined":0.6294,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7847,"skill":0.7627,"combined":0.7077,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6002,"skill":0.4086,"combined":0.4583,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"shisa-de-2.1-repeat","pub":59.23,"same_skills":56.62,"new_domains":48.67,"v":58.84,"lab":false,"jev":false,"rank_v":59,"rank_pub":67,"name":"Shisa DE-2.1 (repeat)","eq":{"S":56.22,"O":62.76},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":33.73,"new_domains":25.1}},"new_in_edition":true},{"engine":"trio-spark-di","name":"Trio-Spark DI (9B)","long_name":"Trio-Spark DI (9B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":8953803264,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/MachineFi/Trio-Spark-DI","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"MachineFi","name":"MachineFi Labs","logo":"data/orgs/MachineFi.png"},"pr":245,"new":true},"completed":110201,"counts":{"ok":110187,"unsupported":14,"error":0},"lifted":false,"calibration":{"acc":0.7441,"conf":0.8709,"ece":0.1274,"brier":0.3864,"over95":0.0364,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0049,"conf":0.0844,"acc":0.0799},{"lo":0.1,"w":0.0198,"conf":0.1422,"acc":0.1421},{"lo":0.2,"w":0.0056,"conf":0.2409,"acc":0.2072},{"lo":0.3,"w":0.0061,"conf":0.358,"acc":0.4056},{"lo":0.4,"w":0.0187,"conf":0.452,"acc":0.1908},{"lo":0.5,"w":0.0324,"conf":0.5531,"acc":0.3769},{"lo":0.6,"w":0.0443,"conf":0.6525,"acc":0.4241},{"lo":0.7,"w":0.0764,"conf":0.7541,"acc":0.4957},{"lo":0.8,"w":0.1516,"conf":0.8557,"acc":0.6458},{"lo":0.9,"w":0.6402,"conf":0.9707,"acc":0.8854}],"areas":{"knowledge":{"ece":0.1787,"acc":0.6415,"conf":0.819},"language":{"ece":0.0706,"acc":0.8355,"conf":0.9058},"retrieval":{"ece":0.0918,"acc":0.8029,"conf":0.8906},"tools":{"ece":0.0565,"acc":0.9195,"conf":0.9759},"arts":{"ece":0.2005,"acc":0.6322,"conf":0.8327}}},"gaps":null,"latency":{"median":209.2,"p95":1746.3,"mean":561.4},"results":{"1":{"score":0.9162,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6277,"jev":null,"metric":"nDCG@10","requests":685,"answered":675,"unsupported":10,"errors":0,"median_ms":null},"3":{"score":0.8504,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8545,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.8716,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7964,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.6364,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.843,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7581,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.6906,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.8646,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6442,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.6143,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":null},"23":{"score":0.6172,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.7752,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4592,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9819,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9514,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8666,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9417,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.5337,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1944,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.5824,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.2036,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.3823,"jev":null,"metric":"nDCG@10","requests":220,"answered":215,"unsupported":5,"errors":0,"median_ms":null},"37":{"score":0.5552,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.3392,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.8601,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.4952,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7709,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.7724,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.6123,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8868,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1102,"jev":null,"metric":"accuracy","requests":501,"answered":499,"unsupported":2,"errors":0,"median_ms":null},"50":{"score":0.4171,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.648,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.5348,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.6946,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.7596,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8625,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8932,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.6629,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9162,"skill":0.8868,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7898,"private_eq":0.8453},"2":{"raw":0.6185,"skill":0.5594,"coverage":0.9854,"random":0.1341,"tracks":[],"combined":0.6683,"private_eq":0.7152},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8645,"private_eq":0.8676},"4":{"raw":0.8545,"skill":0.8526,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7869,"private_eq":0.791},"5":{"raw":0.8716,"skill":0.8708,"coverage":1,"random":0.006,"tracks":[],"combined":0.7059,"private_eq":0.71},"6":{"raw":0.7964,"skill":0.5226,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6364,"skill":0.6364,"coverage":1,"random":0,"tracks":[],"combined":0.4725,"private_eq":0.4775},"10":{"raw":0.843,"skill":0.7388,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7581,"skill":0.6502,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6906,"skill":0.5365,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3507,"private_eq":0.5708},"20":{"raw":0.8658,"skill":0.7316,"coverage":1,"random":0.5,"tracks":[],"combined":0.7336,"private_eq":0.8678},"21":{"raw":0.6442,"skill":0.2884,"coverage":1,"random":0.5,"tracks":[],"combined":0.3371,"private_eq":0.6718},"22":{"raw":0.5996,"skill":0.5964,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.4274,"private_eq":0.4373},"23":{"raw":0.6097,"skill":0.2194,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7752,"skill":0.7003,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4592,"skill":0.2789,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3692,"combined":0.1503},"26":{"raw":0.9819,"skill":0.9759,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9514,"skill":0.9352,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8666,"skill":0.7332,"coverage":1,"random":0.5,"tracks":[],"combined":0.4309,"private_eq":0.7182},"29":{"raw":0.9417,"skill":0.9223,"coverage":1,"random":0.25,"tracks":[],"combined":0.9336,"private_eq":0.95},"30":{"raw":0.5337,"skill":0.4375,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5997,"random":0.25},{"track":"10-choice","raw":0.4678,"random":0.1}],"combined":0.5857,"private_eq":0.6613},"31":{"raw":0.1944,"skill":0.1226,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0821,"private_eq":0.166},"32":{"raw":0.5824,"skill":0.3361,"coverage":1,"random":0.371,"tracks":[],"combined":0.2351,"private_eq":0.5237},"33":{"raw":0.2036,"skill":0.193,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2406,"private_eq":0.2581},"36":{"raw":0.3737,"skill":0.2914,"coverage":0.9773,"random":0.116,"tracks":[],"combined":0.2849,"private_eq":0.3742},"37":{"raw":0.5552,"skill":0.4421,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4873,"private_eq":0.5951},"38":{"raw":0.3392,"skill":0.3181,"coverage":1,"random":0.031,"tracks":[],"combined":0.5002,"private_eq":0.5202},"39":{"raw":0.8601,"skill":0.7942,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5615,"private_eq":0.7046},"40":{"raw":0.4952,"skill":0.3506,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3882,"headline":false},{"track":"A · English","score":0.4952,"headline":true},{"track":"C · Arabic pairs","score":0.9,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0.2798,"private_eq":0.4458},"41":{"raw":0.7709,"skill":0.6564,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.7724,"skill":0.5572,"coverage":1,"random":0.486,"tracks":[],"combined":0.6793,"private_eq":0.8365},"43":{"raw":0.6123,"skill":0.3848,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3572,"private_eq":0.5988},"44":{"raw":0.8868,"skill":0.7736,"coverage":1,"random":0.5,"tracks":[],"combined":0.7279,"private_eq":0.865},"45":{"raw":0.1098,"skill":0,"coverage":0.996,"random":0.1641,"tracks":[]},"50":{"raw":0.4171,"skill":0.154,"coverage":1,"random":0.311,"tracks":[],"combined":0.1234,"private_eq":0.4022},"56":{"raw":0.648,"skill":0.296,"coverage":1,"random":0.5,"tracks":[],"combined":0.5857,"private_eq":0.7947},"57":{"raw":0.5348,"skill":0.4768,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5107,"combined":0.4443},"58":{"raw":0.6946,"skill":0.5573,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4319,"combined":0.1681},"59":{"raw":0.7596,"skill":0.5016,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3492,"private_eq":0.6892},"61":{"raw":0.8625,"skill":0.725,"coverage":1,"random":0.5,"tracks":[],"combined":0.6152,"private_eq":0.8093},"62":{"raw":0.8932,"skill":0.8576,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.6629,"skill":0.5786,"coverage":1,"random":0.2,"tracks":[],"combined":0.6228,"private_eq":0.7009}},"coverage":0.9985,"pending":0.0015,"notes":["New in the 2026-10-20 PR batch (PR #245).","Submitted public 0.3 run adopted (kit rescore 55.52, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (GGUF Q4_K_M); decision agreement with the submitted rows 97.04% on 823 rows (bar 96%).","Served as GGUF Q4_K_M through llama.cpp, as in the author's run, so the quantized agreement bar applies.","The lab built llama.cpp natively for its GPU with CUDA 13.0; the author used CUDA 12.8.","License CC BY-NC 4.0.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":66.53,"balanced_skill":45.59,"breadth_skill":54.23,"public_skill":55.52},"frozen_scores":{"balanced_raw":66.53,"balanced_skill":55.52,"breadth_skill":54.23},"categories":[{"id":"knowledge","raw":0.5176,"skill":0.3919,"combined":0.339,"coverage":0.9996,"pending":0,"n":11},{"id":"language","raw":0.7143,"skill":0.5935,"combined":0.4644,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6948,"skill":0.588,"combined":0.579,"coverage":0.9959,"pending":0,"n":6},{"id":"tools","raw":0.7904,"skill":0.7657,"combined":0.687,"coverage":0.9973,"pending":0,"n":5},{"id":"arts","raw":0.6332,"skill":0.4281,"combined":0.4488,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"trio-spark-di","pub":55.52,"same_skills":50.47,"new_domains":29.42,"v":45.59,"lab":false,"jev":false,"rank_v":109,"rank_pub":97,"name":"Trio-Spark DI (9B)","eq":{"S":50,"O":38.98},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":30,"new_domains":15.59}},"new_in_edition":true},{"engine":"tern-35b-a3b","name":"Tern-35B-A3B","long_name":"Tern-35B-A3B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":34769786497,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/matthewagi/Tern-35B-A3B","code_url":null,"base_model":"Qwen/Qwen3.6-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","weights_repo":null,"served_checkpoint":null,"pr":227,"new":true},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.7299,"conf":0.7553,"ece":0.0336,"brier":0.361,"over95":0.0031,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0033,"conf":0.0859,"acc":0},{"lo":0.1,"w":0.0286,"conf":0.1526,"acc":0.0696},{"lo":0.2,"w":0.0265,"conf":0.2487,"acc":0.1613},{"lo":0.3,"w":0.0369,"conf":0.3511,"acc":0.2896},{"lo":0.4,"w":0.0452,"conf":0.4529,"acc":0.432},{"lo":0.5,"w":0.0917,"conf":0.5477,"acc":0.4987},{"lo":0.6,"w":0.0885,"conf":0.6492,"acc":0.6274},{"lo":0.7,"w":0.1063,"conf":0.7534,"acc":0.7143},{"lo":0.8,"w":0.202,"conf":0.8536,"acc":0.8006},{"lo":0.9,"w":0.3709,"conf":0.9446,"acc":0.9557}],"areas":{"knowledge":{"ece":0.0681,"acc":0.6581,"conf":0.7154},"language":{"ece":0.0386,"acc":0.8606,"conf":0.8305},"retrieval":{"ece":0.138,"acc":0.6501,"conf":0.7877},"tools":{"ece":0.0689,"acc":0.9315,"conf":0.8742},"arts":{"ece":0.0366,"acc":0.5813,"conf":0.6023}}},"gaps":null,"latency":{"median":62.9,"p95":139.2,"mean":82.8},"results":{"1":{"score":0.9362,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6758,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.9048,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.893,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.796,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.5455,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.1052,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8432,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7201,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9274,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6298,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.118,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6506,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8546,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5051,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9684,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8548,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9592,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.7619,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1664,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.5612,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3248,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4381,"jev":null,"metric":"nDCG@10","requests":220,"answered":218,"unsupported":2,"errors":0,"median_ms":null},"37":{"score":0.5221,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.3428,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9278,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6289,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.6545,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8221,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.7351,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.6782,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.0998,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.5967,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.7315,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6922,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7549,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8164,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.87,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8995,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.6989,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9362,"skill":0.9139,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7968,"private_eq":0.8504},"2":{"raw":0.6758,"skill":0.6256,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6959,"private_eq":0.7388},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8804,"private_eq":0.883},"4":{"raw":0.9048,"skill":0.9036,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8617,"private_eq":0.864},"5":{"raw":0.893,"skill":0.8924,"coverage":1,"random":0.006,"tracks":[],"combined":0.7191,"private_eq":0.723},"6":{"raw":0.796,"skill":0.5217,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5455,"skill":0.5455,"coverage":1,"random":0,"tracks":[],"combined":0.5205,"private_eq":0.525},"10":{"raw":0.1052,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8432,"skill":0.7732,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7201,"skill":0.5807,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4983,"private_eq":0.6682},"20":{"raw":0.9289,"skill":0.8579,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6298,"skill":0.2595,"coverage":1,"random":0.5,"tracks":[],"combined":0.1962,"private_eq":0.6022},"22":{"raw":0.1233,"skill":0.1165,"coverage":1,"random":0.0078,"tracks":[],"combined":0.125,"private_eq":0.1407},"23":{"raw":0.6551,"skill":0.3102,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8546,"skill":0.8061,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4971,"combined":0.3228},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9684,"skill":0.9579,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8548,"skill":0.7096,"coverage":1,"random":0.5,"tracks":[],"combined":0.5673,"private_eq":0.7856},"29":{"raw":0.9592,"skill":0.9456,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7619,"skill":0.7157,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8218,"random":0.25},{"track":"10-choice","raw":0.702,"random":0.1}],"combined":0.7437,"private_eq":0.7901},"31":{"raw":0.1664,"skill":0.0921,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0974,"private_eq":0.1798},"32":{"raw":0.5612,"skill":0.3024,"coverage":1,"random":0.371,"tracks":[],"combined":0.444,"private_eq":0.6536},"33":{"raw":0.3248,"skill":0.3158,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3126,"private_eq":0.3283},"36":{"raw":0.4341,"skill":0.3598,"coverage":0.9909,"random":0.116,"tracks":[],"combined":0.3758,"private_eq":0.4536},"37":{"raw":0.5221,"skill":0.4006,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5213,"private_eq":0.6219},"38":{"raw":0.3428,"skill":0.3218,"coverage":1,"random":0.031,"tracks":[],"combined":0.51,"private_eq":0.5296},"39":{"raw":0.9278,"skill":0.8938,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6324,"private_eq":0.7522},"40":{"raw":0.6289,"skill":0.5225,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2302,"headline":false},{"track":"A · English","score":0.6289,"headline":true},{"track":"C · Arabic pairs","score":0.75,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}],"combined":0.6195,"private_eq":0.7068},"41":{"raw":0.6545,"skill":0.4818,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2706,"private_eq":0.5186},"42":{"raw":0.8221,"skill":0.6539,"coverage":1,"random":0.486,"tracks":[],"combined":0.8792,"private_eq":0.9381},"43":{"raw":0.7351,"skill":0.5797,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4993,"private_eq":0.6874},"44":{"raw":0.6782,"skill":0.3564,"coverage":1,"random":0.5,"tracks":[],"combined":0.4066,"private_eq":0.7062},"45":{"raw":0.0998,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5967,"skill":0.4146,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.7315,"skill":0.463,"coverage":1,"random":0.5,"tracks":[],"combined":0.3779,"private_eq":0.692},"57":{"raw":0.6922,"skill":0.6538,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4883,"combined":0.4189},"58":{"raw":0.7549,"skill":0.6447,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4891,"combined":0.2519},"59":{"raw":0.8164,"skill":0.6193,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4343,"private_eq":0.7298},"61":{"raw":0.87,"skill":0.74,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.8995,"skill":0.866,"coverage":1,"random":0.25,"tracks":[],"combined":0.5767,"private_eq":0.6854},"64":{"raw":0.6989,"skill":0.6236,"coverage":1,"random":0.2,"tracks":[],"combined":0.6422,"private_eq":0.7162}},"coverage":0.9997,"pending":0.0003,"notes":["New in the 2026-10-20 PR batch (PR #227).","Submitted public 0.3 run adopted (kit rescore 59.01, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.39% on 827 rows (bar 98.5%).","Listed under a maintainer decision on the agreement bar (98.39% vs 98.5%): the model's answers shift with server load, and the submitted run used 128 concurrent requests where the agreement check sends one at a time.","Training/development overlap with public index requests disclosed by the author.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":68.93,"balanced_skill":56.62,"breadth_skill":57.81,"public_skill":59.01},"frozen_scores":{"balanced_raw":68.93,"balanced_skill":59.01,"breadth_skill":57.81},"categories":[{"id":"knowledge","raw":0.5596,"skill":0.4317,"combined":0.4052,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7501,"skill":0.6487,"combined":0.6125,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7276,"skill":0.6349,"combined":0.5869,"coverage":0.9983,"pending":0,"n":6},{"id":"tools","raw":0.7906,"skill":0.7696,"combined":0.7047,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6054,"skill":0.4304,"combined":0.4191,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"tern-35b-a3b","pub":59.01,"same_skills":56,"new_domains":44.95,"v":56.62,"lab":false,"jev":false,"rank_v":69,"rank_pub":69,"name":"Tern-35B-A3B","eq":{"S":55.59,"O":58.16},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":33.36,"new_domains":23.26}},"new_in_edition":true},{"engine":"jdi-ptr14-0.8b","name":"jdi-ptr14-0.8b","long_name":"jdi-ptr14-0.8b","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":752393024,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/AnamK/jdi-ptr14-0.8b","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"pr":230,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6002,"conf":0.6255,"ece":0.0416,"brier":0.5082,"over95":0.0198,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0613,"conf":0.038,"acc":0.0658},{"lo":0.1,"w":0.0082,"conf":0.1569,"acc":0.0191},{"lo":0.2,"w":0.0253,"conf":0.2647,"acc":0.26},{"lo":0.3,"w":0.0768,"conf":0.351,"acc":0.3566},{"lo":0.4,"w":0.0833,"conf":0.4495,"acc":0.4921},{"lo":0.5,"w":0.2135,"conf":0.5478,"acc":0.5566},{"lo":0.6,"w":0.1478,"conf":0.6462,"acc":0.6504},{"lo":0.7,"w":0.1051,"conf":0.7475,"acc":0.7125},{"lo":0.8,"w":0.0852,"conf":0.8491,"acc":0.8016},{"lo":0.9,"w":0.1933,"conf":0.9691,"acc":0.8425}],"areas":{"knowledge":{"ece":0.0385,"acc":0.5058,"conf":0.5184},"language":{"ece":0.049,"acc":0.7262,"conf":0.6913},"retrieval":{"ece":0.1417,"acc":0.694,"conf":0.8254},"tools":{"ece":0.1043,"acc":0.7028,"conf":0.7818},"arts":{"ece":0.0551,"acc":0.426,"conf":0.4678}}},"gaps":null,"latency":{"median":67,"p95":192,"mean":88.6},"results":{"1":{"score":0.8996,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":72},"2":{"score":0.6009,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":413.6},"3":{"score":0.4724,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":138.4},"4":{"score":0.8927,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":33.1},"5":{"score":0.928,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":46.6},"6":{"score":0.6096,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":103.2},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":170},"10":{"score":0.3854,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":38.1},"11":{"score":0.6638,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":231},"12":{"score":0.4843,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":29.8},"20":{"score":0.5286,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":29.7},"21":{"score":0.5772,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":29.2},"22":{"score":0.036,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":122.2},"23":{"score":0.5884,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.6},"24":{"score":0.4953,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":30},"25":{"score":0.3163,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":30.8},"26":{"score":0.8767,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":29.8},"27":{"score":0.7329,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":29.7},"28":{"score":0.6677,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":29.5},"29":{"score":0.8084,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":30.6},"30":{"score":0.3506,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.095,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.8},"32":{"score":0.5758,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":35.2},"33":{"score":0.0479,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":83.3},"36":{"score":0.3386,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":582},"37":{"score":0.5117,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.3},"38":{"score":0.1102,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":234.6},"39":{"score":0.787,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":65},"40":{"score":0.1188,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":29.9},"41":{"score":0.7431,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":30.2},"42":{"score":0.6213,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32},"43":{"score":0.3526,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":30.1},"44":{"score":0.5404,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.2},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":31.4},"50":{"score":0.3992,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":33},"56":{"score":0.505,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":75.7},"57":{"score":0.2542,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":31},"58":{"score":0.4516,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":30.9},"59":{"score":0.739,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":33.2},"61":{"score":0.793,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":31.3},"62":{"score":0.799,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":32.2},"64":{"score":0.5871,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":30.3}},"benchmarks":{"1":{"raw":0.8996,"skill":0.8645,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7081,"private_eq":0.7855},"2":{"raw":0.6009,"skill":0.5391,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6432,"private_eq":0.6937},"3":{"raw":0.4724,"skill":0.4622,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6809,"private_eq":0.6895},"4":{"raw":0.8927,"skill":0.8913,"coverage":1,"random":0.0127,"tracks":[],"combined":0.788,"private_eq":0.7921},"5":{"raw":0.928,"skill":0.9276,"coverage":1,"random":0.006,"tracks":[],"combined":0.7354,"private_eq":0.739},"6":{"raw":0.6096,"skill":0.0846,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.3854,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6638,"skill":0.5138,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4843,"skill":0.2275,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2365,"private_eq":0.4954},"20":{"raw":0.5266,"skill":0.0531,"coverage":1,"random":0.5,"tracks":[],"combined":0.1695,"private_eq":0.589},"21":{"raw":0.5772,"skill":0.1545,"coverage":1,"random":0.5,"tracks":[],"combined":0.0755,"private_eq":0.5425},"22":{"raw":0.0352,"skill":0.0277,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0194,"private_eq":0.0371},"23":{"raw":0.5793,"skill":0.1586,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4953,"skill":0.3271,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3163,"skill":0.0884,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2789,"combined":0.0285},"26":{"raw":0.8767,"skill":0.8356,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7329,"skill":0.6438,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6677,"skill":0.3354,"coverage":1,"random":0.5,"tracks":[],"combined":0.1283,"private_eq":0.5686},"29":{"raw":0.8084,"skill":0.7445,"coverage":1,"random":0.25,"tracks":[],"combined":0.498,"private_eq":0.627},"30":{"raw":0.3506,"skill":0.2181,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4564,"random":0.25},{"track":"10-choice","raw":0.2449,"random":0.1}],"combined":0.2958,"private_eq":0.4248},"31":{"raw":0.095,"skill":0.0143,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0041,"private_eq":0.0952},"32":{"raw":0.5758,"skill":0.3256,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.0479,"skill":0.0353,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1018,"private_eq":0.1227},"36":{"raw":0.3386,"skill":0.2517,"coverage":1,"random":0.116,"tracks":[],"combined":0.2253,"private_eq":0.3221},"37":{"raw":0.5117,"skill":0.3875,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5345,"private_eq":0.6323},"38":{"raw":0.1102,"skill":0.0817,"coverage":1,"random":0.031,"tracks":[],"combined":0.1922,"private_eq":0.2252},"39":{"raw":0.787,"skill":0.6867,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3714,"private_eq":0.5768},"40":{"raw":0.1188,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3735,"headline":false},{"track":"A · English","score":0.1188,"headline":true},{"track":"C · Arabic pairs","score":0.9,"headline":false},{"track":"C · English pairs","score":0.86,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7431,"skill":0.6147,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.6213,"skill":0.2632,"coverage":1,"random":0.486,"tracks":[],"combined":0.1239,"private_eq":0.5543},"43":{"raw":0.3526,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.1732,"private_eq":0.4842},"44":{"raw":0.5404,"skill":0.0808,"coverage":1,"random":0.5,"tracks":[],"combined":0.139,"private_eq":0.5739},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3992,"skill":0.128,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.505,"skill":0.01,"coverage":1,"random":0.5,"tracks":[],"combined":0.1097,"private_eq":0.5594},"57":{"raw":0.2542,"skill":0.1612,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2295,"combined":0.1244},"58":{"raw":0.4516,"skill":0.2051,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3266,"combined":0.0137},"59":{"raw":0.739,"skill":0.4588,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2533,"private_eq":0.6435},"61":{"raw":0.793,"skill":0.586,"coverage":1,"random":0.5,"tracks":[],"combined":0.5924,"private_eq":0.798},"62":{"raw":0.799,"skill":0.732,"coverage":1,"random":0.25,"tracks":[],"combined":0.5013,"private_eq":0.6295},"64":{"raw":0.5871,"skill":0.4839,"coverage":1,"random":0.2,"tracks":[],"combined":0.5161,"private_eq":0.6165}},"coverage":1,"pending":0,"notes":["New in the 2026-10-20 PR batch (PR #230).","Submitted public 0.3 run adopted (kit rescore 35.79, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 97.38% on 831 rows (bar 98.5%).","Listed under a maintainer decision on the agreement bar (97.38% vs 98.5%): every changed decision is a near-tie between options, and the submitted run used an RTX 3090.","Training uses train splits of several suite source datasets, with suite rows removed by exact and 13-gram match (author-declared).","License CC BY-NC 4.0."],"scores":{"balanced_raw":51.74,"balanced_skill":23.58,"breadth_skill":32.33,"public_skill":35.79},"frozen_scores":{"balanced_raw":51.74,"balanced_skill":35.79,"breadth_skill":32.33},"categories":[{"id":"knowledge","raw":0.3443,"skill":0.1352,"combined":0.0979,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5675,"skill":0.4027,"combined":0.2144,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6668,"skill":0.5255,"combined":0.5053,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5641,"skill":0.5302,"combined":0.5187,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4508,"skill":0.1676,"combined":0.1938,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jdi-ptr14-0.8b","pub":35.79,"same_skills":31.28,"new_domains":8.44,"v":23.58,"lab":false,"jev":false,"rank_v":187,"rank_pub":173,"name":"jdi-ptr14-0.8b","eq":{"S":30.58,"O":13.06},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":18.35,"new_domains":5.23}},"new_in_edition":true},{"engine":"glide-v1.2","name":"Fastino GLiDE 1.2","long_name":"Fastino GLiDE 1.2","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/fastino/GLiDE","code_url":"https://huggingface.co/fastino/GLiDE/tree/main/glide","base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"fastino/GLiDE","served_checkpoint":"a7e45896be7c96ba3354891da790c5ec99c30b6c","weights_note":null,"org":{"id":"fastino","name":"Fastino","logo":"data/orgs/fastino.png"},"pr":255,"replaces":"decision-27b-ckpt-nothink","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8149,"conf":0.8262,"ece":0.0204,"brier":0.2568,"over95":0.0072,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.009,"conf":0.0816,"acc":0.1135},{"lo":0.1,"w":0.0187,"conf":0.1404,"acc":0.227},{"lo":0.2,"w":0.0066,"conf":0.2448,"acc":0.3639},{"lo":0.3,"w":0.0231,"conf":0.3529,"acc":0.4347},{"lo":0.4,"w":0.0347,"conf":0.4522,"acc":0.4216},{"lo":0.5,"w":0.0789,"conf":0.5484,"acc":0.5137},{"lo":0.6,"w":0.0684,"conf":0.6494,"acc":0.6223},{"lo":0.7,"w":0.0816,"conf":0.7502,"acc":0.6683},{"lo":0.8,"w":0.1189,"conf":0.8562,"acc":0.838},{"lo":0.9,"w":0.56,"conf":0.976,"acc":0.9736}],"areas":{"knowledge":{"ece":0.0753,"acc":0.7287,"conf":0.7753},"language":{"ece":0.0181,"acc":0.9015,"conf":0.8834},"retrieval":{"ece":0.0348,"acc":0.8163,"conf":0.8358},"tools":{"ece":0.0388,"acc":0.9474,"conf":0.986},"arts":{"ece":0.0415,"acc":0.751,"conf":0.7216}}},"gaps":null,"latency":{"median":92.9,"p95":1121.1,"mean":284},"results":{"1":{"score":0.9734,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":308.2},"2":{"score":0.7122,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3217.8},"3":{"score":0.8701,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1613.4},"4":{"score":0.9201,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":187.8},"5":{"score":0.9589,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":325.4},"6":{"score":0.7964,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":906},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":3996.2},"10":{"score":0.6727,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":233.7},"11":{"score":0.847,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":6667.3},"12":{"score":0.806,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":64.6},"20":{"score":0.9838,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":65},"21":{"score":0.6861,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":55.1},"22":{"score":0.7935,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1388.7},"23":{"score":0.6628,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":120.7},"24":{"score":0.8841,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":59.3},"25":{"score":0.5816,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":68.1},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":56.6},"27":{"score":0.9718,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":58.1},"28":{"score":0.9463,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":54.7},"29":{"score":0.9771,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":81.9},"30":{"score":0.8529,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2512,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":187},"32":{"score":0.762,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":191.6},"33":{"score":0.4491,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":681},"36":{"score":0.4856,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4497},"37":{"score":0.6145,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":85.5},"38":{"score":0.6282,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1974.4},"39":{"score":0.9426,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":83.4},"40":{"score":0.6943,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":54.9},"41":{"score":0.8076,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":65.4},"42":{"score":0.872,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":119.9},"43":{"score":0.8316,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":64.8},"44":{"score":0.9286,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":64.1},"45":{"score":0.1816,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":81},"50":{"score":0.6444,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":167.4},"56":{"score":0.8585,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":459.7},"57":{"score":0.7804,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":67.8},"58":{"score":0.9299,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":65.2},"59":{"score":0.8562,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":162.7},"61":{"score":0.8898,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":101.3},"62":{"score":0.9162,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":161},"64":{"score":0.8295,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":66.8}},"benchmarks":{"1":{"raw":0.9734,"skill":0.9641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.7122,"skill":0.6676,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6994,"private_eq":0.7418},"3":{"raw":0.8701,"skill":0.8676,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9433,"private_eq":0.944},"4":{"raw":0.9201,"skill":0.9191,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8854,"private_eq":0.8871},"5":{"raw":0.9589,"skill":0.9587,"coverage":1,"random":0.006,"tracks":[],"combined":0.8829,"private_eq":0.8839},"6":{"raw":0.7964,"skill":0.5226,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9252,"private_eq":0.925},"10":{"raw":0.6727,"skill":0.4554,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.847,"skill":0.7787,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.806,"skill":0.7094,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5298,"private_eq":0.689},"20":{"raw":0.984,"skill":0.9679,"coverage":1,"random":0.5,"tracks":[],"combined":0.9577,"private_eq":0.9786},"21":{"raw":0.6861,"skill":0.3721,"coverage":1,"random":0.5,"tracks":[],"combined":0.2968,"private_eq":0.6519},"22":{"raw":0.794,"skill":0.7924,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5675,"private_eq":0.5748},"23":{"raw":0.6762,"skill":0.3524,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8841,"skill":0.8455,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5816,"skill":0.4422,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.709,"combined":0.6085},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9718,"skill":0.9624,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9463,"skill":0.8926,"coverage":1,"random":0.5,"tracks":[],"combined":0.6581,"private_eq":0.8305},"29":{"raw":0.9771,"skill":0.9695,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8529,"skill":0.8245,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8916,"random":0.25},{"track":"10-choice","raw":0.8143,"random":0.1}],"combined":0.7661,"private_eq":0.8084},"31":{"raw":0.2512,"skill":0.1844,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0702,"private_eq":0.1552},"32":{"raw":0.762,"skill":0.6216,"coverage":1,"random":0.371,"tracks":[],"combined":0.4054,"private_eq":0.6296},"33":{"raw":0.4491,"skill":0.4418,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4154,"private_eq":0.4286},"36":{"raw":0.4856,"skill":0.4181,"coverage":1,"random":0.116,"tracks":[],"combined":0.4226,"private_eq":0.4945},"37":{"raw":0.6145,"skill":0.5164,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6477,"private_eq":0.7215},"38":{"raw":0.6282,"skill":0.6163,"coverage":1,"random":0.031,"tracks":[],"combined":0.7548,"private_eq":0.7641},"39":{"raw":0.9426,"skill":0.9156,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5803,"private_eq":0.7172},"40":{"raw":0.6943,"skill":0.6067,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4857,"headline":false},{"track":"A · English","score":0.6943,"headline":true},{"track":"C · Arabic pairs","score":0.95,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.5795,"private_eq":0.6761},"41":{"raw":0.8076,"skill":0.7115,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4952,"private_eq":0.6666},"42":{"raw":0.872,"skill":0.751,"coverage":1,"random":0.486,"tracks":[],"combined":0.9955,"private_eq":0.9972},"43":{"raw":0.8316,"skill":0.7328,"coverage":1,"random":0.3697,"tracks":[],"combined":0.8169,"private_eq":0.8853},"44":{"raw":0.9286,"skill":0.8572,"coverage":1,"random":0.5,"tracks":[],"combined":0.7708,"private_eq":0.8862},"45":{"raw":0.1816,"skill":0.0209,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6444,"skill":0.4839,"coverage":1,"random":0.311,"tracks":[],"combined":0.3338,"private_eq":0.5455},"56":{"raw":0.8585,"skill":0.717,"coverage":1,"random":0.5,"tracks":[],"combined":0.8626,"private_eq":0.9316},"57":{"raw":0.7804,"skill":0.753,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.8995,"combined":0.8867},"58":{"raw":0.9299,"skill":0.8984,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5199,"combined":0.2971},"59":{"raw":0.8562,"skill":0.7018,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2716,"private_eq":0.6522},"61":{"raw":0.8898,"skill":0.7796,"coverage":1,"random":0.5,"tracks":[],"combined":0.6834,"private_eq":0.843},"62":{"raw":0.9162,"skill":0.8883,"coverage":1,"random":0.25,"tracks":[],"combined":0.693,"private_eq":0.7716},"64":{"raw":0.8295,"skill":0.7869,"coverage":1,"random":0.2,"tracks":[],"combined":0.778,"private_eq":0.8236}},"coverage":1,"pending":0,"notes":["New in the 2026-10-21 PR batch (PR #255).","Replaces the earlier board entry decision-27b-ckpt-nothink (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 71.99). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.90% on 895 rows (bar 98.5%).","Weights are public on the Hugging Face Hub (fastino-community license); the lab scored revision a7e4589."],"scores":{"balanced_raw":78.89,"balanced_skill":71.52,"breadth_skill":71.44,"public_skill":71.99},"frozen_scores":{"balanced_raw":78.89,"balanced_skill":71.99,"breadth_skill":71.44},"categories":[{"id":"knowledge","raw":0.6816,"skill":0.6059,"combined":0.5727,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8285,"skill":0.7549,"combined":0.6564,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7879,"skill":0.7224,"combined":0.7307,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8964,"skill":0.8804,"combined":0.8491,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.769,"skill":0.6259,"combined":0.5867,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"glide-v1.2","pub":71.99,"same_skills":68.57,"new_domains":59.66,"v":71.52,"lab":false,"jev":false,"rank_v":2,"rank_pub":5,"name":"Fastino GLiDE 1.2","eq":{"S":68.31,"O":76.32},"rank_tie":1,"tied":true,"parts":{"public":0,"same_skills":40.99,"new_domains":30.53}},"new_in_edition":true},{"engine":"allec","name":"allec (Gemma 4 31B)","long_name":"allec (Gemma 4 31B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/plantwaterer/allec","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"pr":250,"new":true,"show_notes":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7552,"conf":0.8009,"ece":0.0457,"brier":0.3293,"over95":0.0116,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0004,"conf":0.0879,"acc":0},{"lo":0.1,"w":0.0115,"conf":0.1541,"acc":0.0884},{"lo":0.2,"w":0.0179,"conf":0.2534,"acc":0.2362},{"lo":0.3,"w":0.0264,"conf":0.3541,"acc":0.2367},{"lo":0.4,"w":0.0544,"conf":0.4534,"acc":0.2926},{"lo":0.5,"w":0.0891,"conf":0.5484,"acc":0.5012},{"lo":0.6,"w":0.086,"conf":0.6508,"acc":0.5843},{"lo":0.7,"w":0.1114,"conf":0.7512,"acc":0.6707},{"lo":0.8,"w":0.1354,"conf":0.852,"acc":0.8201},{"lo":0.9,"w":0.4674,"conf":0.9768,"acc":0.9565}],"areas":{"knowledge":{"ece":0.0662,"acc":0.6996,"conf":0.7619},"language":{"ece":0.0224,"acc":0.8559,"conf":0.8426},"retrieval":{"ece":0.0763,"acc":0.6601,"conf":0.7362},"tools":{"ece":0.0262,"acc":0.9372,"conf":0.9634},"arts":{"ece":0.1063,"acc":0.6516,"conf":0.7579}}},"gaps":null,"latency":{"median":102.6,"p95":1714.2,"mean":406.9},"results":{"1":{"score":0.9669,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":348.8},"2":{"score":0.6541,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2632.1},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":6889.5},"4":{"score":0.7732,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":320.1},"5":{"score":0.8975,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":316.7},"6":{"score":0.7991,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1177.9},"9":{"score":0.6705,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":4608.4},"10":{"score":0.3606,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":257.5},"11":{"score":0.7011,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":9215.2},"12":{"score":0.7597,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":92.5},"20":{"score":0.9622,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":89},"21":{"score":0.645,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":78.6},"22":{"score":0.536,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":3476.1},"23":{"score":0.6462,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":121.2},"24":{"score":0.875,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":93.6},"25":{"score":0.5102,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":94.7},"26":{"score":0.9928,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":87.8},"27":{"score":0.9744,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":92.7},"28":{"score":0.8832,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":73.6},"29":{"score":0.9332,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":96.1},"30":{"score":0.7418,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2286,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":155.5},"32":{"score":0.6596,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":202.7},"33":{"score":0.317,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":815.1},"36":{"score":0.4711,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4450.9},"37":{"score":0.5612,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.6},"38":{"score":0.2859,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1863},"39":{"score":0.9381,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":122.1},"40":{"score":0.7226,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":95.5},"41":{"score":0.7252,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":94.7},"42":{"score":0.8516,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":113.5},"43":{"score":0.8228,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":94},"44":{"score":0.8388,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":93.2},"45":{"score":0.1337,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":96.3},"50":{"score":0.4576,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":178.6},"56":{"score":0.881,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":516.9},"57":{"score":0.7094,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":95.6},"58":{"score":0.7814,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":95.4},"59":{"score":0.7806,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":173.5},"61":{"score":0.793,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":105.4},"62":{"score":0.807,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":161.7},"64":{"score":0.7405,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":92.9}},"benchmarks":{"1":{"raw":0.9669,"skill":0.9554,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8989,"private_eq":0.9252},"2":{"raw":0.6541,"skill":0.6005,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6959,"private_eq":0.7388},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.7732,"skill":0.7703,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8551,"private_eq":0.8576},"5":{"raw":0.8975,"skill":0.8969,"coverage":1,"random":0.006,"tracks":[],"combined":0.7774,"private_eq":0.7803},"6":{"raw":0.7991,"skill":0.529,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6705,"skill":0.6705,"coverage":1,"random":0,"tracks":[],"combined":0.6571,"private_eq":0.66},"10":{"raw":0.3606,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7011,"skill":0.5677,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7597,"skill":0.6401,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5563,"private_eq":0.7065},"20":{"raw":0.9632,"skill":0.9264,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.645,"skill":0.29,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.5314,"skill":0.5277,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5243,"private_eq":0.5324},"23":{"raw":0.6362,"skill":0.2723,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.875,"skill":0.8333,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5102,"skill":0.3469,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5127,"combined":0.3438},"26":{"raw":0.9928,"skill":0.9904,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9744,"skill":0.9659,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8832,"skill":0.7664,"coverage":1,"random":0.5,"tracks":[],"combined":0.7338,"private_eq":0.8679},"29":{"raw":0.9332,"skill":0.9109,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7418,"skill":0.6917,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8067,"random":0.25},{"track":"10-choice","raw":0.677,"random":0.1}],"combined":0.7678,"private_eq":0.8098},"31":{"raw":0.2286,"skill":0.1598,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1189,"private_eq":0.1994},"32":{"raw":0.6596,"skill":0.4588,"coverage":1,"random":0.371,"tracks":[],"combined":0.4285,"private_eq":0.644},"33":{"raw":0.317,"skill":0.3079,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3383,"private_eq":0.3534},"36":{"raw":0.4711,"skill":0.4017,"coverage":1,"random":0.116,"tracks":[],"combined":0.4025,"private_eq":0.4769},"37":{"raw":0.5612,"skill":0.4497,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5269,"private_eq":0.6263},"38":{"raw":0.2859,"skill":0.2631,"coverage":1,"random":0.031,"tracks":[],"combined":0.4655,"private_eq":0.487},"39":{"raw":0.9381,"skill":0.909,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5377,"private_eq":0.6886},"40":{"raw":0.7226,"skill":0.6432,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4933,"headline":false},{"track":"A · English","score":0.7226,"headline":true},{"track":"C · Arabic pairs","score":0.91,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.7648,"private_eq":0.8185},"41":{"raw":0.7252,"skill":0.5878,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5284,"private_eq":0.6885},"42":{"raw":0.8516,"skill":0.7113,"coverage":1,"random":0.486,"tracks":[],"combined":0.7527,"private_eq":0.8738},"43":{"raw":0.8228,"skill":0.7189,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6332,"private_eq":0.7708},"44":{"raw":0.8388,"skill":0.6776,"coverage":1,"random":0.5,"tracks":[],"combined":0.6423,"private_eq":0.8227},"45":{"raw":0.1337,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4576,"skill":0.2129,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.881,"skill":0.762,"coverage":1,"random":0.5,"tracks":[],"combined":0.802,"private_eq":0.9016},"57":{"raw":0.7094,"skill":0.6732,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6397,"combined":0.5911},"58":{"raw":0.7814,"skill":0.6831,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5319,"combined":0.3147},"59":{"raw":0.7806,"skill":0.5451,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2905,"private_eq":0.6612},"61":{"raw":0.793,"skill":0.586,"coverage":1,"random":0.5,"tracks":[],"combined":0.5131,"private_eq":0.7588},"62":{"raw":0.807,"skill":0.7427,"coverage":1,"random":0.25,"tracks":[],"combined":0.5082,"private_eq":0.6346},"64":{"raw":0.7405,"skill":0.6756,"coverage":1,"random":0.2,"tracks":[],"combined":0.7392,"private_eq":0.7929}},"coverage":1,"pending":0,"notes":["New in the 2026-10-21 PR batch (PR #250).","Submitted public 0.3 run adopted (kit rescore 61.60, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.80% on 831 rows (bar 98.5%).","Same author and engine family as garas.","Two private new-domain tests are not counted for this model: our checks found signs its training data overlapped them. Its new-domain score uses the remaining tests, put on the same scale with the same stock models."],"scores":{"balanced_raw":70.74,"balanced_skill":66.01,"breadth_skill":60.89,"public_skill":61.6},"frozen_scores":{"balanced_raw":70.74,"balanced_skill":61.6,"breadth_skill":60.89},"categories":[{"id":"knowledge","raw":0.6025,"skill":0.4982,"combined":0.4915,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7486,"skill":0.6477,"combined":0.6201,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7281,"skill":0.6485,"combined":0.6491,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7995,"skill":0.7744,"combined":0.7494,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6623,"skill":0.4841,"combined":0.5451,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"allec","pub":61.6,"same_skills":61.63,"new_domains":57.04,"v":66.01,"lab":false,"jev":false,"rank_v":26,"rank_pub":52,"name":"allec (Gemma 4 31B)","eq":{"S":61.29,"O":73.1},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":36.77,"new_domains":29.24}},"new_in_edition":true},{"engine":"lumma-fev-27b","name":"Lumma-fev-27B","long_name":"Lumma-fev-27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/FrontiersMind/Lumma-fev-27B","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"FrontiersMind","name":"FrontiersMind","logo":"data/orgs/FrontiersMind.png"},"pr":254,"new":true},"completed":110201,"counts":{"ok":110177,"unsupported":0,"error":24},"lifted":false,"calibration":{"acc":0.737,"conf":0.805,"ece":0.0683,"brier":0.3556,"over95":0.0159,"n":34407,"benchmarks":31,"rel":[{"lo":0,"w":0.0006,"conf":0.0877,"acc":0.3131},{"lo":0.1,"w":0.0108,"conf":0.1588,"acc":0.1293},{"lo":0.2,"w":0.0218,"conf":0.2596,"acc":0.2116},{"lo":0.3,"w":0.0311,"conf":0.3549,"acc":0.2369},{"lo":0.4,"w":0.0452,"conf":0.4541,"acc":0.3592},{"lo":0.5,"w":0.1001,"conf":0.5506,"acc":0.458},{"lo":0.6,"w":0.0807,"conf":0.654,"acc":0.5564},{"lo":0.7,"w":0.0942,"conf":0.7548,"acc":0.6065},{"lo":0.8,"w":0.1126,"conf":0.8532,"acc":0.7837},{"lo":0.9,"w":0.5028,"conf":0.9763,"acc":0.9369}],"areas":{"knowledge":{"ece":0.1006,"acc":0.6529,"conf":0.7535},"language":{"ece":0.0079,"acc":0.8387,"conf":0.8463},"retrieval":{"ece":0.127,"acc":0.7135,"conf":0.8405},"tools":{"ece":0.0523,"acc":0.9415,"conf":0.9821},"arts":{"ece":0.092,"acc":0.6185,"conf":0.6989}}},"gaps":null,"latency":{"median":100,"p95":1075.4,"mean":303.7},"results":{"1":{"score":0.9681,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":332.2},"2":{"score":0.6937,"jev":null,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":0,"errors":2,"median_ms":3502.4},"3":{"score":0.8498,"jev":null,"metric":"accuracy","requests":508,"answered":506,"unsupported":0,"errors":2,"median_ms":1567.2},"4":{"score":0.9146,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":212.2},"5":{"score":0.905,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":342.1},"6":{"score":0.7975,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1063},"9":{"score":0.7159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":4418},"10":{"score":0.4784,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":262.5},"11":{"score":0.7591,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":7399.1},"12":{"score":0.7204,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":100.5},"20":{"score":0.9262,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":100.2},"21":{"score":0.6218,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":99.4},"22":{"score":0.4835,"jev":null,"metric":"accuracy","requests":2000,"answered":1975,"unsupported":0,"errors":25,"median_ms":1305.6},"23":{"score":0.6436,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123.7},"24":{"score":0.8404,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":99.2},"25":{"score":0.4796,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":100.1},"26":{"score":0.9903,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":99.6},"27":{"score":0.9701,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":99.4},"28":{"score":0.8319,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":100.5},"29":{"score":0.9334,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":100.5},"30":{"score":0.6042,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1796,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":217.9},"32":{"score":0.6197,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":223.6},"33":{"score":0.3109,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":791.6},"36":{"score":0.499,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":6087.8},"37":{"score":0.5617,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111},"38":{"score":0.1924,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2478.1},"39":{"score":0.9337,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":105.5},"40":{"score":0.5629,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":100.7},"41":{"score":0.6973,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":99.1},"42":{"score":0.8318,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":125.3},"43":{"score":0.7386,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":99.5},"44":{"score":0.734,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.1},"45":{"score":0.102,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":0,"errors":1,"median_ms":100},"50":{"score":0.4403,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":197.2},"56":{"score":0.6255,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":581.5},"57":{"score":0.6502,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":99.2},"58":{"score":0.785,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":99.1},"59":{"score":0.781,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":180.6},"61":{"score":0.7027,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":111.5},"62":{"score":0.8028,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":176.5},"64":{"score":0.7235,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":99.2}},"benchmarks":{"1":{"raw":0.9681,"skill":0.957,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6917,"skill":0.644,"coverage":0.9971,"random":0.1341,"tracks":[],"combined":0.6937,"private_eq":0.7369},"3":{"raw":0.8465,"skill":0.8435,"coverage":0.9961,"random":0.0189,"tracks":[],"combined":0.9013,"private_eq":0.9033},"4":{"raw":0.9146,"skill":0.9135,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8904,"private_eq":0.892},"5":{"raw":0.905,"skill":0.9044,"coverage":1,"random":0.006,"tracks":[],"combined":0.7432,"private_eq":0.7466},"6":{"raw":0.7975,"skill":0.5252,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7159,"skill":0.7159,"coverage":1,"random":0,"tracks":[],"combined":0.6647,"private_eq":0.6675},"10":{"raw":0.4784,"skill":0.1321,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7591,"skill":0.6515,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7204,"skill":0.5812,"coverage":1,"random":0.3324,"tracks":[],"combined":0.448,"private_eq":0.635},"20":{"raw":0.9291,"skill":0.8582,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6218,"skill":0.2435,"coverage":1,"random":0.5,"tracks":[],"combined":0.2466,"private_eq":0.6271},"22":{"raw":0.473,"skill":0.4689,"coverage":0.9875,"random":0.0078,"tracks":[],"combined":0.3834,"private_eq":0.3942},"23":{"raw":0.637,"skill":0.274,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8404,"skill":0.7872,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4607,"combined":0.2737},"26":{"raw":0.9903,"skill":0.9871,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9701,"skill":0.9601,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8319,"skill":0.6638,"coverage":1,"random":0.5,"tracks":[],"combined":0.5319,"private_eq":0.7681},"29":{"raw":0.9334,"skill":0.9112,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.6042,"skill":0.5282,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7111,"random":0.25},{"track":"10-choice","raw":0.4973,"random":0.1}],"combined":0.679,"private_eq":0.7374},"31":{"raw":0.1796,"skill":0.1065,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.6197,"skill":0.3954,"coverage":1,"random":0.371,"tracks":[],"combined":0.328,"private_eq":0.5815},"33":{"raw":0.3109,"skill":0.3018,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2663,"private_eq":0.2832},"36":{"raw":0.499,"skill":0.4332,"coverage":1,"random":0.116,"tracks":[],"combined":0.3892,"private_eq":0.4653},"37":{"raw":0.5617,"skill":0.4502,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5571,"private_eq":0.6501},"38":{"raw":0.1924,"skill":0.1666,"coverage":1,"random":0.031,"tracks":[],"combined":0.3019,"private_eq":0.3303},"39":{"raw":0.9337,"skill":0.9025,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5054,"private_eq":0.6669},"40":{"raw":0.5629,"skill":0.4377,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3684,"headline":false},{"track":"A · English","score":0.5629,"headline":true},{"track":"C · Arabic pairs","score":0.735,"headline":false},{"track":"C · English pairs","score":0.95,"headline":false}],"combined":0.4146,"private_eq":0.5494},"41":{"raw":0.6973,"skill":0.546,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5258,"private_eq":0.6868},"42":{"raw":0.8318,"skill":0.6728,"coverage":1,"random":0.486,"tracks":[],"combined":0.9266,"private_eq":0.9622},"43":{"raw":0.7386,"skill":0.5853,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5619,"private_eq":0.7264},"44":{"raw":0.734,"skill":0.468,"coverage":1,"random":0.5,"tracks":[],"combined":0.4655,"private_eq":0.7353},"45":{"raw":0.1018,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.4403,"skill":0.1877,"coverage":1,"random":0.311,"tracks":[],"combined":0.167,"private_eq":0.4319},"56":{"raw":0.6255,"skill":0.251,"coverage":1,"random":0.5,"tracks":[],"combined":0.3779,"private_eq":0.692},"57":{"raw":0.6502,"skill":0.6066,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5606,"combined":0.5011},"58":{"raw":0.785,"skill":0.6884,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4744,"combined":0.2304},"59":{"raw":0.781,"skill":0.5459,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3494,"private_eq":0.6893},"61":{"raw":0.7027,"skill":0.4054,"coverage":1,"random":0.5,"tracks":[],"combined":0.5187,"private_eq":0.7616},"62":{"raw":0.8028,"skill":0.7371,"coverage":1,"random":0.25,"tracks":[],"combined":0.6725,"private_eq":0.7564},"64":{"raw":0.7235,"skill":0.6544,"coverage":1,"random":0.2,"tracks":[],"combined":0.6422,"private_eq":0.7162}},"coverage":0.9993,"pending":0.0007,"notes":["New in the 2026-10-21 PR batch (PR #254).","Submitted public 0.3 run adopted (kit rescore 57.15, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.25% on 825 rows (bar 98.5%).","Questions over its declared 8,192-token limit count as unsupported."],"scores":{"balanced_raw":68.17,"balanced_skill":59.97,"breadth_skill":56.01,"public_skill":57.15},"frozen_scores":{"balanced_raw":68.17,"balanced_skill":57.15,"breadth_skill":56.01},"categories":[{"id":"knowledge","raw":0.5504,"skill":0.4253,"combined":0.3915,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.7173,"skill":0.6079,"combined":0.567,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7079,"skill":0.577,"combined":0.588,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8126,"skill":0.7884,"combined":0.7882,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.6374,"skill":0.4478,"combined":0.4649,"coverage":0.9979,"pending":0,"n":6}],"v03":{"engine":"lumma-fev-27b","pub":57.15,"same_skills":56.59,"new_domains":51.01,"v":59.97,"lab":false,"jev":false,"rank_v":50,"rank_pub":90,"name":"Lumma-fev-27B","eq":{"S":56.2,"O":65.64},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":33.72,"new_domains":26.26}},"new_in_edition":true},{"engine":"muria","name":"muria (Gemma 4 12B)","long_name":"muria (Gemma 4 12B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":11959730224,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/plantwaterer/muria","code_url":null,"base_model":"google/gemma-4-12B-it","base_url":"https://huggingface.co/google/gemma-4-12B-it","weights_repo":null,"served_checkpoint":null,"pr":256,"new":true,"show_notes":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6924,"conf":0.7631,"ece":0.0707,"brier":0.4214,"over95":0.0108,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0008,"conf":0.0872,"acc":0.05},{"lo":0.1,"w":0.0116,"conf":0.1547,"acc":0.127},{"lo":0.2,"w":0.0192,"conf":0.2589,"acc":0.1307},{"lo":0.3,"w":0.037,"conf":0.3527,"acc":0.2399},{"lo":0.4,"w":0.0522,"conf":0.4511,"acc":0.341},{"lo":0.5,"w":0.1126,"conf":0.5509,"acc":0.4832},{"lo":0.6,"w":0.1161,"conf":0.6532,"acc":0.6017},{"lo":0.7,"w":0.1303,"conf":0.7526,"acc":0.6756},{"lo":0.8,"w":0.163,"conf":0.853,"acc":0.7532},{"lo":0.9,"w":0.3571,"conf":0.9651,"acc":0.9145}],"areas":{"knowledge":{"ece":0.0865,"acc":0.619,"conf":0.7055},"language":{"ece":0.0106,"acc":0.8024,"conf":0.8101},"retrieval":{"ece":0.1799,"acc":0.5913,"conf":0.7613},"tools":{"ece":0.0186,"acc":0.9183,"conf":0.9009},"arts":{"ece":0.1271,"acc":0.5867,"conf":0.7127}}},"gaps":null,"latency":{"median":53.5,"p95":698.9,"mean":183.1},"results":{"1":{"score":0.9551,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":166.6},"2":{"score":0.6492,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1133.9},"3":{"score":0.811,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2936.6},"4":{"score":0.7511,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":175.2},"5":{"score":0.8631,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":176.4},"6":{"score":0.799,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":490.9},"9":{"score":0.4432,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1968.8},"10":{"score":0.0744,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":115.8},"11":{"score":0.6828,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3652.8},"12":{"score":0.7171,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":66},"20":{"score":0.916,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":64.1},"21":{"score":0.6366,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":74.2},"22":{"score":0.2515,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1452.6},"23":{"score":0.6432,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":71.8},"24":{"score":0.789,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":73.9},"25":{"score":0.3724,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":74.2},"26":{"score":0.9853,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":64.7},"27":{"score":0.9471,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":74},"28":{"score":0.7419,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":48.8},"29":{"score":0.8725,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":57.2},"30":{"score":0.3654,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1414,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":81},"32":{"score":0.621,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":94.9},"33":{"score":0.2691,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":346.6},"36":{"score":0.4542,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1965.1},"37":{"score":0.531,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77.1},"38":{"score":0.2141,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":836.9},"39":{"score":0.9261,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":78.9},"40":{"score":0.6331,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":76.4},"41":{"score":0.5467,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":51.1},"42":{"score":0.8237,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":77.6},"43":{"score":0.5789,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":54.9},"44":{"score":0.734,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":64.9},"45":{"score":0.1078,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":75.8},"50":{"score":0.4433,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":86.3},"56":{"score":0.8175,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":238},"57":{"score":0.5516,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":74.3},"58":{"score":0.6777,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":71.1},"59":{"score":0.763,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":83},"61":{"score":0.7248,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":76.2},"62":{"score":0.7675,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":81.5},"64":{"score":0.6231,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":62.3}},"benchmarks":{"1":{"raw":0.9551,"skill":0.9394,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8852,"private_eq":0.9152},"2":{"raw":0.6492,"skill":0.5949,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6842,"private_eq":0.7288},"3":{"raw":0.811,"skill":0.8074,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8278,"private_eq":0.832},"4":{"raw":0.7511,"skill":0.7479,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8138,"private_eq":0.8173},"5":{"raw":0.8631,"skill":0.8623,"coverage":1,"random":0.006,"tracks":[],"combined":0.6709,"private_eq":0.6756},"6":{"raw":0.799,"skill":0.5287,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4432,"skill":0.4432,"coverage":1,"random":0,"tracks":[],"combined":0.5408,"private_eq":0.545},"10":{"raw":0.0744,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6828,"skill":0.5413,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7171,"skill":0.5762,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5542,"private_eq":0.7051},"20":{"raw":0.9179,"skill":0.8358,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.6366,"skill":0.2732,"coverage":1,"random":0.5,"tracks":[],"combined":0.1661,"private_eq":0.5873},"22":{"raw":0.23,"skill":0.224,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2814,"private_eq":0.2941},"23":{"raw":0.6448,"skill":0.2895,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.789,"skill":0.7187,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3724,"skill":0.1633,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4015,"combined":0.1938},"26":{"raw":0.9853,"skill":0.9804,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9471,"skill":0.9294,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7419,"skill":0.4838,"coverage":1,"random":0.5,"tracks":[],"combined":0.6984,"private_eq":0.8504},"29":{"raw":0.8725,"skill":0.83,"coverage":1,"random":0.25,"tracks":[],"combined":0.7603,"private_eq":0.8215},"30":{"raw":0.3654,"skill":0.2365,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4738,"random":0.25},{"track":"10-choice","raw":0.257,"random":0.1}],"combined":0.4896,"private_eq":0.5829},"31":{"raw":0.1414,"skill":0.0649,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0755,"private_eq":0.16},"32":{"raw":0.621,"skill":0.3975,"coverage":1,"random":0.371,"tracks":[],"combined":0.2582,"private_eq":0.5381},"33":{"raw":0.2691,"skill":0.2594,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1635,"private_eq":0.1829},"36":{"raw":0.4542,"skill":0.3825,"coverage":1,"random":0.116,"tracks":[],"combined":0.3657,"private_eq":0.4448},"37":{"raw":0.531,"skill":0.4118,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4257,"private_eq":0.5465},"38":{"raw":0.2141,"skill":0.189,"coverage":1,"random":0.031,"tracks":[],"combined":0.3827,"private_eq":0.4077},"39":{"raw":0.9261,"skill":0.8913,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5377,"private_eq":0.6886},"40":{"raw":0.6331,"skill":0.5279,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3118,"headline":false},{"track":"A · English","score":0.6331,"headline":true},{"track":"C · Arabic pairs","score":0.72,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}],"combined":0.6352,"private_eq":0.7189},"41":{"raw":0.5467,"skill":0.3201,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4745,"private_eq":0.653},"42":{"raw":0.8237,"skill":0.657,"coverage":1,"random":0.486,"tracks":[],"combined":0.5868,"private_eq":0.7895},"43":{"raw":0.5789,"skill":0.332,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5579,"private_eq":0.7239},"44":{"raw":0.734,"skill":0.468,"coverage":1,"random":0.5,"tracks":[],"combined":0.4441,"private_eq":0.7247},"45":{"raw":0.1078,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4433,"skill":0.1921,"coverage":1,"random":0.311,"tracks":[],"combined":0.2974,"private_eq":0.5207},"56":{"raw":0.8175,"skill":0.635,"coverage":1,"random":0.5,"tracks":[],"combined":0.7935,"private_eq":0.8974},"57":{"raw":0.5516,"skill":0.4957,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5714,"combined":0.5134},"58":{"raw":0.6777,"skill":0.5328,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4608,"combined":0.2104},"59":{"raw":0.763,"skill":0.5086,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2777,"private_eq":0.6551},"61":{"raw":0.7248,"skill":0.4496,"coverage":1,"random":0.5,"tracks":[],"combined":0.4904,"private_eq":0.7476},"62":{"raw":0.7675,"skill":0.69,"coverage":1,"random":0.25,"tracks":[],"combined":0.5562,"private_eq":0.6702},"64":{"raw":0.6231,"skill":0.5289,"coverage":1,"random":0.2,"tracks":[],"combined":0.6228,"private_eq":0.7009}},"coverage":1,"pending":0,"notes":["New in the 2026-10-21 PR batch (PR #256).","Submitted public 0.3 run adopted (kit rescore 51.40, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.41% on 831 rows (bar 98.5%).","Same author and engine family as garas, garum and allec.","Two private new-domain tests are not counted for this model: our checks found signs its training data overlapped them. Its new-domain score uses the remaining tests, put on the same scale with the same stock models."],"scores":{"balanced_raw":63.16,"balanced_skill":52.55,"breadth_skill":49.63,"public_skill":51.4},"frozen_scores":{"balanced_raw":63.16,"balanced_skill":51.4,"breadth_skill":49.63},"categories":[{"id":"knowledge","raw":0.471,"skill":0.3141,"combined":0.3637,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6912,"skill":0.5662,"combined":0.5324,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6902,"skill":0.589,"combined":0.5955,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7369,"skill":0.7082,"combined":0.7105,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5826,"skill":0.3906,"combined":0.4419,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"muria","pub":51.4,"same_skills":53.5,"new_domains":39.78,"v":52.55,"lab":false,"jev":false,"rank_v":87,"rank_pub":112,"name":"muria (Gemma 4 12B)","eq":{"S":53.07,"O":51.78},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":31.84,"new_domains":20.71}},"new_in_edition":true},{"engine":"mica-v0.5-4b","name":"Mica v0.5 4B","long_name":"Mica v0.5 4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sky7350/Mica-v0.5-4B","code_url":"https://github.com/akivet/Mica/tree/4adbdd398d6e2a9ca0a63b1e3d3b49240f03cee5","base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"akivet","name":"akivet","logo":"data/orgs/akivet.png"},"pr":258,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7181,"conf":0.8181,"ece":0.101,"brier":0.3972,"over95":0.0114,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0003,"conf":0.0721,"acc":0.3333},{"lo":0.1,"w":0.0024,"conf":0.1528,"acc":0.0414},{"lo":0.2,"w":0.0092,"conf":0.2607,"acc":0.3092},{"lo":0.3,"w":0.023,"conf":0.361,"acc":0.1832},{"lo":0.4,"w":0.0369,"conf":0.4547,"acc":0.2798},{"lo":0.5,"w":0.0843,"conf":0.5496,"acc":0.3927},{"lo":0.6,"w":0.086,"conf":0.6521,"acc":0.5156},{"lo":0.7,"w":0.1057,"conf":0.7514,"acc":0.5692},{"lo":0.8,"w":0.1721,"conf":0.8548,"acc":0.7163},{"lo":0.9,"w":0.48,"conf":0.9609,"acc":0.9158}],"areas":{"knowledge":{"ece":0.1504,"acc":0.6307,"conf":0.7769},"language":{"ece":0.0169,"acc":0.8469,"conf":0.8635},"retrieval":{"ece":0.2072,"acc":0.6535,"conf":0.8606},"tools":{"ece":0.0233,"acc":0.9394,"conf":0.9576},"arts":{"ece":0.1229,"acc":0.5799,"conf":0.701}}},"gaps":null,"latency":{"median":19.7,"p95":147.5,"mean":70.5},"results":{"1":{"score":0.9498,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":73.9},"2":{"score":0.6638,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":886.7},"3":{"score":0.8583,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":222.7},"4":{"score":0.864,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":28.7},"5":{"score":0.9227,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":35.9},"6":{"score":0.7949,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":146.5},"9":{"score":0.7727,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":246.4},"10":{"score":0.1523,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":50},"11":{"score":0.8155,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":205.5},"12":{"score":0.6842,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":17.1},"20":{"score":0.9022,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.2},"21":{"score":0.6142,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":16.5},"22":{"score":0.394,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":227.3},"23":{"score":0.6338,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":59.2},"24":{"score":0.7691,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":40.1},"25":{"score":0.5051,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":38.8},"26":{"score":0.9756,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":16.5},"27":{"score":0.9377,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":16.6},"28":{"score":0.8327,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":16.5},"29":{"score":0.9281,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":18.6},"30":{"score":0.5428,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.175,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.3},"32":{"score":0.5838,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":41.8},"33":{"score":0.3461,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":157.9},"36":{"score":0.4514,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1360.7},"37":{"score":0.554,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.1},"38":{"score":0.4239,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":354.9},"39":{"score":0.9064,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":33.7},"40":{"score":0.5551,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":16.5},"41":{"score":0.7787,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":17.2},"42":{"score":0.7808,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":26.8},"43":{"score":0.514,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":17},"44":{"score":0.7998,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.9},"45":{"score":0.1377,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":19.1},"50":{"score":0.395,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":36.8},"56":{"score":0.6585,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":158.6},"57":{"score":0.5696,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":17.7},"58":{"score":0.7529,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":18.4},"59":{"score":0.78,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":35.8},"61":{"score":0.86,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":24.2},"62":{"score":0.8798,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":34.9},"64":{"score":0.6951,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":17.6}},"benchmarks":{"1":{"raw":0.9498,"skill":0.9323,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8103,"private_eq":0.8603},"2":{"raw":0.6638,"skill":0.6117,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6825,"private_eq":0.7273},"3":{"raw":0.8583,"skill":0.8556,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7596,"private_eq":0.7658},"4":{"raw":0.864,"skill":0.8623,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7929,"private_eq":0.7969},"5":{"raw":0.9227,"skill":0.9222,"coverage":1,"random":0.006,"tracks":[],"combined":0.6852,"private_eq":0.6897},"6":{"raw":0.7949,"skill":0.5191,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7727,"skill":0.7727,"coverage":1,"random":0,"tracks":[],"combined":0.5307,"private_eq":0.535},"10":{"raw":0.1523,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8155,"skill":0.7331,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6842,"skill":0.527,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4506,"private_eq":0.6367},"20":{"raw":0.9015,"skill":0.803,"coverage":1,"random":0.5,"tracks":[],"combined":0.842,"private_eq":0.9214},"21":{"raw":0.6142,"skill":0.2283,"coverage":1,"random":0.5,"tracks":[],"combined":0.3069,"private_eq":0.6569},"22":{"raw":0.3753,"skill":0.3704,"coverage":1,"random":0.0078,"tracks":[],"combined":0.2944,"private_eq":0.3069},"23":{"raw":0.6514,"skill":0.3027,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7691,"skill":0.6921,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3786,"combined":0.1629},"26":{"raw":0.9756,"skill":0.9675,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9377,"skill":0.9169,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8327,"skill":0.6654,"coverage":1,"random":0.5,"tracks":[],"combined":0.4714,"private_eq":0.7382},"29":{"raw":0.9281,"skill":0.9041,"coverage":1,"random":0.25,"tracks":[],"combined":0.8742,"private_eq":0.906},"30":{"raw":0.5428,"skill":0.4526,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6452,"random":0.25},{"track":"10-choice","raw":0.4405,"random":0.1}],"combined":0.5942,"private_eq":0.6682},"31":{"raw":0.175,"skill":0.1014,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0875,"private_eq":0.1709},"32":{"raw":0.5838,"skill":0.3383,"coverage":1,"random":0.371,"tracks":[],"combined":0.2119,"private_eq":0.5093},"33":{"raw":0.3461,"skill":0.3374,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3433,"private_eq":0.3583},"36":{"raw":0.4514,"skill":0.3793,"coverage":1,"random":0.116,"tracks":[],"combined":0.2995,"private_eq":0.3869},"37":{"raw":0.554,"skill":0.4406,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4372,"private_eq":0.5556},"38":{"raw":0.4239,"skill":0.4055,"coverage":1,"random":0.031,"tracks":[],"combined":0.5221,"private_eq":0.5412},"39":{"raw":0.9064,"skill":0.8623,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5422,"private_eq":0.6916},"40":{"raw":0.5551,"skill":0.4276,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3896,"headline":false},{"track":"A · English","score":0.5551,"headline":true},{"track":"C · Arabic pairs","score":0.85,"headline":false},{"track":"C · English pairs","score":0.935,"headline":false}],"combined":0.3915,"private_eq":0.5316},"41":{"raw":0.7787,"skill":0.6681,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4157,"private_eq":0.6142},"42":{"raw":0.7808,"skill":0.5735,"coverage":1,"random":0.486,"tracks":[],"combined":0.6279,"private_eq":0.8104},"43":{"raw":0.514,"skill":0.229,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4534,"private_eq":0.6588},"44":{"raw":0.7998,"skill":0.5996,"coverage":1,"random":0.5,"tracks":[],"combined":0.5406,"private_eq":0.7724},"45":{"raw":0.1377,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.395,"skill":0.1219,"coverage":1,"random":0.311,"tracks":[],"combined":0.2395,"private_eq":0.4813},"56":{"raw":0.6585,"skill":0.317,"coverage":1,"random":0.5,"tracks":[],"combined":0.6549,"private_eq":0.8289},"57":{"raw":0.5696,"skill":0.5159,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5928,"combined":0.5377},"58":{"raw":0.7529,"skill":0.6418,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4001,"combined":0.1214},"59":{"raw":0.78,"skill":0.5439,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2919,"private_eq":0.6619},"61":{"raw":0.86,"skill":0.72,"coverage":1,"random":0.5,"tracks":[],"combined":0.5699,"private_eq":0.7869},"62":{"raw":0.8798,"skill":0.8397,"coverage":1,"random":0.25,"tracks":[],"combined":0.6384,"private_eq":0.7311},"64":{"raw":0.6951,"skill":0.6189,"coverage":1,"random":0.2,"tracks":[],"combined":0.6131,"private_eq":0.6932}},"coverage":1,"pending":0,"notes":["New in the 2026-10-21 PR batch (PR #258).","Submitted public 0.3 run adopted (kit rescore 57.56, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (INT8 W8A8); decision agreement with the submitted rows 97.28% on 831 rows (bar 96%).","Served in INT8 (W8A8) as in the author's run, so the quantized agreement bar applies.","Runs with the author's compiled serving runtime; evaluated by the lab in a sandbox."],"scores":{"balanced_raw":68.47,"balanced_skill":47.7,"breadth_skill":55.85,"public_skill":57.56},"frozen_scores":{"balanced_raw":68.47,"balanced_skill":57.56,"breadth_skill":55.85},"categories":[{"id":"knowledge","raw":0.5266,"skill":0.3878,"combined":0.3503,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.742,"skill":0.631,"combined":0.5215,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7209,"skill":0.6173,"combined":0.575,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8307,"skill":0.8092,"combined":0.6917,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6054,"skill":0.4075,"combined":0.4592,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"mica-v0.5-4b","pub":57.56,"same_skills":52.25,"new_domains":31.5,"v":47.7,"lab":false,"jev":false,"rank_v":98,"rank_pub":86,"name":"Mica v0.5 4B","eq":{"S":51.81,"O":41.55},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":31.08,"new_domains":16.62}},"new_in_edition":true},{"engine":"jeffan-4b-2010","name":"Jeffan-4B-2010","long_name":"Jeffan-4B-2010","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4205751296,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tattrongvu/Jeffan-4B-2010","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"pr":259,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7854,"conf":0.8243,"ece":0.0389,"brier":0.2912,"over95":0.0171,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0016,"conf":0.0871,"acc":0.0498},{"lo":0.1,"w":0.0115,"conf":0.1567,"acc":0.1054},{"lo":0.2,"w":0.0229,"conf":0.2538,"acc":0.1932},{"lo":0.3,"w":0.0356,"conf":0.3525,"acc":0.2695},{"lo":0.4,"w":0.0376,"conf":0.4525,"acc":0.4222},{"lo":0.5,"w":0.0787,"conf":0.5488,"acc":0.5276},{"lo":0.6,"w":0.0753,"conf":0.6511,"acc":0.5814},{"lo":0.7,"w":0.0695,"conf":0.7524,"acc":0.7136},{"lo":0.8,"w":0.0924,"conf":0.8524,"acc":0.8023},{"lo":0.9,"w":0.5747,"conf":0.9806,"acc":0.9484}],"areas":{"knowledge":{"ece":0.0524,"acc":0.6804,"conf":0.7328},"language":{"ece":0.013,"acc":0.8838,"conf":0.8968},"retrieval":{"ece":0.0873,"acc":0.7944,"conf":0.8804},"tools":{"ece":0.0441,"acc":0.9453,"conf":0.9848},"arts":{"ece":0.0332,"acc":0.7151,"conf":0.7354}}},"gaps":null,"latency":{"median":50.4,"p95":236.1,"mean":87.4},"results":{"1":{"score":0.964,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":41.7},"2":{"score":0.7367,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":329.9},"3":{"score":0.8602,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":127.2},"4":{"score":0.9141,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":37.6},"5":{"score":0.9526,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":39.7},"6":{"score":0.7886,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":77.3},"9":{"score":0.9773,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":322.5},"10":{"score":0.6515,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":37.7},"11":{"score":0.8405,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":513.8},"12":{"score":0.7579,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":36.1},"20":{"score":0.9042,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36},"21":{"score":0.6583,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":35.9},"22":{"score":0.76,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":109.6},"23":{"score":0.622,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.3},"24":{"score":0.781,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":35.8},"25":{"score":0.4592,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":35.9},"26":{"score":0.9878,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":35.6},"27":{"score":0.9522,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":35.6},"28":{"score":0.8871,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":35.6},"29":{"score":0.9502,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":35.9},"30":{"score":0.6107,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1836,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":37.3},"32":{"score":0.8444,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":37.7},"33":{"score":0.4261,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":77.2},"36":{"score":0.4487,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":590.9},"37":{"score":0.5998,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":36.7},"38":{"score":0.5883,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":224.1},"39":{"score":0.9321,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":36.5},"40":{"score":0.5437,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":36.3},"41":{"score":0.7915,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":36},"42":{"score":0.8489,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":36.8},"43":{"score":0.593,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":36},"44":{"score":0.9256,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.9},"45":{"score":0.1038,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":36.5},"50":{"score":0.6271,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":37.4},"56":{"score":0.59,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":65.5},"57":{"score":0.6253,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":36.1},"58":{"score":0.8651,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":36},"59":{"score":0.8185,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":37.2},"61":{"score":0.887,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":36.6},"62":{"score":0.9069,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":37.2},"64":{"score":0.7538,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":36.2}},"benchmarks":{"1":{"raw":0.964,"skill":0.9514,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.7367,"skill":0.6959,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7042,"private_eq":0.7459},"3":{"raw":0.8602,"skill":0.8575,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.9141,"skill":0.913,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8558,"private_eq":0.8583},"5":{"raw":0.9526,"skill":0.9523,"coverage":1,"random":0.006,"tracks":[],"combined":0.8733,"private_eq":0.8745},"6":{"raw":0.7886,"skill":0.5043,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9773,"skill":0.9773,"coverage":1,"random":0,"tracks":[],"combined":0.8771,"private_eq":0.8775},"10":{"raw":0.6515,"skill":0.4201,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8405,"skill":0.7693,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7579,"skill":0.6374,"coverage":1,"random":0.3324,"tracks":[],"combined":0.516,"private_eq":0.6799},"20":{"raw":0.906,"skill":0.8121,"coverage":1,"random":0.5,"tracks":[],"combined":0.7987,"private_eq":0.9},"21":{"raw":0.6583,"skill":0.3166,"coverage":1,"random":0.5,"tracks":[],"combined":0.3975,"private_eq":0.7017},"22":{"raw":0.7556,"skill":0.7537,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6066,"private_eq":0.6131},"23":{"raw":0.6219,"skill":0.2439,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.781,"skill":0.708,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4592,"skill":0.2789,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5376,"combined":0.3774},"26":{"raw":0.9878,"skill":0.9837,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9522,"skill":0.9362,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8871,"skill":0.7742,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"29":{"raw":0.9502,"skill":0.9336,"coverage":1,"random":0.25,"tracks":[],"combined":0.8742,"private_eq":0.906},"30":{"raw":0.6107,"skill":0.5341,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6998,"random":0.25},{"track":"10-choice","raw":0.5216,"random":0.1}],"combined":0.561,"private_eq":0.6411},"31":{"raw":0.1836,"skill":0.1108,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1135,"private_eq":0.1945},"32":{"raw":0.8444,"skill":0.7526,"coverage":1,"random":0.371,"tracks":[],"combined":0.328,"private_eq":0.5815},"33":{"raw":0.4261,"skill":0.4185,"coverage":1,"random":0.0131,"tracks":[],"combined":0.364,"private_eq":0.3785},"36":{"raw":0.4487,"skill":0.3763,"coverage":1,"random":0.116,"tracks":[],"combined":0.374,"private_eq":0.452},"37":{"raw":0.5998,"skill":0.498,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5788,"private_eq":0.6672},"38":{"raw":0.5883,"skill":0.5751,"coverage":1,"random":0.031,"tracks":[],"combined":0.6705,"private_eq":0.6834},"39":{"raw":0.9321,"skill":0.9001,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6425,"private_eq":0.759},"40":{"raw":0.5437,"skill":0.413,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5494,"headline":false},{"track":"A · English","score":0.5437,"headline":true},{"track":"C · Arabic pairs","score":0.895,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"combined":0.4435,"private_eq":0.5716},"41":{"raw":0.7915,"skill":0.6873,"coverage":1,"random":0.3333,"tracks":[],"combined":0.0758,"private_eq":0.3902},"42":{"raw":0.8489,"skill":0.706,"coverage":1,"random":0.486,"tracks":[],"combined":0.4327,"private_eq":0.7112},"43":{"raw":0.593,"skill":0.3542,"coverage":1,"random":0.3697,"tracks":[],"combined":0.491,"private_eq":0.6822},"44":{"raw":0.9256,"skill":0.8512,"coverage":1,"random":0.5,"tracks":[],"combined":0.8137,"private_eq":0.9074},"45":{"raw":0.1038,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6271,"skill":0.4588,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.59,"skill":0.18,"coverage":1,"random":0.5,"tracks":[],"combined":0.3087,"private_eq":0.6578},"57":{"raw":0.6253,"skill":0.5786,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6651,"combined":0.62},"58":{"raw":0.8651,"skill":0.8045,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4844,"combined":0.245},"59":{"raw":0.8185,"skill":0.6237,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3255,"private_eq":0.6779},"61":{"raw":0.887,"skill":0.774,"coverage":1,"random":0.5,"tracks":[],"combined":0.6152,"private_eq":0.8093},"62":{"raw":0.9069,"skill":0.8759,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.7538,"skill":0.6923,"coverage":1,"random":0.2,"tracks":[],"combined":0.6422,"private_eq":0.7162}},"coverage":1,"pending":0,"notes":["New in the 2026-10-22 PR batch (PR #259).","Submitted public 0.3 run adopted (kit rescore 64.85, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.80% on 831 rows (bar 98.5%).","Full-attention layers read the prompt bidirectionally at inference; decision-only, not a generative model.","Training data undisclosed."],"scores":{"balanced_raw":73.92,"balanced_skill":52.21,"breadth_skill":63.76,"public_skill":64.85},"frozen_scores":{"balanced_raw":73.92,"balanced_skill":64.85,"breadth_skill":63.76},"categories":[{"id":"knowledge","raw":0.5927,"skill":0.4954,"combined":0.4562,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7886,"skill":0.6978,"combined":0.507,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7356,"skill":0.6276,"combined":0.6101,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8907,"skill":0.874,"combined":0.7981,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7204,"skill":0.5462,"combined":0.5383,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jeffan-4b-2010","pub":64.85,"same_skills":58.03,"new_domains":33.52,"v":52.21,"lab":false,"jev":false,"rank_v":88,"rank_pub":41,"name":"Jeffan-4B-2010","eq":{"S":57.65,"O":44.04},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":34.59,"new_domains":17.62}},"new_in_edition":true},{"engine":"tasksource-decider-nano-v5","name":"tasksource Decider nano v5 (150M)","long_name":"tasksource Decider nano v5 (150M)","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":149015041,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/tasksource/tasksource-decider-nano","code_url":null,"base_model":"cross-encoder/ettin-reranker-150m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-150m-v1","weights_repo":null,"served_checkpoint":null,"org":{"id":"tasksource","name":"tasksource","logo":"data/orgs/tasksource.png"},"pr":263,"replaces":"tasksource-decider-nano-v4","new":true},"completed":110201,"counts":{"ok":110195,"unsupported":6,"error":0},"lifted":false,"calibration":{"acc":0.5792,"conf":0.6492,"ece":0.07,"brier":0.5118,"over95":0.0123,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0507,"conf":0.0709,"acc":0.0345},{"lo":0.1,"w":0.021,"conf":0.1356,"acc":0.089},{"lo":0.2,"w":0.0356,"conf":0.2591,"acc":0.2085},{"lo":0.3,"w":0.0711,"conf":0.3527,"acc":0.2697},{"lo":0.4,"w":0.0755,"conf":0.4506,"acc":0.3636},{"lo":0.5,"w":0.1927,"conf":0.5449,"acc":0.5311},{"lo":0.6,"w":0.1179,"conf":0.646,"acc":0.5811},{"lo":0.7,"w":0.0924,"conf":0.7478,"acc":0.6302},{"lo":0.8,"w":0.0864,"conf":0.851,"acc":0.6801},{"lo":0.9,"w":0.2568,"conf":0.9762,"acc":0.9103}],"areas":{"knowledge":{"ece":0.0931,"acc":0.4349,"conf":0.528},"language":{"ece":0.0601,"acc":0.7254,"conf":0.7849},"retrieval":{"ece":0.0696,"acc":0.7505,"conf":0.8183},"tools":{"ece":0.083,"acc":0.6728,"conf":0.7291},"arts":{"ece":0.0525,"acc":0.4116,"conf":0.4641}}},"gaps":null,"latency":{"median":13,"p95":21.5,"mean":15.3},"results":{"1":{"score":0.8896,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":24.1},"2":{"score":0.6272,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":159},"3":{"score":0.376,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":171.5},"4":{"score":0.8941,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":20.5},"5":{"score":0.872,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":37.4},"6":{"score":0.6601,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":39.4},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":47.9},"10":{"score":0.6057,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":24.1},"11":{"score":0.7875,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":1,"errors":0,"median_ms":127.1},"12":{"score":0.4542,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":15.5},"20":{"score":0.6688,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22},"21":{"score":0.4867,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":21.5},"22":{"score":0.0145,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":131.3},"23":{"score":0.515,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.3},"24":{"score":0.3782,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":22},"25":{"score":0.2143,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":22.4},"26":{"score":0.7163,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":21.9},"27":{"score":0.5145,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":22},"28":{"score":0.5935,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":21.6},"29":{"score":0.6452,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":22.7},"30":{"score":0.257,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0996,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.7},"32":{"score":0.5851,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":26},"33":{"score":0.1097,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":35.3},"36":{"score":0.2925,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":309.6},"37":{"score":0.4033,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.9},"38":{"score":0.2918,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":116.3},"39":{"score":0.7157,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":16.4},"40":{"score":0.2993,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":22.5},"41":{"score":0.7501,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":22.6},"42":{"score":0.5911,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":23.8},"43":{"score":0.4298,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":22.2},"44":{"score":0.5284,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22},"45":{"score":0.1437,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":16},"50":{"score":0.4039,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":17.4},"56":{"score":0.5075,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":29.8},"57":{"score":0.1733,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":23.4},"58":{"score":0.3931,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":22.9},"59":{"score":0.676,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":24.7},"61":{"score":0.8425,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":23.3},"62":{"score":0.7968,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":24.8},"64":{"score":0.5947,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":22.7}},"benchmarks":{"1":{"raw":0.8896,"skill":0.851,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6741,"private_eq":0.7606},"2":{"raw":0.6227,"skill":0.5642,"coverage":0.9927,"random":0.1341,"tracks":[],"combined":0.6482,"private_eq":0.698},"3":{"raw":0.376,"skill":0.364,"coverage":1,"random":0.0189,"tracks":[],"combined":0.261,"private_eq":0.2822},"4":{"raw":0.8941,"skill":0.8927,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7233,"private_eq":0.7289},"5":{"raw":0.872,"skill":0.8712,"coverage":1,"random":0.006,"tracks":[],"combined":0.5559,"private_eq":0.5626},"6":{"raw":0.6601,"skill":0.203,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.6057,"skill":0.3439,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7747,"skill":0.6742,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.4542,"skill":0.1824,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2265,"private_eq":0.4888},"20":{"raw":0.6574,"skill":0.3148,"coverage":1,"random":0.5,"tracks":[],"combined":0.4878,"private_eq":0.7463},"21":{"raw":0.4867,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.014,"skill":0.0063,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5244,"skill":0.0489,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3782,"skill":0.1709,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2143,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2664,"combined":0.0116},"26":{"raw":0.7163,"skill":0.6216,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5145,"skill":0.3525,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5935,"skill":0.187,"coverage":1,"random":0.5,"tracks":[],"combined":0.2191,"private_eq":0.6135},"29":{"raw":0.6452,"skill":0.5269,"coverage":1,"random":0.25,"tracks":[],"combined":0.399,"private_eq":0.5536},"30":{"raw":0.257,"skill":0.1023,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3503,"random":0.25},{"track":"10-choice","raw":0.1638,"random":0.1}],"combined":0.0889,"private_eq":0.2561},"31":{"raw":0.0996,"skill":0.0193,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0074,"private_eq":0.0982},"32":{"raw":0.5851,"skill":0.3404,"coverage":1,"random":0.371,"tracks":[],"combined":0.0107,"private_eq":0.3842},"33":{"raw":0.1097,"skill":0.0979,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1533,"private_eq":0.1729},"36":{"raw":0.2925,"skill":0.1997,"coverage":1,"random":0.116,"tracks":[],"combined":0.2032,"private_eq":0.3028},"37":{"raw":0.4033,"skill":0.2516,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4503,"private_eq":0.5659},"38":{"raw":0.2918,"skill":0.2691,"coverage":1,"random":0.031,"tracks":[],"combined":0.3293,"private_eq":0.3565},"39":{"raw":0.7157,"skill":0.5819,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5477,"private_eq":0.6953},"40":{"raw":0.2993,"skill":0.0985,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.2993,"headline":true},{"track":"C · Arabic pairs","score":0.465,"headline":false},{"track":"C · English pairs","score":0.375,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7501,"skill":0.6251,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5911,"skill":0.2045,"coverage":1,"random":0.486,"tracks":[],"combined":0.1511,"private_eq":0.5681},"43":{"raw":0.4298,"skill":0.0954,"coverage":1,"random":0.3697,"tracks":[],"combined":0.006,"private_eq":0.38},"44":{"raw":0.5284,"skill":0.0568,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5024},"45":{"raw":0.1437,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4039,"skill":0.1349,"coverage":1,"random":0.311,"tracks":[],"combined":0.1523,"private_eq":0.4219},"56":{"raw":0.5075,"skill":0.015,"coverage":1,"random":0.5,"tracks":[],"combined":0.0492,"private_eq":0.5295},"57":{"raw":0.1733,"skill":0.0702,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3931,"skill":0.1203,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3399,"combined":0.0332},"59":{"raw":0.676,"skill":0.3282,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8425,"skill":0.685,"coverage":1,"random":0.5,"tracks":[],"combined":0.5244,"private_eq":0.7644},"62":{"raw":0.7968,"skill":0.7291,"coverage":1,"random":0.25,"tracks":[],"combined":0.3234,"private_eq":0.4976},"64":{"raw":0.5947,"skill":0.4934,"coverage":1,"random":0.2,"tracks":[],"combined":0.584,"private_eq":0.6702}},"coverage":0.9995,"pending":0.0005,"notes":["New in the 2026-10-22 PR batch (PR #263).","Replaces the earlier board entry tasksource-decider-nano-v4 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 33.51, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.29% on 827 rows (bar 98.5%).","Questions over its declared 8,192-token context count as unsupported.","Training uses train splits of public datasets behind the index benchmarks (author-declared)."],"scores":{"balanced_raw":50.38,"balanced_skill":19.85,"breadth_skill":29.56,"public_skill":33.51},"frozen_scores":{"balanced_raw":50.38,"balanced_skill":33.51,"breadth_skill":29.56},"categories":[{"id":"knowledge","raw":0.3193,"skill":0.0971,"combined":0.0523,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5764,"skill":0.3865,"combined":0.208,"coverage":0.9991,"pending":0,"n":9},{"id":"retrieval","raw":0.6399,"skill":0.5012,"combined":0.4247,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5441,"skill":0.5095,"combined":0.3858,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.4468,"skill":0.1664,"combined":0.2406,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"tasksource-decider-nano-v5","pub":33.51,"same_skills":26.1,"new_domains":7.25,"v":19.85,"lab":false,"jev":false,"rank_v":197,"rank_pub":177,"name":"tasksource Decider nano v5 (150M)","eq":{"S":25.35,"O":11.6},"rank_tie":46,"tied":true,"parts":{"public":0,"same_skills":15.21,"new_domains":4.64}},"new_in_edition":true},{"engine":"senova-4b","name":"Senova-4B","long_name":"Senova-4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4539265536,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/WangSenQWQ/Senova-4B","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"pr":264,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7535,"conf":0.9056,"ece":0.1521,"brier":0.389,"over95":0.088,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.001,"conf":0.0869,"acc":0.0769},{"lo":0.1,"w":0.0129,"conf":0.1491,"acc":0.0579},{"lo":0.2,"w":0.0094,"conf":0.245,"acc":0.0625},{"lo":0.3,"w":0.0117,"conf":0.3563,"acc":0.126},{"lo":0.4,"w":0.0196,"conf":0.456,"acc":0.2642},{"lo":0.5,"w":0.0355,"conf":0.5518,"acc":0.4018},{"lo":0.6,"w":0.0361,"conf":0.6505,"acc":0.4997},{"lo":0.7,"w":0.0517,"conf":0.7503,"acc":0.4966},{"lo":0.8,"w":0.0566,"conf":0.8525,"acc":0.5756},{"lo":0.9,"w":0.7655,"conf":0.9903,"acc":0.8555}],"areas":{"knowledge":{"ece":0.2119,"acc":0.6224,"conf":0.8343},"language":{"ece":0.0867,"acc":0.8561,"conf":0.9422},"retrieval":{"ece":0.2398,"acc":0.7311,"conf":0.9707},"tools":{"ece":0.06,"acc":0.9263,"conf":0.9863},"arts":{"ece":0.1368,"acc":0.7452,"conf":0.882}}},"gaps":null,"latency":{"median":48.3,"p95":346.8,"mean":156.1},"results":{"1":{"score":0.9132,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":356.4},"2":{"score":0.7077,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3723},"3":{"score":0.8189,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":137.5},"4":{"score":0.937,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":113.5},"5":{"score":0.9528,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":114.6},"6":{"score":0.7988,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":240.7},"9":{"score":0.9773,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2088.9},"10":{"score":0.4463,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":119.6},"11":{"score":0.8572,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2049.1},"12":{"score":0.7141,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":114.6},"20":{"score":0.9802,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":114.5},"21":{"score":0.6556,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":116.4},"22":{"score":0.8465,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":128.2},"23":{"score":0.6624,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.4},"24":{"score":0.7576,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":115.3},"25":{"score":0.4031,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":108.3},"26":{"score":0.9802,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":115.9},"27":{"score":0.936,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":113.2},"28":{"score":0.9029,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":116.3},"29":{"score":0.9558,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":115.4},"30":{"score":0.5883,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1008,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":116.3},"32":{"score":0.5625,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":118.6},"33":{"score":0.1152,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":865.1},"36":{"score":0.4662,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3691.8},"37":{"score":0.5861,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":115.1},"38":{"score":0.3756,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":7035.1},"39":{"score":0.8848,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":209.6},"40":{"score":0.5588,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":120.7},"41":{"score":0.7705,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":112.1},"42":{"score":0.8008,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":113.5},"43":{"score":0.5298,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":112.3},"44":{"score":0.942,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":108.3},"45":{"score":0.1317,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":109.6},"50":{"score":0.5865,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":110.6},"56":{"score":0.7185,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":844.1},"57":{"score":0.5482,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":106.6},"58":{"score":0.7672,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":103.4},"59":{"score":0.8129,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":102.9},"61":{"score":0.8825,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":98.4},"62":{"score":0.9168,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":102.5},"64":{"score":0.7083,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":100.2}},"benchmarks":{"1":{"raw":0.9132,"skill":0.8829,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8103,"private_eq":0.8603},"2":{"raw":0.7077,"skill":0.6624,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6656,"private_eq":0.7129},"3":{"raw":0.8189,"skill":0.8154,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8068,"private_eq":0.8116},"4":{"raw":0.937,"skill":0.9362,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8373,"private_eq":0.8402},"5":{"raw":0.9528,"skill":0.9525,"coverage":1,"random":0.006,"tracks":[],"combined":0.7092,"private_eq":0.7132},"6":{"raw":0.7988,"skill":0.5283,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9773,"skill":0.9773,"coverage":1,"random":0,"tracks":[],"combined":0.5888,"private_eq":0.5925},"10":{"raw":0.4463,"skill":0.0787,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8572,"skill":0.7935,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7141,"skill":0.5717,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4631,"private_eq":0.645},"20":{"raw":0.9806,"skill":0.9611,"coverage":1,"random":0.5,"tracks":[],"combined":0.8853,"private_eq":0.9428},"21":{"raw":0.6556,"skill":0.3113,"coverage":1,"random":0.5,"tracks":[],"combined":0.2064,"private_eq":0.6072},"22":{"raw":0.8479,"skill":0.8467,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4683,"private_eq":0.4775},"23":{"raw":0.6818,"skill":0.3635,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7576,"skill":0.6768,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4031,"skill":0.2041,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3901,"combined":0.1784},"26":{"raw":0.9802,"skill":0.9736,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.936,"skill":0.9146,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9029,"skill":0.8058,"coverage":1,"random":0.5,"tracks":[],"combined":0.4663,"private_eq":0.7357},"29":{"raw":0.9558,"skill":0.9411,"coverage":1,"random":0.25,"tracks":[],"combined":0.9929,"private_eq":0.994},"30":{"raw":0.5883,"skill":0.5058,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6687,"random":0.25},{"track":"10-choice","raw":0.508,"random":0.1}],"combined":0.5458,"private_eq":0.6287},"31":{"raw":0.1008,"skill":0.0206,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0387,"private_eq":0.1266},"32":{"raw":0.5625,"skill":0.3045,"coverage":1,"random":0.371,"tracks":[],"combined":0.0649,"private_eq":0.4179},"33":{"raw":0.1152,"skill":0.1035,"coverage":1,"random":0.0131,"tracks":[],"combined":0.0042,"private_eq":0.0275},"36":{"raw":0.4662,"skill":0.3961,"coverage":1,"random":0.116,"tracks":[],"combined":0.3516,"private_eq":0.4325},"37":{"raw":0.5861,"skill":0.4808,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5331,"private_eq":0.6312},"38":{"raw":0.3756,"skill":0.3556,"coverage":1,"random":0.031,"tracks":[],"combined":0.3752,"private_eq":0.4005},"39":{"raw":0.8848,"skill":0.8306,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5578,"private_eq":0.7021},"40":{"raw":0.5588,"skill":0.4324,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4412,"headline":false},{"track":"A · English","score":0.5588,"headline":true},{"track":"C · Arabic pairs","score":0.695,"headline":false},{"track":"C · English pairs","score":0.915,"headline":false}],"combined":0.243,"private_eq":0.4175},"41":{"raw":0.7705,"skill":0.6557,"coverage":1,"random":0.3333,"tracks":[],"combined":0.335,"private_eq":0.561},"42":{"raw":0.8008,"skill":0.6125,"coverage":1,"random":0.486,"tracks":[],"combined":0.6472,"private_eq":0.8202},"43":{"raw":0.5298,"skill":0.254,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4241,"private_eq":0.6405},"44":{"raw":0.942,"skill":0.884,"coverage":1,"random":0.5,"tracks":[],"combined":0.803,"private_eq":0.9021},"45":{"raw":0.1317,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5865,"skill":0.3999,"coverage":1,"random":0.311,"tracks":[],"combined":0.174,"private_eq":0.4367},"56":{"raw":0.7185,"skill":0.437,"coverage":1,"random":0.5,"tracks":[],"combined":0.2654,"private_eq":0.6364},"57":{"raw":0.5482,"skill":0.4918,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.373,"combined":0.2877},"58":{"raw":0.7672,"skill":0.6626,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4044,"combined":0.1278},"59":{"raw":0.8129,"skill":0.6121,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1732,"private_eq":0.6053},"61":{"raw":0.8825,"skill":0.765,"coverage":1,"random":0.5,"tracks":[],"combined":0.6266,"private_eq":0.8149},"62":{"raw":0.9168,"skill":0.8891,"coverage":1,"random":0.25,"tracks":[],"combined":0.5835,"private_eq":0.6904},"64":{"raw":0.7083,"skill":0.6354,"coverage":1,"random":0.2,"tracks":[],"combined":0.6228,"private_eq":0.7009}},"coverage":1,"pending":0,"notes":["New in the 2026-10-22 PR batch (PR #264).","Submitted public 0.3 run adopted (kit rescore 61.60, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.70% on 831 rows (bar 98.5%).","RAGTruth-shaped questions are rewritten into the model's training layout by the author's serving code."],"scores":{"balanced_raw":71.15,"balanced_skill":47.32,"breadth_skill":59.84,"public_skill":61.6},"frozen_scores":{"balanced_raw":71.15,"balanced_skill":61.6,"breadth_skill":59.84},"categories":[{"id":"knowledge","raw":0.5119,"skill":0.3891,"combined":0.2919,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7515,"skill":0.6498,"combined":0.4855,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7597,"skill":0.6704,"combined":0.561,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8667,"skill":0.8457,"combined":0.6997,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7434,"skill":0.5863,"combined":0.4714,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"senova-4b","pub":61.6,"same_skills":49.89,"new_domains":33.64,"v":47.32,"lab":false,"jev":false,"rank_v":101,"rank_pub":53,"name":"Senova-4B","eq":{"S":49.42,"O":44.19},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":29.65,"new_domains":17.67}},"new_in_edition":true},{"engine":"standard-one-3b-sh-v4","name":"Standard One 3B SH (v4)","long_name":"Standard One 3B SH (v4)","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":3849090048,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/StandardThinking/StandardOne-3B-SH","code_url":null,"base_model":"StandardThinking/StandardOne-3B","base_url":"https://huggingface.co/StandardThinking/StandardOne-3B","weights_repo":null,"served_checkpoint":null,"org":{"id":"StandardThinking","name":"Standard Thinking","logo":"data/orgs/StandardThinking.png"},"pr":265,"replaces":"standard-one-3b-sh-v2","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7223,"conf":0.8354,"ece":0.1136,"brier":0.4028,"over95":0.0505,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0062,"conf":0.08,"acc":0.1006},{"lo":0.1,"w":0.0183,"conf":0.148,"acc":0.1569},{"lo":0.2,"w":0.0103,"conf":0.2482,"acc":0.1865},{"lo":0.3,"w":0.0176,"conf":0.3559,"acc":0.3216},{"lo":0.4,"w":0.0371,"conf":0.4526,"acc":0.3648},{"lo":0.5,"w":0.0694,"conf":0.5495,"acc":0.4741},{"lo":0.6,"w":0.0752,"conf":0.6499,"acc":0.4773},{"lo":0.7,"w":0.068,"conf":0.7491,"acc":0.5721},{"lo":0.8,"w":0.0897,"conf":0.8538,"acc":0.641},{"lo":0.9,"w":0.6082,"conf":0.9734,"acc":0.8756}],"areas":{"knowledge":{"ece":0.1308,"acc":0.6019,"conf":0.7308},"language":{"ece":0.0899,"acc":0.8254,"conf":0.9148},"retrieval":{"ece":0.2414,"acc":0.6651,"conf":0.9036},"tools":{"ece":0.0352,"acc":0.9549,"conf":0.9811},"arts":{"ece":0.0789,"acc":0.6839,"conf":0.7595}}},"gaps":null,"latency":{"median":79.9,"p95":139.9,"mean":102},"results":{"1":{"score":0.9634,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":276.8},"2":{"score":0.6586,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":19683.7},"3":{"score":0.8799,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1921.4},"4":{"score":0.9086,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":215.8},"5":{"score":0.9527,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":285.6},"6":{"score":0.7829,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":424.8},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":411.1},"10":{"score":0.1641,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":305.8},"11":{"score":0.8559,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":363.7},"12":{"score":0.6266,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":138.8},"20":{"score":0.9218,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":129},"21":{"score":0.5925,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":121},"22":{"score":0.6905,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1514},"23":{"score":0.606,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":174.9},"24":{"score":0.6805,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":126.9},"25":{"score":0.2959,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":122.9},"26":{"score":0.9478,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":119.9},"27":{"score":0.8712,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":121.7},"28":{"score":0.8129,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":116.7},"29":{"score":0.9039,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":135.8},"30":{"score":0.5106,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.213,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":222.5},"32":{"score":0.6995,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":300.9},"33":{"score":0.3333,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":286.8},"36":{"score":0.4351,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5623.1},"37":{"score":0.5691,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":157.9},"38":{"score":0.4078,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2251.4},"39":{"score":0.8837,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":156.8},"40":{"score":0.4872,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":120.9},"41":{"score":0.6156,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":140},"42":{"score":0.7706,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":175.9},"43":{"score":0.4702,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":136.9},"44":{"score":0.8828,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":138.2},"45":{"score":0.0978,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":165.9},"50":{"score":0.5973,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":196.9},"56":{"score":0.8475,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":203.8},"57":{"score":0.4117,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":131.9},"58":{"score":0.767,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":129.9},"59":{"score":0.7619,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":196.8},"61":{"score":0.8005,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":157.1},"62":{"score":0.8729,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":215.8},"64":{"score":0.608,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":172.9}},"benchmarks":{"1":{"raw":0.9634,"skill":0.9506,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.6586,"skill":0.6058,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6478,"private_eq":0.6976},"3":{"raw":0.8799,"skill":0.8776,"coverage":1,"random":0.0189,"tracks":[],"combined":0.812,"private_eq":0.8167},"4":{"raw":0.9086,"skill":0.9074,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7929,"private_eq":0.7969},"5":{"raw":0.9527,"skill":0.9524,"coverage":1,"random":0.006,"tracks":[],"combined":0.6619,"private_eq":0.6668},"6":{"raw":0.7829,"skill":0.491,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.7583,"private_eq":0.76},"10":{"raw":0.1641,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8559,"skill":0.7916,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6266,"skill":0.4407,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3946,"private_eq":0.5998},"20":{"raw":0.9202,"skill":0.8404,"coverage":1,"random":0.5,"tracks":[],"combined":0.8637,"private_eq":0.9321},"21":{"raw":0.5925,"skill":0.1849,"coverage":1,"random":0.5,"tracks":[],"combined":0.0656,"private_eq":0.5376},"22":{"raw":0.6743,"skill":0.6717,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4976,"private_eq":0.5062},"23":{"raw":0.6128,"skill":0.2257,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6805,"skill":0.574,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2959,"skill":0.0612,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3194,"combined":0.0831},"26":{"raw":0.9478,"skill":0.9304,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8712,"skill":0.8282,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8129,"skill":0.6258,"coverage":1,"random":0.5,"tracks":[],"combined":0.4967,"private_eq":0.7507},"29":{"raw":0.9039,"skill":0.8719,"coverage":1,"random":0.25,"tracks":[],"combined":0.7901,"private_eq":0.8436},"30":{"raw":0.5106,"skill":0.4118,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5997,"random":0.25},{"track":"10-choice","raw":0.4215,"random":0.1}],"combined":0.4739,"private_eq":0.5701},"31":{"raw":0.213,"skill":0.1428,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0983,"private_eq":0.1807},"32":{"raw":0.6995,"skill":0.5223,"coverage":1,"random":0.371,"tracks":[],"combined":0.181,"private_eq":0.4901},"33":{"raw":0.3333,"skill":0.3245,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1071,"private_eq":0.1278},"36":{"raw":0.4351,"skill":0.361,"coverage":1,"random":0.116,"tracks":[],"combined":0.2794,"private_eq":0.3694},"37":{"raw":0.5691,"skill":0.4595,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5603,"private_eq":0.6526},"38":{"raw":0.4078,"skill":0.3889,"coverage":1,"random":0.031,"tracks":[],"combined":0.4266,"private_eq":0.4497},"39":{"raw":0.8837,"skill":0.8289,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5876,"private_eq":0.7221},"40":{"raw":0.4872,"skill":0.3403,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1147,"headline":false},{"track":"A · English","score":0.4872,"headline":true},{"track":"C · Arabic pairs","score":0.675,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"combined":0.3001,"private_eq":0.4614},"41":{"raw":0.6156,"skill":0.4234,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4183,"private_eq":0.6159},"42":{"raw":0.7706,"skill":0.5537,"coverage":1,"random":0.486,"tracks":[],"combined":0.768,"private_eq":0.8816},"43":{"raw":0.4702,"skill":0.1594,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3907,"private_eq":0.6197},"44":{"raw":0.8828,"skill":0.7656,"coverage":1,"random":0.5,"tracks":[],"combined":0.7975,"private_eq":0.8994},"45":{"raw":0.0978,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5973,"skill":0.4155,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.8475,"skill":0.695,"coverage":1,"random":0.5,"tracks":[],"combined":0.4386,"private_eq":0.722},"57":{"raw":0.4117,"skill":0.3383,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3584,"combined":0.2711},"58":{"raw":0.767,"skill":0.6623,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3896,"combined":0.1061},"59":{"raw":0.7619,"skill":0.5063,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.8005,"skill":0.601,"coverage":1,"random":0.5,"tracks":[],"combined":0.5244,"private_eq":0.7644},"62":{"raw":0.8729,"skill":0.8305,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"64":{"raw":0.608,"skill":0.51,"coverage":1,"random":0.2,"tracks":[],"combined":0.4287,"private_eq":0.5474}},"coverage":1,"pending":0,"notes":["New in the 2026-10-22 PR batch (PR #265).","Replaces the earlier board entry standard-one-3b-sh-v2 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 58.08, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.50% on 830 rows (bar 98.5%).","Training uses the author's request-format conversions of train splits of public datasets, some of which are sources of index benchmarks (author-declared)."],"scores":{"balanced_raw":68.88,"balanced_skill":39.79,"breadth_skill":55.97,"public_skill":58.08},"frozen_scores":{"balanced_raw":68.88,"balanced_skill":58.08,"breadth_skill":55.97},"categories":[{"id":"knowledge","raw":0.4976,"skill":0.3626,"combined":0.296,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7042,"skill":0.5753,"combined":0.4657,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7535,"skill":0.6698,"combined":0.5461,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8784,"skill":0.8574,"combined":0.7516,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6675,"skill":0.4747,"combined":0.4248,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"standard-one-3b-sh-v4","pub":58.08,"same_skills":49.79,"new_domains":18.52,"v":39.79,"lab":false,"jev":false,"rank_v":136,"rank_pub":78,"name":"Standard One 3B SH (v4)","eq":{"S":49.31,"O":25.51},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":29.58,"new_domains":10.2}},"new_in_edition":true},{"engine":"knowline-35b-a3b-gen1.5","name":"KnowLine 35B-A3B Gen1.5","long_name":"KnowLine 35B-A3B Gen1.5","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":35951822704,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/PelaAI/KnowLine-35B-A3B-Gen1.5","code_url":null,"base_model":"Qwen/Qwen3.5-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","weights_repo":null,"served_checkpoint":null,"org":{"id":"PelaAI","name":"PelaAI","logo":"data/orgs/PelaAI.png"},"pr":268,"replaces":"knowline-35b-a3b-gen1","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8121,"conf":0.8537,"ece":0.0442,"brier":0.251,"over95":0.0115,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0038,"conf":0.0869,"acc":0.0928},{"lo":0.1,"w":0.0157,"conf":0.1503,"acc":0.2081},{"lo":0.2,"w":0.0124,"conf":0.2513,"acc":0.2849},{"lo":0.3,"w":0.026,"conf":0.3555,"acc":0.303},{"lo":0.4,"w":0.0309,"conf":0.4556,"acc":0.3649},{"lo":0.5,"w":0.0651,"conf":0.5486,"acc":0.4758},{"lo":0.6,"w":0.0561,"conf":0.6504,"acc":0.5804},{"lo":0.7,"w":0.0615,"conf":0.7515,"acc":0.6163},{"lo":0.8,"w":0.0833,"conf":0.8545,"acc":0.7371},{"lo":0.9,"w":0.6453,"conf":0.9841,"acc":0.9655}],"areas":{"knowledge":{"ece":0.082,"acc":0.7171,"conf":0.7908},"language":{"ece":0.0178,"acc":0.905,"conf":0.9219},"retrieval":{"ece":0.025,"acc":0.8103,"conf":0.8281},"tools":{"ece":0.0412,"acc":0.9483,"conf":0.9851},"arts":{"ece":0.0513,"acc":0.7549,"conf":0.7982}}},"gaps":null,"latency":{"median":47.1,"p95":219.2,"mean":77.3},"results":{"1":{"score":0.9746,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.714,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.878,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.9101,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9401,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7997,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.7177,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8548,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7877,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.991,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6842,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.8335,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6936,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8799,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.551,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.977,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.9503,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9695,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.765,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2506,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.7287,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.4073,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4902,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.6139,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.5329,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9552,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.7083,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7866,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.864,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.7877,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.9418,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1457,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.645,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.983,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7127,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7772,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8082,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8838,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.9146,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8087,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9746,"skill":0.9657,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9602,"private_eq":0.9701},"2":{"raw":0.714,"skill":0.6697,"coverage":1,"random":0.1341,"tracks":[],"combined":0.671,"private_eq":0.7175},"3":{"raw":0.878,"skill":0.8756,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8645,"private_eq":0.8676},"4":{"raw":0.9101,"skill":0.9089,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9006,"private_eq":0.902},"5":{"raw":0.9401,"skill":0.9397,"coverage":1,"random":0.006,"tracks":[],"combined":0.8324,"private_eq":0.8343},"6":{"raw":0.7997,"skill":0.5304,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8164,"private_eq":0.8175},"10":{"raw":0.7177,"skill":0.5303,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8548,"skill":0.7899,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7877,"skill":0.682,"coverage":1,"random":0.3324,"tracks":[],"combined":0.515,"private_eq":0.6792},"20":{"raw":0.9911,"skill":0.9823,"coverage":1,"random":0.5,"tracks":[],"combined":0.9721,"private_eq":0.9857},"21":{"raw":0.6842,"skill":0.3683,"coverage":1,"random":0.5,"tracks":[],"combined":0.7698,"private_eq":0.8857},"22":{"raw":0.8323,"skill":0.831,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6237,"private_eq":0.6299},"23":{"raw":0.7121,"skill":0.4242,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8799,"skill":0.8399,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.551,"skill":0.4014,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5002,"combined":0.3269},"26":{"raw":0.9899,"skill":0.9865,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.977,"skill":0.9693,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9503,"skill":0.9006,"coverage":1,"random":0.5,"tracks":[],"combined":0.6227,"private_eq":0.813},"29":{"raw":0.9695,"skill":0.9593,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.765,"skill":0.7194,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8249,"random":0.25},{"track":"10-choice","raw":0.7051,"random":0.1}],"combined":0.7774,"private_eq":0.8176},"31":{"raw":0.2506,"skill":0.1838,"coverage":1,"random":0.0819,"tracks":[],"combined":0.055,"private_eq":0.1414},"32":{"raw":0.7287,"skill":0.5687,"coverage":1,"random":0.371,"tracks":[],"combined":0.3666,"private_eq":0.6055},"33":{"raw":0.4073,"skill":0.3994,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4102,"private_eq":0.4236},"36":{"raw":0.4902,"skill":0.4233,"coverage":1,"random":0.116,"tracks":[],"combined":0.406,"private_eq":0.48},"37":{"raw":0.6139,"skill":0.5157,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6131,"private_eq":0.6942},"38":{"raw":0.5329,"skill":0.518,"coverage":1,"random":0.031,"tracks":[],"combined":0.6834,"private_eq":0.6957},"39":{"raw":0.9552,"skill":0.9341,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4889,"private_eq":0.6558},"40":{"raw":0.7083,"skill":0.6248,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5389,"headline":false},{"track":"A · English","score":0.7083,"headline":true},{"track":"C · Arabic pairs","score":0.96,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"combined":0.6255,"private_eq":0.7114},"41":{"raw":0.7866,"skill":0.68,"coverage":1,"random":0.3333,"tracks":[],"combined":0.6188,"private_eq":0.7481},"42":{"raw":0.864,"skill":0.7354,"coverage":1,"random":0.486,"tracks":[],"combined":0.9762,"private_eq":0.9874},"43":{"raw":0.7877,"skill":0.6632,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5663,"private_eq":0.7291},"44":{"raw":0.9418,"skill":0.8836,"coverage":1,"random":0.5,"tracks":[],"combined":0.7975,"private_eq":0.8994},"45":{"raw":0.1457,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.645,"skill":0.4848,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.983,"skill":0.966,"coverage":1,"random":0.5,"tracks":[],"combined":0.7502,"private_eq":0.876},"57":{"raw":0.7127,"skill":0.6769,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.541,"combined":0.4788},"58":{"raw":0.7772,"skill":0.6771,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5186,"combined":0.2952},"59":{"raw":0.8082,"skill":0.6023,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4155,"private_eq":0.7208},"61":{"raw":0.8838,"skill":0.7676,"coverage":1,"random":0.5,"tracks":[],"combined":0.7231,"private_eq":0.8626},"62":{"raw":0.9146,"skill":0.8861,"coverage":1,"random":0.25,"tracks":[],"combined":0.6246,"private_eq":0.7209},"64":{"raw":0.8087,"skill":0.7609,"coverage":1,"random":0.2,"tracks":[],"combined":0.778,"private_eq":0.8236}},"coverage":1,"pending":0,"notes":["New in the 2026-10-22 PR batch (PR #268).","Replaces the earlier board entry knowline-35b-a3b-gen1 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 70.72, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8 at load); decision agreement with the submitted rows 97.82% on 831 rows (bar 96%).","Served in FP8 at load as in the submitted run, so the quantized agreement bar applies.","Read twice: short text-only requests (up to 3,000 tokens) are written a second time in the same prompt before the answer is read; one prefill per question, same weights as the earlier generation.","Training includes train splits of Decision Index benchmarks in request format (author-declared).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":77.68,"balanced_skill":65.77,"breadth_skill":69.92,"public_skill":70.72},"frozen_scores":{"balanced_raw":77.68,"balanced_skill":70.72,"breadth_skill":69.92},"categories":[{"id":"knowledge","raw":0.6371,"skill":0.5504,"combined":0.4668,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8105,"skill":0.7292,"combined":0.6699,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.8014,"skill":0.7539,"combined":0.705,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8985,"skill":0.8825,"combined":0.7966,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7789,"skill":0.6419,"combined":0.6737,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"knowline-35b-a3b-gen1.5","pub":70.72,"same_skills":65.49,"new_domains":51.82,"v":65.77,"lab":false,"jev":false,"rank_v":27,"rank_pub":10,"name":"KnowLine 35B-A3B Gen1.5","eq":{"S":65.19,"O":66.65},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":39.11,"new_domains":26.66}},"new_in_edition":true},{"engine":"knowline-26b-a4b-gen2.5","name":"KnowLine 26B-A4B Gen2.5","long_name":"KnowLine 26B-A4B Gen2.5","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/PelaAI/KnowLine-26B-A4B-Gen2.5","code_url":null,"base_model":"google/gemma-4-26B-A4B-it","base_url":"https://huggingface.co/google/gemma-4-26B-A4B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"PelaAI","name":"PelaAI","logo":"data/orgs/PelaAI.png"},"pr":271,"replaces":"knowline-26b-a4b-gen2","new":true},"completed":110201,"counts":{"ok":110200,"unsupported":0,"error":1},"lifted":false,"calibration":{"acc":0.7985,"conf":0.8143,"ece":0.0194,"brier":0.2655,"over95":0.0085,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0043,"conf":0.0857,"acc":0.0829},{"lo":0.1,"w":0.0164,"conf":0.1463,"acc":0.2253},{"lo":0.2,"w":0.0159,"conf":0.2558,"acc":0.2873},{"lo":0.3,"w":0.0334,"conf":0.3549,"acc":0.2881},{"lo":0.4,"w":0.046,"conf":0.4538,"acc":0.4152},{"lo":0.5,"w":0.0848,"conf":0.5477,"acc":0.5185},{"lo":0.6,"w":0.0742,"conf":0.6508,"acc":0.5999},{"lo":0.7,"w":0.0786,"conf":0.7493,"acc":0.7397},{"lo":0.8,"w":0.096,"conf":0.8537,"acc":0.8171},{"lo":0.9,"w":0.5505,"conf":0.9794,"acc":0.9738}],"areas":{"knowledge":{"ece":0.0516,"acc":0.7102,"conf":0.7498},"language":{"ece":0.0061,"acc":0.8867,"conf":0.8871},"retrieval":{"ece":0.0228,"acc":0.7846,"conf":0.7975},"tools":{"ece":0.0341,"acc":0.9611,"conf":0.9817},"arts":{"ece":0.028,"acc":0.7297,"conf":0.7251}}},"gaps":null,"latency":{"median":71.7,"p95":417.5,"mean":124.9},"results":{"1":{"score":0.9817,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6957,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.9075,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8283,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9298,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7993,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":9999,"unsupported":0,"errors":1,"median_ms":null},"9":{"score":0.7159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6829,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8188,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7604,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9744,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6914,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.7895,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.682,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8716,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4541,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9882,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.977,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.9234,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9538,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.8654,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.299,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.7021,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3897,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.5018,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.6108,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.4822,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9273,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.7136,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7932,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8432,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8772,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8802,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.0918,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.5973,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.9085,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6911,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.848,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8472,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8605,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.9072,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7992,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9817,"skill":0.9753,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.6957,"skill":0.6486,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6906,"private_eq":0.7343},"3":{"raw":0.9075,"skill":0.9057,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8804,"private_eq":0.883},"4":{"raw":0.8283,"skill":0.8261,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8715,"private_eq":0.8736},"5":{"raw":0.9298,"skill":0.9294,"coverage":1,"random":0.006,"tracks":[],"combined":0.7763,"private_eq":0.7792},"6":{"raw":0.7992,"skill":0.5292,"coverage":0.9999,"random":0.5735,"tracks":[]},"9":{"raw":0.7159,"skill":0.7159,"coverage":1,"random":0,"tracks":[],"combined":0.6268,"private_eq":0.63},"10":{"raw":0.6829,"skill":0.4724,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8188,"skill":0.738,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7604,"skill":0.6411,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5239,"private_eq":0.6851},"20":{"raw":0.9748,"skill":0.9497,"coverage":1,"random":0.5,"tracks":[],"combined":0.9577,"private_eq":0.9786},"21":{"raw":0.6914,"skill":0.3828,"coverage":1,"random":0.5,"tracks":[],"combined":0.3472,"private_eq":0.6768},"22":{"raw":0.7887,"skill":0.7871,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6043,"private_eq":0.6109},"23":{"raw":0.7008,"skill":0.4017,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8716,"skill":0.8288,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4541,"skill":0.2721,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4836,"combined":0.3045},"26":{"raw":0.9882,"skill":0.9843,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.977,"skill":0.9693,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9234,"skill":0.8468,"coverage":1,"random":0.5,"tracks":[],"combined":0.8145,"private_eq":0.9078},"29":{"raw":0.9538,"skill":0.9384,"coverage":1,"random":0.25,"tracks":[],"combined":0.9633,"private_eq":0.972},"30":{"raw":0.8654,"skill":0.8383,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8901,"random":0.25},{"track":"10-choice","raw":0.8408,"random":0.1}],"combined":0.8335,"private_eq":0.8634},"31":{"raw":0.299,"skill":0.2365,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0734,"private_eq":0.1581},"32":{"raw":0.7021,"skill":0.5264,"coverage":1,"random":0.371,"tracks":[],"combined":0.3434,"private_eq":0.5911},"33":{"raw":0.3897,"skill":0.3816,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3331,"private_eq":0.3483},"36":{"raw":0.5018,"skill":0.4365,"coverage":1,"random":0.116,"tracks":[],"combined":0.3854,"private_eq":0.462},"37":{"raw":0.6108,"skill":0.5118,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6021,"private_eq":0.6856},"38":{"raw":0.4822,"skill":0.4656,"coverage":1,"random":0.031,"tracks":[],"combined":0.671,"private_eq":0.6839},"39":{"raw":0.9273,"skill":0.8931,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5606,"private_eq":0.704},"40":{"raw":0.7136,"skill":0.6316,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5996,"headline":false},{"track":"A · English","score":0.7136,"headline":true},{"track":"C · Arabic pairs","score":0.97,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.7808,"private_eq":0.8308},"41":{"raw":0.7932,"skill":0.6898,"coverage":1,"random":0.3333,"tracks":[],"combined":0.587,"private_eq":0.7271},"42":{"raw":0.8432,"skill":0.6949,"coverage":1,"random":0.486,"tracks":[],"combined":0.89,"private_eq":0.9436},"43":{"raw":0.8772,"skill":0.8052,"coverage":1,"random":0.3697,"tracks":[],"combined":0.629,"private_eq":0.7682},"44":{"raw":0.8802,"skill":0.7604,"coverage":1,"random":0.5,"tracks":[],"combined":0.7119,"private_eq":0.8571},"45":{"raw":0.0918,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5973,"skill":0.4155,"coverage":1,"random":0.311,"tracks":[],"combined":0.2756,"private_eq":0.5059},"56":{"raw":0.9085,"skill":0.817,"coverage":1,"random":0.5,"tracks":[],"combined":0.5337,"private_eq":0.769},"57":{"raw":0.6911,"skill":0.6526,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4951,"combined":0.4266},"58":{"raw":0.848,"skill":0.7797,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5143,"combined":0.2889},"59":{"raw":0.8472,"skill":0.6832,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3492,"private_eq":0.6892},"61":{"raw":0.8605,"skill":0.721,"coverage":1,"random":0.5,"tracks":[],"combined":0.6721,"private_eq":0.8374},"62":{"raw":0.9072,"skill":0.8763,"coverage":1,"random":0.25,"tracks":[],"combined":0.6384,"private_eq":0.7311},"64":{"raw":0.7992,"skill":0.749,"coverage":1,"random":0.2,"tracks":[],"combined":0.7101,"private_eq":0.7699}},"coverage":1,"pending":0,"notes":["New in the 2026-10-22 PR batch (PR #271).","Replaces the earlier board entry knowline-26b-a4b-gen2 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 68.21, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.11% on 831 rows (bar 98.5%).","Read twice: short text-only requests (up to 3,000 tokens) are written a second time in the same prompt before the answer is read; one prefill per question, same weights as the earlier generation.","Training includes train splits of Decision Index benchmarks in request format (author-declared).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":75.56,"balanced_skill":64.57,"breadth_skill":67.61,"public_skill":68.21},"frozen_scores":{"balanced_raw":75.56,"balanced_skill":68.21,"breadth_skill":67.61},"categories":[{"id":"knowledge","raw":0.6353,"skill":0.5508,"combined":0.4746,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.796,"skill":0.7119,"combined":0.6694,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7715,"skill":0.7091,"combined":0.6436,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8492,"skill":0.833,"combined":0.7606,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7587,"skill":0.6143,"combined":0.579,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"knowline-26b-a4b-gen2.5","pub":68.21,"same_skills":62.82,"new_domains":52.67,"v":64.57,"lab":false,"jev":false,"rank_v":34,"rank_pub":16,"name":"KnowLine 26B-A4B Gen2.5","eq":{"S":62.49,"O":67.7},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":37.5,"new_domains":27.08}},"new_in_edition":true},{"engine":"knowline-4b-gen5.5","name":"KnowLine 4B Gen5.5","long_name":"KnowLine 4B Gen5.5","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/PelaAI/KnowLine-4B-Gen5.5","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"PelaAI","name":"PelaAI","logo":"data/orgs/PelaAI.png"},"pr":272,"replaces":"knowline-4b-gen5","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7825,"conf":0.8275,"ece":0.0472,"brier":0.2928,"over95":0.0142,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0073,"conf":0.082,"acc":0.1127},{"lo":0.1,"w":0.0166,"conf":0.1389,"acc":0.1941},{"lo":0.2,"w":0.0111,"conf":0.249,"acc":0.2437},{"lo":0.3,"w":0.023,"conf":0.3463,"acc":0.302},{"lo":0.4,"w":0.0453,"conf":0.4544,"acc":0.4094},{"lo":0.5,"w":0.0752,"conf":0.5506,"acc":0.519},{"lo":0.6,"w":0.0756,"conf":0.6492,"acc":0.5638},{"lo":0.7,"w":0.0753,"conf":0.7487,"acc":0.6397},{"lo":0.8,"w":0.0869,"conf":0.8549,"acc":0.7421},{"lo":0.9,"w":0.5836,"conf":0.9802,"acc":0.9526}],"areas":{"knowledge":{"ece":0.1006,"acc":0.6728,"conf":0.7663},"language":{"ece":0.0282,"acc":0.8697,"conf":0.8974},"retrieval":{"ece":0.0339,"acc":0.7706,"conf":0.7829},"tools":{"ece":0.0225,"acc":0.9637,"conf":0.9782},"arts":{"ece":0.0411,"acc":0.7461,"conf":0.7689}}},"gaps":null,"latency":{"median":35.2,"p95":170.3,"mean":61.1},"results":{"1":{"score":0.9728,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6839,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8898,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.9031,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9256,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.782,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.9886,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6518,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8446,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7327,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9702,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6735,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.843,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6946,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.7947,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4694,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9861,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9565,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.88,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9474,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.7017,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.228,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.617,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3994,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4785,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5804,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.5063,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9067,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6073,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.784,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8303,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.6789,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8982,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1317,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.6241,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.9835,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.5992,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.8211,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8091,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8808,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.9047,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7367,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9728,"skill":0.9633,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.6839,"skill":0.635,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6802,"private_eq":0.7254},"3":{"raw":0.8898,"skill":0.8877,"coverage":1,"random":0.0189,"tracks":[],"combined":0.854,"private_eq":0.8574},"4":{"raw":0.9031,"skill":0.9019,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8564,"private_eq":0.8588},"5":{"raw":0.9256,"skill":0.9252,"coverage":1,"random":0.006,"tracks":[],"combined":0.7931,"private_eq":0.7957},"6":{"raw":0.782,"skill":0.4889,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9886,"skill":0.9886,"coverage":1,"random":0,"tracks":[],"combined":0.7507,"private_eq":0.7525},"10":{"raw":0.6518,"skill":0.4206,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8446,"skill":0.7752,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7327,"skill":0.5996,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5095,"private_eq":0.6756},"20":{"raw":0.971,"skill":0.942,"coverage":1,"random":0.5,"tracks":[],"combined":0.8999,"private_eq":0.95},"21":{"raw":0.6735,"skill":0.347,"coverage":1,"random":0.5,"tracks":[],"combined":0.5584,"private_eq":0.7812},"22":{"raw":0.8424,"skill":0.8412,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6304,"private_eq":0.6365},"23":{"raw":0.7013,"skill":0.4025,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7947,"skill":0.7263,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4694,"skill":0.2925,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4181,"combined":0.2162},"26":{"raw":0.9861,"skill":0.9815,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9565,"skill":0.942,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.88,"skill":0.76,"coverage":1,"random":0.5,"tracks":[],"combined":0.7892,"private_eq":0.8953},"29":{"raw":0.9474,"skill":0.9299,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7017,"skill":0.6428,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7688,"random":0.25},{"track":"10-choice","raw":0.6346,"random":0.1}],"combined":0.6604,"private_eq":0.7222},"31":{"raw":0.228,"skill":0.1592,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0658,"private_eq":0.1512},"32":{"raw":0.617,"skill":0.3911,"coverage":1,"random":0.371,"tracks":[],"combined":0.3201,"private_eq":0.5766},"33":{"raw":0.3994,"skill":0.3914,"coverage":1,"random":0.0131,"tracks":[],"combined":0.369,"private_eq":0.3834},"36":{"raw":0.4785,"skill":0.41,"coverage":1,"random":0.116,"tracks":[],"combined":0.3795,"private_eq":0.4568},"37":{"raw":0.5804,"skill":0.4737,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5426,"private_eq":0.6387},"38":{"raw":0.5063,"skill":0.4905,"coverage":1,"random":0.031,"tracks":[],"combined":0.6228,"private_eq":0.6377},"39":{"raw":0.9067,"skill":0.8628,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5214,"private_eq":0.6776},"40":{"raw":0.6073,"skill":0.4948,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5,"headline":false},{"track":"A · English","score":0.6073,"headline":true},{"track":"C · Arabic pairs","score":0.935,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"combined":0.5193,"private_eq":0.6298},"41":{"raw":0.784,"skill":0.676,"coverage":1,"random":0.3333,"tracks":[],"combined":0.664,"private_eq":0.7779},"42":{"raw":0.8303,"skill":0.6698,"coverage":1,"random":0.486,"tracks":[],"combined":0.8444,"private_eq":0.9204},"43":{"raw":0.6789,"skill":0.4906,"coverage":1,"random":0.3697,"tracks":[],"combined":0.516,"private_eq":0.6978},"44":{"raw":0.8982,"skill":0.7964,"coverage":1,"random":0.5,"tracks":[],"combined":0.7172,"private_eq":0.8597},"45":{"raw":0.1317,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6241,"skill":0.4545,"coverage":1,"random":0.311,"tracks":[],"combined":0.1523,"private_eq":0.4219},"56":{"raw":0.9835,"skill":0.967,"coverage":1,"random":0.5,"tracks":[],"combined":0.7502,"private_eq":0.876},"57":{"raw":0.5992,"skill":0.5492,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4375,"combined":0.3611},"58":{"raw":0.8211,"skill":0.7407,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4584,"combined":0.2069},"59":{"raw":0.8091,"skill":0.6042,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3162,"private_eq":0.6735},"61":{"raw":0.8808,"skill":0.7616,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"62":{"raw":0.9047,"skill":0.8729,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.7367,"skill":0.6709,"coverage":1,"random":0.2,"tracks":[],"combined":0.7003,"private_eq":0.7622}},"coverage":1,"pending":0,"notes":["New in the 2026-10-22 PR batch (PR #272).","Replaces the earlier board entry knowline-4b-gen5 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 66.75, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8 at load); decision agreement with the submitted rows 98.47% on 831 rows (bar 96%).","Served in FP8 at load as in the submitted run, so the quantized agreement bar applies.","Read twice: short text-only requests (up to 3,000 tokens) are written a second time in the same prompt before the answer is read; one prefill per question, same weights as the earlier generation.","Training includes train splits of Decision Index benchmarks in request format (author-declared).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":74.86,"balanced_skill":55.99,"breadth_skill":65.5,"public_skill":66.76},"frozen_scores":{"balanced_raw":74.86,"balanced_skill":66.76,"breadth_skill":65.5},"categories":[{"id":"knowledge","raw":0.5837,"skill":0.4735,"combined":0.4201,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7771,"skill":0.6818,"combined":0.631,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7899,"skill":0.7404,"combined":0.6614,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8912,"skill":0.8736,"combined":0.7712,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7582,"skill":0.6097,"combined":0.5883,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"knowline-4b-gen5.5","pub":66.76,"same_skills":61.2,"new_domains":37.28,"v":55.99,"lab":false,"jev":false,"rank_v":72,"rank_pub":25,"name":"KnowLine 4B Gen5.5","eq":{"S":60.85,"O":48.69},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":36.51,"new_domains":19.48}},"new_in_edition":true},{"engine":"gliclass-v4-edge","name":"GLiClass v4 edge (0.4B)","long_name":"GLiClass v4 edge (0.4B)","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":439790595,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/knowledgator/gliclass-v4-edge","code_url":null,"base_model":"microsoft/deberta-v3-large","base_url":"https://huggingface.co/microsoft/deberta-v3-large","weights_repo":null,"served_checkpoint":null,"org":{"id":"knowledgator","name":"Knowledgator Engineering","logo":"data/orgs/knowledgator.png"},"pr":270,"new":true},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.68,"conf":0.7581,"ece":0.0887,"brier":0.4259,"over95":0.0315,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0278,"conf":0.0784,"acc":0.1911},{"lo":0.1,"w":0.0296,"conf":0.139,"acc":0.2121},{"lo":0.2,"w":0.0206,"conf":0.2571,"acc":0.1929},{"lo":0.3,"w":0.0332,"conf":0.3634,"acc":0.2759},{"lo":0.4,"w":0.0574,"conf":0.4501,"acc":0.4176},{"lo":0.5,"w":0.1064,"conf":0.5476,"acc":0.4805},{"lo":0.6,"w":0.0912,"conf":0.6496,"acc":0.5277},{"lo":0.7,"w":0.0692,"conf":0.7498,"acc":0.6201},{"lo":0.8,"w":0.0952,"conf":0.8541,"acc":0.7159},{"lo":0.9,"w":0.4694,"conf":0.9756,"acc":0.8969}],"areas":{"knowledge":{"ece":0.1393,"acc":0.5206,"conf":0.658},"language":{"ece":0.0849,"acc":0.7904,"conf":0.8753},"retrieval":{"ece":0.0838,"acc":0.7699,"conf":0.8485},"tools":{"ece":0.0467,"acc":0.9092,"conf":0.8695},"arts":{"ece":0.0859,"acc":0.5907,"conf":0.6083}}},"gaps":null,"latency":{"median":21.8,"p95":188.2,"mean":55.7},"results":{"1":{"score":0.9055,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":19.9},"2":{"score":0.5502,"jev":null,"metric":"nDCG@10","requests":685,"answered":683,"unsupported":2,"errors":0,"median_ms":127.6},"3":{"score":0.8287,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":216.5},"4":{"score":0.908,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":11.6},"5":{"score":0.9229,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":11.8},"6":{"score":0.682,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":39.8},"9":{"score":0.125,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":179.1},"10":{"score":0.6009,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":19},"11":{"score":0.7896,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":314.2},"12":{"score":0.6171,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":10.7},"20":{"score":0.917,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.7},"21":{"score":0.6377,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":10.6},"22":{"score":0.255,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":84.7},"23":{"score":0.5658,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.1},"24":{"score":0.5151,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":10.6},"25":{"score":0.3265,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":10.9},"26":{"score":0.8556,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":10.6},"27":{"score":0.7022,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":10.5},"28":{"score":0.8019,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":10.5},"29":{"score":0.828,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":10.9},"30":{"score":0.1914,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.18,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":13.3},"32":{"score":0.5891,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":14.6},"33":{"score":0.2545,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":29.7},"36":{"score":0.3138,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":175.7},"37":{"score":0.5158,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.4},"38":{"score":0.16,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":120.8},"39":{"score":0.9072,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":12.1},"40":{"score":0.5109,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":10.6},"41":{"score":0.7305,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":10.7},"42":{"score":0.7186,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":11.6},"43":{"score":0.3158,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":10.7},"44":{"score":0.7112,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.7},"45":{"score":0.1537,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":11},"50":{"score":0.5442,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":13.8},"56":{"score":0.934,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":29.9},"57":{"score":0.2931,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":10.7},"58":{"score":0.5039,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":10.7},"59":{"score":0.8004,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":13.3},"61":{"score":0.838,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":11.5},"62":{"score":0.6229,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":14},"64":{"score":0.6212,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":10.9}},"benchmarks":{"1":{"raw":0.9055,"skill":0.8725,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7081,"private_eq":0.7855},"2":{"raw":0.5486,"skill":0.4787,"coverage":0.9971,"random":0.1341,"tracks":[],"combined":0.5937,"private_eq":0.6513},"3":{"raw":0.8287,"skill":0.8254,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6126,"private_eq":0.6233},"4":{"raw":0.908,"skill":0.9068,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8499,"private_eq":0.8525},"5":{"raw":0.9229,"skill":0.9224,"coverage":1,"random":0.006,"tracks":[],"combined":0.678,"private_eq":0.6826},"6":{"raw":0.682,"skill":0.2544,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1,"random":0,"tracks":[],"combined":0.0755,"private_eq":0.085},"10":{"raw":0.6009,"skill":0.3359,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7896,"skill":0.6957,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6171,"skill":0.4265,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5519,"private_eq":0.7036},"20":{"raw":0.9118,"skill":0.8236,"coverage":1,"random":0.5,"tracks":[],"combined":0.7914,"private_eq":0.8964},"21":{"raw":0.6377,"skill":0.2755,"coverage":1,"random":0.5,"tracks":[],"combined":0.2666,"private_eq":0.637},"22":{"raw":0.2364,"skill":0.2305,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1781,"private_eq":0.1928},"23":{"raw":0.5604,"skill":0.1208,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5151,"skill":0.3535,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3265,"skill":0.102,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3215,"combined":0.0859},"26":{"raw":0.8556,"skill":0.8074,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7022,"skill":0.6028,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8019,"skill":0.6038,"coverage":1,"random":0.5,"tracks":[],"combined":0.4916,"private_eq":0.7482},"29":{"raw":0.828,"skill":0.7707,"coverage":1,"random":0.25,"tracks":[],"combined":0.6168,"private_eq":0.7151},"30":{"raw":0.1914,"skill":0.0209,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2737,"random":0.25},{"track":"10-choice","raw":0.1092,"random":0.1}],"combined":0.0238,"private_eq":0.203},"31":{"raw":0.18,"skill":0.1069,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0463,"private_eq":0.1335},"32":{"raw":0.5891,"skill":0.3467,"coverage":1,"random":0.371,"tracks":[],"combined":0.0418,"private_eq":0.4035},"33":{"raw":0.2545,"skill":0.2446,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1018,"private_eq":0.1227},"36":{"raw":0.3138,"skill":0.2237,"coverage":1,"random":0.116,"tracks":[],"combined":0.2669,"private_eq":0.3584},"37":{"raw":0.5158,"skill":0.3926,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4442,"private_eq":0.5611},"38":{"raw":0.16,"skill":0.1331,"coverage":1,"random":0.031,"tracks":[],"combined":0.2104,"private_eq":0.2426},"39":{"raw":0.9072,"skill":0.8635,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5609,"private_eq":0.7042},"40":{"raw":0.5109,"skill":0.3707,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3159,"headline":false},{"track":"A · English","score":0.5109,"headline":true},{"track":"C · Arabic pairs","score":0.755,"headline":false},{"track":"C · English pairs","score":0.93,"headline":false}],"combined":0.3079,"private_eq":0.4674},"41":{"raw":0.7305,"skill":0.5958,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4268,"private_eq":0.6215},"42":{"raw":0.7186,"skill":0.4525,"coverage":1,"random":0.486,"tracks":[],"combined":0.6425,"private_eq":0.8178},"43":{"raw":0.3158,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.098,"private_eq":0.4373},"44":{"raw":0.7112,"skill":0.4224,"coverage":1,"random":0.5,"tracks":[],"combined":0.4014,"private_eq":0.7036},"45":{"raw":0.1537,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5442,"skill":0.3384,"coverage":1,"random":0.311,"tracks":[],"combined":0.3264,"private_eq":0.5405},"56":{"raw":0.934,"skill":0.868,"coverage":1,"random":0.5,"tracks":[],"combined":0.6203,"private_eq":0.8118},"57":{"raw":0.2931,"skill":0.2049,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5244,"combined":0.4599},"58":{"raw":0.5039,"skill":0.2809,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3519,"combined":0.0508},"59":{"raw":0.8004,"skill":0.5861,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2034,"private_eq":0.6197},"61":{"raw":0.838,"skill":0.676,"coverage":1,"random":0.5,"tracks":[],"combined":0.5924,"private_eq":0.798},"62":{"raw":0.6229,"skill":0.4972,"coverage":1,"random":0.25,"tracks":[],"combined":0.4329,"private_eq":0.5788},"64":{"raw":0.6212,"skill":0.5265,"coverage":1,"random":0.2,"tracks":[],"combined":0.4482,"private_eq":0.5628}},"coverage":0.9999,"pending":0.0001,"notes":["New in the 2026-10-22 PR batch (PR #270).","Submitted public 0.3 run adopted (kit rescore 47.49, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.59% on 829 rows (bar 98.5%).","Inputs over its declared 16,384-token limit count as unsupported.","Runs with a new in-process engine added by the PR; evaluated by the lab in a sandbox."],"scores":{"balanced_raw":59.7,"balanced_skill":31.21,"breadth_skill":44.54,"public_skill":47.49},"frozen_scores":{"balanced_raw":59.7,"balanced_skill":47.49,"breadth_skill":44.54},"categories":[{"id":"knowledge","raw":0.3947,"skill":0.2143,"combined":0.1869,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6757,"skill":0.5462,"combined":0.4469,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7366,"skill":0.6667,"combined":0.5774,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6255,"skill":0.5812,"combined":0.4976,"coverage":0.9995,"pending":0,"n":5},{"id":"arts","raw":0.5853,"skill":0.3859,"combined":0.4022,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"gliclass-v4-edge","pub":47.49,"same_skills":42.16,"new_domains":10.53,"v":31.21,"lab":false,"jev":false,"rank_v":164,"rank_pub":131,"name":"GLiClass v4 edge (0.4B)","eq":{"S":41.59,"O":15.64},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":24.95,"new_domains":6.26}},"new_in_edition":true},{"engine":"vllm-sr-d3-ultra","name":"vLLM-SR d3-ultra (Decision 3.0, 27B)","long_name":"vLLM-SR d3-ultra (Decision 3.0, 27B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/d3-ultra","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":"vllm-sr/d3-ultra","served_checkpoint":"37fee01926f3de396620e794201b074fb427ac2d","weights_note":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pr":302,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.809,"conf":0.8755,"ece":0.0666,"brier":0.273,"over95":0.0204,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0009,"conf":0.092,"acc":0.0909},{"lo":0.1,"w":0.012,"conf":0.1549,"acc":0.152},{"lo":0.2,"w":0.0094,"conf":0.2474,"acc":0.209},{"lo":0.3,"w":0.0132,"conf":0.356,"acc":0.2851},{"lo":0.4,"w":0.0254,"conf":0.4602,"acc":0.3698},{"lo":0.5,"w":0.0571,"conf":0.5499,"acc":0.4941},{"lo":0.6,"w":0.0509,"conf":0.6473,"acc":0.5432},{"lo":0.7,"w":0.0694,"conf":0.7511,"acc":0.6067},{"lo":0.8,"w":0.0905,"conf":0.8516,"acc":0.6457},{"lo":0.9,"w":0.6712,"conf":0.9852,"acc":0.9468}],"areas":{"knowledge":{"ece":0.0995,"acc":0.7451,"conf":0.8446},"language":{"ece":0.02,"acc":0.9029,"conf":0.9194},"retrieval":{"ece":0.1244,"acc":0.7295,"conf":0.8538},"tools":{"ece":0.0393,"acc":0.9549,"conf":0.9906},"arts":{"ece":0.066,"acc":0.7437,"conf":0.8067}}},"gaps":null,"latency":{"median":200.5,"p95":673.5,"mean":423.7},"results":{"1":{"score":0.9728,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":321.1},"2":{"score":0.714,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":5213.6},"3":{"score":0.8661,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1642.5},"4":{"score":0.9416,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":445.7},"5":{"score":0.9607,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":705},"6":{"score":0.7948,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":649},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":991.9},"10":{"score":0.4275,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":312},"11":{"score":0.8469,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":911},"12":{"score":0.7649,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":133.3},"20":{"score":0.983,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":130},"21":{"score":0.6769,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":104.8},"22":{"score":0.7825,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2182.2},"23":{"score":0.7034,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":188.5},"24":{"score":0.8873,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":112.2},"25":{"score":0.6071,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":140.7},"26":{"score":0.9928,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":96.2},"27":{"score":0.9753,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":99.7},"28":{"score":0.9605,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":93.5},"29":{"score":0.9816,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":162.5},"30":{"score":0.8525,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2656,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":402},"32":{"score":0.7992,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":455.5},"33":{"score":0.3861,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":428.3},"36":{"score":0.515,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":6420.1},"37":{"score":0.6155,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":177.8},"38":{"score":0.6358,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3439.8},"39":{"score":0.9334,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":144.1},"40":{"score":0.5473,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":102.7},"41":{"score":0.8286,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":138.2},"42":{"score":0.8627,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":181.1},"43":{"score":0.8947,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":120.3},"44":{"score":0.9486,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":130.5},"45":{"score":0.1956,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":178.7},"50":{"score":0.6253,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":289.7},"56":{"score":0.802,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":357.7},"57":{"score":0.7601,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":141.5},"58":{"score":0.9237,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":115.9},"59":{"score":0.8335,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":226.2},"61":{"score":0.8958,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":168},"62":{"score":0.908,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":250.9},"64":{"score":0.8409,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":158.7}},"benchmarks":{"1":{"raw":0.9728,"skill":0.9633,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9739,"private_eq":0.9801},"2":{"raw":0.714,"skill":0.6697,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7219,"private_eq":0.7611},"3":{"raw":0.8661,"skill":0.8635,"coverage":1,"random":0.0189,"tracks":[],"combined":0.938,"private_eq":0.9389},"4":{"raw":0.9416,"skill":0.9408,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9198,"private_eq":0.9207},"5":{"raw":0.9607,"skill":0.9605,"coverage":1,"random":0.006,"tracks":[],"combined":0.8763,"private_eq":0.8774},"6":{"raw":0.7948,"skill":0.5189,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.9479,"private_eq":0.9475},"10":{"raw":0.4275,"skill":0.0474,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8469,"skill":0.7786,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7649,"skill":0.6478,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5307,"private_eq":0.6896},"20":{"raw":0.9839,"skill":0.9677,"coverage":1,"random":0.5,"tracks":[],"combined":0.9577,"private_eq":0.9786},"21":{"raw":0.6769,"skill":0.3539,"coverage":1,"random":0.5,"tracks":[],"combined":0.2768,"private_eq":0.642},"22":{"raw":0.7839,"skill":0.7822,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5654,"private_eq":0.5727},"23":{"raw":0.7243,"skill":0.4487,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8873,"skill":0.8497,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.6071,"skill":0.4762,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6737,"combined":0.5609},"26":{"raw":0.9928,"skill":0.9904,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9753,"skill":0.9671,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9605,"skill":0.921,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"29":{"raw":0.9816,"skill":0.9755,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8525,"skill":0.8239,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8901,"random":0.25},{"track":"10-choice","raw":0.815,"random":0.1}],"combined":0.7453,"private_eq":0.7914},"31":{"raw":0.2656,"skill":0.2001,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1081,"private_eq":0.1896},"32":{"raw":0.7992,"skill":0.6808,"coverage":1,"random":0.371,"tracks":[],"combined":0.4285,"private_eq":0.644},"33":{"raw":0.3861,"skill":0.378,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4512,"private_eq":0.4636},"36":{"raw":0.515,"skill":0.4513,"coverage":1,"random":0.116,"tracks":[],"combined":0.4392,"private_eq":0.509},"37":{"raw":0.6155,"skill":0.5177,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6174,"private_eq":0.6976},"38":{"raw":0.6358,"skill":0.6241,"coverage":1,"random":0.031,"tracks":[],"combined":0.7286,"private_eq":0.739},"39":{"raw":0.9334,"skill":0.902,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5316,"private_eq":0.6845},"40":{"raw":0.5473,"skill":0.4176,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.6025,"headline":false},{"track":"A · English","score":0.5473,"headline":true},{"track":"C · Arabic pairs","score":0.91,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.4124,"private_eq":0.5477},"41":{"raw":0.8286,"skill":0.7428,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4484,"private_eq":0.6358},"42":{"raw":0.8627,"skill":0.7329,"coverage":1,"random":0.486,"tracks":[],"combined":0.9892,"private_eq":0.994},"43":{"raw":0.8947,"skill":0.833,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6582,"private_eq":0.7864},"44":{"raw":0.9486,"skill":0.8972,"coverage":1,"random":0.5,"tracks":[],"combined":0.8511,"private_eq":0.9259},"45":{"raw":0.1956,"skill":0.0377,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6253,"skill":0.4562,"coverage":1,"random":0.311,"tracks":[],"combined":0.2539,"private_eq":0.4911},"56":{"raw":0.802,"skill":0.604,"coverage":1,"random":0.5,"tracks":[],"combined":0.8885,"private_eq":0.9444},"57":{"raw":0.7601,"skill":0.7302,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6651,"combined":0.62},"58":{"raw":0.9237,"skill":0.8894,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.535,"combined":0.3192},"59":{"raw":0.8335,"skill":0.6548,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3074,"private_eq":0.6693},"61":{"raw":0.8958,"skill":0.7916,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.908,"skill":0.8773,"coverage":1,"random":0.25,"tracks":[],"combined":0.5972,"private_eq":0.7006},"64":{"raw":0.8409,"skill":0.8011,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"notes":["New in the 2026-10-26 PR batch (PR #302).","Submitted public 0.3 run adopted (kit rescore 71.57, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.81% on 791 rows (bar 98.5%).","A separate model from vLLM-SR d3-turbo v1.1 (27B) and vLLM-SR d3 (27B), which stay on the board.","Weights are public on the Hugging Face Hub (Apache-2.0); the lab scored revision 37fee01.","Served with the model repo's own engine on an RTX PRO 6000, where the submitted run used an AMD MI325X."],"scores":{"balanced_raw":78.59,"balanced_skill":70.18,"breadth_skill":71.13,"public_skill":71.57},"frozen_scores":{"balanced_raw":78.59,"balanced_skill":71.57,"breadth_skill":71.13},"categories":[{"id":"knowledge","raw":0.6903,"skill":0.6232,"combined":0.5378,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8068,"skill":0.7235,"combined":0.6257,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.79,"skill":0.7176,"combined":0.7321,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8942,"skill":0.8776,"combined":0.8447,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7725,"skill":0.635,"combined":0.5721,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vllm-sr-d3-ultra","pub":71.57,"same_skills":66.77,"new_domains":59.16,"v":70.18,"lab":false,"jev":false,"rank_v":5,"rank_pub":9,"name":"vLLM-SR d3-ultra (Decision 3.0, 27B)","eq":{"S":66.49,"O":75.71},"rank_tie":2,"tied":true,"parts":{"public":0,"same_skills":39.89,"new_domains":30.29}},"new_in_edition":true},{"engine":"standard-one-35b-sh-v1","name":"Standard One 35B SH (v1)","long_name":"Standard One 35B SH (v1)","variant":null,"meta":{"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":35107181936,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/StandardThinking/StandardOne-35B-SH","code_url":null,"base_model":"Qwen/Qwen3.6-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","weights_repo":null,"served_checkpoint":null,"org":{"id":"StandardThinking","name":"Standard Thinking","logo":"data/orgs/StandardThinking.png"},"pr":297,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7794,"conf":0.8331,"ece":0.0537,"brier":0.3027,"over95":0.0106,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0009,"conf":0.0899,"acc":0.0833},{"lo":0.1,"w":0.0097,"conf":0.1574,"acc":0.1137},{"lo":0.2,"w":0.0123,"conf":0.2514,"acc":0.2371},{"lo":0.3,"w":0.0226,"conf":0.3585,"acc":0.2135},{"lo":0.4,"w":0.0414,"conf":0.4536,"acc":0.2962},{"lo":0.5,"w":0.0736,"conf":0.551,"acc":0.4941},{"lo":0.6,"w":0.0658,"conf":0.651,"acc":0.5703},{"lo":0.7,"w":0.0777,"conf":0.7523,"acc":0.6501},{"lo":0.8,"w":0.1309,"conf":0.8558,"acc":0.7226},{"lo":0.9,"w":0.5651,"conf":0.9691,"acc":0.9543}],"areas":{"knowledge":{"ece":0.0827,"acc":0.6843,"conf":0.7669},"language":{"ece":0.0199,"acc":0.8746,"conf":0.8883},"retrieval":{"ece":0.08,"acc":0.7485,"conf":0.8278},"tools":{"ece":0.0293,"acc":0.9507,"conf":0.9755},"arts":{"ece":0.0643,"acc":0.7205,"conf":0.7849}}},"gaps":null,"latency":{"median":319.9,"p95":409.8,"mean":376.4},"results":{"1":{"score":0.9717,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":986.7},"2":{"score":0.6944,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":26184.6},"3":{"score":0.8642,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":6766.3},"4":{"score":0.8986,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":739.8},"5":{"score":0.9342,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":939.7},"6":{"score":0.7902,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1706.8},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1751.3},"10":{"score":0.5887,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":1063.6},"11":{"score":0.8482,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1858.6},"12":{"score":0.7504,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":423.7},"20":{"score":0.9552,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":385.9},"21":{"score":0.6119,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":341.9},"22":{"score":0.7715,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":5943.5},"23":{"score":0.6584,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":631.8},"24":{"score":0.856,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":377.9},"25":{"score":0.5255,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":431.9},"26":{"score":0.9865,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":357.9},"27":{"score":0.9599,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":441.3},"28":{"score":0.9171,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":352},"29":{"score":0.963,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":437.8},"30":{"score":0.7661,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2554,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":827.8},"32":{"score":0.6609,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":1208.3},"33":{"score":0.3661,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":925.9},"36":{"score":0.4889,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":21350.4},"37":{"score":0.6021,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":540.8},"38":{"score":0.4251,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":6862.5},"39":{"score":0.9106,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":520.9},"40":{"score":0.6866,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":412.7},"41":{"score":0.7023,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":437.9},"42":{"score":0.8178,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":648.9},"43":{"score":0.7368,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":399.9},"44":{"score":0.9084,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":428.9},"45":{"score":0.1198,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":605.8},"50":{"score":0.5955,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":713.9},"56":{"score":0.9245,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":749.8},"57":{"score":0.676,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":419.9},"58":{"score":0.8426,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":386.8},"59":{"score":0.741,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":756},"61":{"score":0.8655,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":527.9},"62":{"score":0.8604,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":882.8},"64":{"score":0.7083,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":347.9}},"benchmarks":{"1":{"raw":0.9717,"skill":0.9617,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9192,"private_eq":0.9401},"2":{"raw":0.6944,"skill":0.6471,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7162,"private_eq":0.7562},"3":{"raw":0.8642,"skill":0.8616,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8804,"private_eq":0.883},"4":{"raw":0.8986,"skill":0.8973,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8752,"private_eq":0.8772},"5":{"raw":0.9342,"skill":0.9338,"coverage":1,"random":0.006,"tracks":[],"combined":0.738,"private_eq":0.7415},"6":{"raw":0.7902,"skill":0.5081,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8493,"private_eq":0.85},"10":{"raw":0.5887,"skill":0.3156,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8482,"skill":0.7804,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7504,"skill":0.6261,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4912,"private_eq":0.6635},"20":{"raw":0.9555,"skill":0.9111,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.6119,"skill":0.2237,"coverage":1,"random":0.5,"tracks":[],"combined":0.176,"private_eq":0.5922},"22":{"raw":0.7729,"skill":0.7711,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5069,"private_eq":0.5153},"23":{"raw":0.6611,"skill":0.3222,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.856,"skill":0.808,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5255,"skill":0.3673,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4545,"combined":0.2653},"26":{"raw":0.9865,"skill":0.982,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9599,"skill":0.9465,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9171,"skill":0.8342,"coverage":1,"random":0.5,"tracks":[],"combined":0.547,"private_eq":0.7756},"29":{"raw":0.963,"skill":0.9507,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7661,"skill":0.7204,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8226,"random":0.25},{"track":"10-choice","raw":0.7096,"random":0.1}],"combined":0.7234,"private_eq":0.7736},"31":{"raw":0.2554,"skill":0.189,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1059,"private_eq":0.1876},"32":{"raw":0.6609,"skill":0.4609,"coverage":1,"random":0.371,"tracks":[],"combined":0.382,"private_eq":0.6151},"33":{"raw":0.3661,"skill":0.3577,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2612,"private_eq":0.2782},"36":{"raw":0.4889,"skill":0.4219,"coverage":1,"random":0.116,"tracks":[],"combined":0.3899,"private_eq":0.4659},"37":{"raw":0.6021,"skill":0.5009,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5641,"private_eq":0.6556},"38":{"raw":0.4251,"skill":0.4067,"coverage":1,"random":0.031,"tracks":[],"combined":0.647,"private_eq":0.6609},"39":{"raw":0.9106,"skill":0.8685,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4924,"private_eq":0.6581},"40":{"raw":0.6866,"skill":0.5969,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0969,"headline":false},{"track":"A · English","score":0.6866,"headline":true},{"track":"C · Arabic pairs","score":0.535,"headline":false},{"track":"C · English pairs","score":0.94,"headline":false}],"combined":0.7162,"private_eq":0.7811},"41":{"raw":0.7023,"skill":0.5535,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5022,"private_eq":0.6712},"42":{"raw":0.8178,"skill":0.6455,"coverage":1,"random":0.486,"tracks":[],"combined":0.9127,"private_eq":0.9551},"43":{"raw":0.7368,"skill":0.5825,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5452,"private_eq":0.716},"44":{"raw":0.9084,"skill":0.8168,"coverage":1,"random":0.5,"tracks":[],"combined":0.7226,"private_eq":0.8624},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5955,"skill":0.4129,"coverage":1,"random":0.311,"tracks":[],"combined":0.2902,"private_eq":0.5158},"56":{"raw":0.9245,"skill":0.849,"coverage":1,"random":0.5,"tracks":[],"combined":0.9838,"private_eq":0.9915},"57":{"raw":0.676,"skill":0.6356,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4658,"combined":0.3933},"58":{"raw":0.8426,"skill":0.7719,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4623,"combined":0.2126},"59":{"raw":0.741,"skill":0.463,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3152,"private_eq":0.673},"61":{"raw":0.8655,"skill":0.731,"coverage":1,"random":0.5,"tracks":[],"combined":0.6549,"private_eq":0.8289},"62":{"raw":0.8604,"skill":0.8139,"coverage":1,"random":0.25,"tracks":[],"combined":0.5425,"private_eq":0.66},"64":{"raw":0.7083,"skill":0.6354,"coverage":1,"random":0.2,"tracks":[],"combined":0.6907,"private_eq":0.7546}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #297).","Submitted public 0.3 run adopted (kit rescore 66.26, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 98.96% on 831 rows (bar 98.5%).","The submitted run used the author's SGLang build; the lab served the repository's reference server.","Training data undisclosed."],"scores":{"balanced_raw":74.68,"balanced_skill":60.14,"breadth_skill":65.34,"public_skill":66.26},"frozen_scores":{"balanced_raw":74.68,"balanced_skill":66.26,"breadth_skill":65.34},"categories":[{"id":"knowledge","raw":0.6159,"skill":0.5215,"combined":0.4113,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7646,"skill":0.6603,"combined":0.64,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7846,"skill":0.7249,"combined":0.6979,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8811,"skill":0.8609,"combined":0.7903,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7175,"skill":0.5461,"combined":0.5229,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"standard-one-35b-sh-v1","pub":66.26,"same_skills":61.77,"new_domains":44.98,"v":60.14,"lab":false,"jev":false,"rank_v":49,"rank_pub":27,"name":"Standard One 35B SH (v1)","eq":{"S":61.43,"O":58.2},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":36.86,"new_domains":23.28}},"new_in_edition":true},{"engine":"crystal-0.8b","name":"crystal-0.8b","long_name":"crystal-0.8b","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":852985920,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Kyunghwan-kim/crystal-0.8b","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"pr":278,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6523,"conf":0.6647,"ece":0.0173,"brier":0.4412,"over95":0.0029,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0542,"conf":0.0521,"acc":0.0418},{"lo":0.1,"w":0.0144,"conf":0.1478,"acc":0.1019},{"lo":0.2,"w":0.014,"conf":0.2626,"acc":0.3772},{"lo":0.3,"w":0.0559,"conf":0.3591,"acc":0.3565},{"lo":0.4,"w":0.0788,"conf":0.4505,"acc":0.4398},{"lo":0.5,"w":0.1576,"conf":0.5504,"acc":0.5231},{"lo":0.6,"w":0.1414,"conf":0.6477,"acc":0.6333},{"lo":0.7,"w":0.1233,"conf":0.7513,"acc":0.7083},{"lo":0.8,"w":0.1387,"conf":0.8519,"acc":0.8444},{"lo":0.9,"w":0.2216,"conf":0.954,"acc":0.9577}],"areas":{"knowledge":{"ece":0.0663,"acc":0.5267,"conf":0.5907},"language":{"ece":0.0123,"acc":0.7576,"conf":0.7638},"retrieval":{"ece":0.0524,"acc":0.7056,"conf":0.6563},"tools":{"ece":0.0303,"acc":0.8709,"conf":0.8528},"arts":{"ece":0.0213,"acc":0.5399,"conf":0.5284}}},"gaps":null,"latency":{"median":53.8,"p95":133.3,"mean":71.6},"results":{"1":{"score":0.8961,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":50},"2":{"score":0.6363,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":392.7},"3":{"score":0.8209,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":257.2},"4":{"score":0.8938,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":32.9},"5":{"score":0.8902,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":50.9},"6":{"score":0.7118,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":136.4},"9":{"score":0.1477,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":541},"10":{"score":0.389,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":37.6},"11":{"score":0.8376,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":937.5},"12":{"score":0.5008,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31.2},"20":{"score":0.8392,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.2},"21":{"score":0.6218,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":31},"22":{"score":0.016,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":212.9},"23":{"score":0.5854,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.4},"24":{"score":0.5538,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":31},"25":{"score":0.3418,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":31.3},"26":{"score":0.907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":31},"27":{"score":0.7824,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":30.9},"28":{"score":0.6938,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":31},"29":{"score":0.8231,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":31.5},"30":{"score":0.376,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0894,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":33.2},"32":{"score":0.5851,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":33.7},"33":{"score":0.2248,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":105.2},"36":{"score":0.3726,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":635.3},"37":{"score":0.5491,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.6},"38":{"score":0.3143,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":363.3},"39":{"score":0.9029,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":31.9},"40":{"score":0.2379,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":31},"41":{"score":0.4265,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":31.2},"42":{"score":0.6843,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":31.8},"43":{"score":0.3018,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":31.1},"44":{"score":0.5892,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.1},"45":{"score":0.1277,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":31.3},"50":{"score":0.58,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":33},"56":{"score":0.5195,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":96.7},"57":{"score":0.3412,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":31.2},"58":{"score":0.5251,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":31.1},"59":{"score":0.7723,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":32.7},"61":{"score":0.8482,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":31.7},"62":{"score":0.8234,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":32.5},"64":{"score":0.6117,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":31.2}},"benchmarks":{"1":{"raw":0.8961,"skill":0.8597,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8581,"private_eq":0.8953},"2":{"raw":0.6363,"skill":0.58,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6716,"private_eq":0.718},"3":{"raw":0.8209,"skill":0.8174,"coverage":1,"random":0.0189,"tracks":[],"combined":0.5496,"private_eq":0.5622},"4":{"raw":0.8938,"skill":0.8924,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7828,"private_eq":0.787},"5":{"raw":0.8902,"skill":0.8895,"coverage":1,"random":0.006,"tracks":[],"combined":0.7452,"private_eq":0.7486},"6":{"raw":0.7118,"skill":0.3243,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1477,"skill":0.1477,"coverage":1,"random":0,"tracks":[],"combined":0.0527,"private_eq":0.0625},"10":{"raw":0.389,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8376,"skill":0.7652,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5008,"skill":0.2522,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4719,"private_eq":0.6508},"20":{"raw":0.8381,"skill":0.6763,"coverage":1,"random":0.5,"tracks":[],"combined":0.7914,"private_eq":0.8964},"21":{"raw":0.6218,"skill":0.2435,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.0141,"skill":0.0064,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0134},"23":{"raw":0.5855,"skill":0.1709,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5538,"skill":0.4051,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3418,"skill":0.1224,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2914,"combined":0.0453},"26":{"raw":0.907,"skill":0.876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7824,"skill":0.7098,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6938,"skill":0.3876,"coverage":1,"random":0.5,"tracks":[],"combined":0.1586,"private_eq":0.5836},"29":{"raw":0.8231,"skill":0.7641,"coverage":1,"random":0.25,"tracks":[],"combined":0.4781,"private_eq":0.6123},"30":{"raw":0.376,"skill":0.2497,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4867,"random":0.25},{"track":"10-choice","raw":0.2654,"random":0.1}],"combined":0.3193,"private_eq":0.444},"31":{"raw":0.0894,"skill":0.0082,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0138,"private_eq":0.104},"32":{"raw":0.5851,"skill":0.3404,"coverage":1,"random":0.371,"tracks":[],"combined":0.0262,"private_eq":0.3938},"33":{"raw":0.2248,"skill":0.2145,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2098,"private_eq":0.228},"36":{"raw":0.3726,"skill":0.2902,"coverage":1,"random":0.116,"tracks":[],"combined":0.2595,"private_eq":0.352},"37":{"raw":0.5491,"skill":0.4344,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4548,"private_eq":0.5695},"38":{"raw":0.3143,"skill":0.2924,"coverage":1,"random":0.031,"tracks":[],"combined":0.4967,"private_eq":0.5169},"39":{"raw":0.9029,"skill":0.8572,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4782,"private_eq":0.6486},"40":{"raw":0.2379,"skill":0.0195,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3882,"headline":false},{"track":"A · English","score":0.2379,"headline":true},{"track":"C · Arabic pairs","score":0.885,"headline":false},{"track":"C · English pairs","score":0.87,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4265,"skill":0.1398,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4703,"private_eq":0.6502},"42":{"raw":0.6843,"skill":0.3858,"coverage":1,"random":0.486,"tracks":[],"combined":0.523,"private_eq":0.7571},"43":{"raw":0.3018,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2443,"private_eq":0.5285},"44":{"raw":0.5892,"skill":0.1784,"coverage":1,"random":0.5,"tracks":[],"combined":0.2567,"private_eq":0.6321},"45":{"raw":0.1277,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.58,"skill":0.3904,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.5195,"skill":0.039,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5038},"57":{"raw":0.3412,"skill":0.259,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4629,"combined":0.39},"58":{"raw":0.5251,"skill":0.3116,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3488,"combined":0.0462},"59":{"raw":0.7723,"skill":0.5279,"coverage":1,"random":0.5177,"tracks":[],"combined":0.241,"private_eq":0.6376},"61":{"raw":0.8482,"skill":0.6964,"coverage":1,"random":0.5,"tracks":[],"combined":0.5924,"private_eq":0.798},"62":{"raw":0.8234,"skill":0.7645,"coverage":1,"random":0.25,"tracks":[],"combined":0.6108,"private_eq":0.7107},"64":{"raw":0.6117,"skill":0.5146,"coverage":1,"random":0.2,"tracks":[],"combined":0.5064,"private_eq":0.6088}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #278).","Submitted public 0.3 run adopted (kit rescore 42.79, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.75% on 831 rows (bar 98.5%).","Released as a weight merge of the base and two fine-tunes.","Training includes train splits of public datasets that are also sources of index benchmarks (author-declared)."],"scores":{"balanced_raw":57.26,"balanced_skill":29.37,"breadth_skill":39.91,"public_skill":42.79},"frozen_scores":{"balanced_raw":57.26,"balanced_skill":42.79,"breadth_skill":39.91},"categories":[{"id":"knowledge","raw":0.3835,"skill":0.1907,"combined":0.1702,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6132,"skill":0.4476,"combined":0.3975,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6825,"skill":0.554,"combined":0.4833,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6792,"skill":0.649,"combined":0.5601,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5419,"skill":0.3337,"combined":0.3541,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"crystal-0.8b","pub":42.79,"same_skills":39.31,"new_domains":10.29,"v":29.37,"lab":false,"jev":false,"rank_v":170,"rank_pub":142,"name":"crystal-0.8b","eq":{"S":38.72,"O":15.35},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":23.23,"new_domains":6.14}},"new_in_edition":true},{"engine":"jev-ettin-150m","name":"JEV Ettin 150M","long_name":"JEV Ettin 150M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":149610241,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Elfsong/jev-ettin-150m-v16","code_url":null,"base_model":"cross-encoder/ettin-reranker-150m-v1","base_url":"https://huggingface.co/cross-encoder/ettin-reranker-150m-v1","weights_repo":null,"served_checkpoint":null,"pr":277,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5942,"conf":0.7229,"ece":0.1287,"brier":0.5362,"over95":0.0398,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0131,"conf":0.0808,"acc":0.0692},{"lo":0.1,"w":0.0358,"conf":0.1481,"acc":0.0337},{"lo":0.2,"w":0.0304,"conf":0.2516,"acc":0.1441},{"lo":0.3,"w":0.0561,"conf":0.3511,"acc":0.3261},{"lo":0.4,"w":0.0662,"conf":0.4497,"acc":0.3433},{"lo":0.5,"w":0.1271,"conf":0.5494,"acc":0.4603},{"lo":0.6,"w":0.1045,"conf":0.6471,"acc":0.5291},{"lo":0.7,"w":0.0905,"conf":0.7493,"acc":0.5912},{"lo":0.8,"w":0.1012,"conf":0.8518,"acc":0.622},{"lo":0.9,"w":0.375,"conf":0.9812,"acc":0.8439}],"areas":{"knowledge":{"ece":0.1259,"acc":0.4688,"conf":0.5947},"language":{"ece":0.0809,"acc":0.7586,"conf":0.8394},"retrieval":{"ece":0.1406,"acc":0.7046,"conf":0.8452},"tools":{"ece":0.2011,"acc":0.7409,"conf":0.9394},"arts":{"ece":0.1688,"acc":0.373,"conf":0.5418}}},"gaps":null,"latency":{"median":77.1,"p95":151.5,"mean":89.6},"results":{"1":{"score":0.7627,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":10.2},"2":{"score":0.6097,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":58.6},"3":{"score":0.4173,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":47.7},"4":{"score":0.9184,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":8},"5":{"score":0.9255,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":9.3},"6":{"score":0.673,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":17.1},"9":{"score":0.8182,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":50.1},"10":{"score":0.3715,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":8.1},"11":{"score":0.8139,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":138.2},"12":{"score":0.475,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":6.1},"20":{"score":0.4232,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.1},"21":{"score":0.5122,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":6},"22":{"score":0.028,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":31.4},"23":{"score":0.4876,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.3},"24":{"score":0.3676,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":6.1},"25":{"score":0.2602,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":6.2},"26":{"score":0.697,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":6.1},"27":{"score":0.4991,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":6},"28":{"score":0.6369,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":6},"29":{"score":0.6425,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":6.3},"30":{"score":0.1827,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.086,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.3},"32":{"score":0.5399,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":8},"33":{"score":0.0509,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":14},"36":{"score":0.2836,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":113.4},"37":{"score":0.4383,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.5},"38":{"score":0.4454,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":30.3},"39":{"score":0.8554,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":6.4},"40":{"score":0.1871,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":6.2},"41":{"score":0.7404,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":6.3},"42":{"score":0.5825,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":6.5},"43":{"score":0.4526,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":6.1},"44":{"score":0.8798,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.1},"45":{"score":0.1457,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":6.3},"50":{"score":0.3848,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":7.6},"56":{"score":0.5015,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":9.7},"57":{"score":0.1519,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":6.3},"58":{"score":0.3802,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":6.2},"59":{"score":0.7254,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":7.2},"61":{"score":0.8522,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":6.5},"62":{"score":0.6936,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":7},"64":{"score":0.5265,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":6.2}},"benchmarks":{"1":{"raw":0.7627,"skill":0.6796,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6332,"private_eq":0.7306},"2":{"raw":0.6097,"skill":0.5493,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6076,"private_eq":0.6632},"3":{"raw":0.4173,"skill":0.4061,"coverage":1,"random":0.0189,"tracks":[],"combined":0.3292,"private_eq":0.3484},"4":{"raw":0.9184,"skill":0.9174,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7107,"private_eq":0.7166},"5":{"raw":0.9255,"skill":0.9251,"coverage":1,"random":0.006,"tracks":[],"combined":0.7094,"private_eq":0.7134},"6":{"raw":0.673,"skill":0.2333,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8182,"skill":0.8182,"coverage":1,"random":0,"tracks":[],"combined":0.3385,"private_eq":0.345},"10":{"raw":0.3715,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8139,"skill":0.7309,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.475,"skill":0.2136,"coverage":1,"random":0.3324,"tracks":[],"combined":0.2039,"private_eq":0.4739},"20":{"raw":0.4243,"skill":0,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"21":{"raw":0.5122,"skill":0.0244,"coverage":1,"random":0.5,"tracks":[],"combined":0.0656,"private_eq":0.5376},"22":{"raw":0.029,"skill":0.0214,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.4656,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3676,"skill":0.1568,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.25,"combined":0},"26":{"raw":0.697,"skill":0.5959,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.4991,"skill":0.332,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6369,"skill":0.2738,"coverage":1,"random":0.5,"tracks":[],"combined":0.2242,"private_eq":0.616},"29":{"raw":0.6425,"skill":0.5233,"coverage":1,"random":0.25,"tracks":[],"combined":0.2406,"private_eq":0.4362},"30":{"raw":0.1827,"skill":0.0096,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2593,"random":0.25},{"track":"10-choice","raw":0.1061,"random":0.1}],"combined":0.0372,"private_eq":0.2139},"31":{"raw":0.086,"skill":0.0045,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0019,"private_eq":0.0932},"32":{"raw":0.5399,"skill":0.2685,"coverage":1,"random":0.371,"tracks":[],"combined":0.003,"private_eq":0.3794},"33":{"raw":0.0509,"skill":0.0383,"coverage":1,"random":0.0131,"tracks":[],"combined":0,"private_eq":0.0175},"36":{"raw":0.2836,"skill":0.1896,"coverage":1,"random":0.116,"tracks":[],"combined":0.2456,"private_eq":0.3398},"37":{"raw":0.4383,"skill":0.2955,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3554,"private_eq":0.4911},"38":{"raw":0.4454,"skill":0.4277,"coverage":1,"random":0.031,"tracks":[],"combined":0.3896,"private_eq":0.4143},"39":{"raw":0.8554,"skill":0.7873,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4148,"private_eq":0.606},"40":{"raw":0.1871,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0284,"headline":false},{"track":"A · English","score":0.1871,"headline":true},{"track":"C · Arabic pairs","score":0.55,"headline":false},{"track":"C · English pairs","score":0.465,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.7404,"skill":0.6106,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.5825,"skill":0.1877,"coverage":1,"random":0.486,"tracks":[],"combined":0,"private_eq":0.486},"43":{"raw":0.4526,"skill":0.1315,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0018,"private_eq":0.3774},"44":{"raw":0.8798,"skill":0.7596,"coverage":1,"random":0.5,"tracks":[],"combined":0.6316,"private_eq":0.8174},"45":{"raw":0.1457,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3848,"skill":0.1072,"coverage":1,"random":0.311,"tracks":[],"combined":0.1162,"private_eq":0.3973},"56":{"raw":0.5015,"skill":0.003,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"57":{"raw":0.1519,"skill":0.0461,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1397,"combined":0.0223},"58":{"raw":0.3802,"skill":0.1016,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3363,"combined":0.0279},"59":{"raw":0.7254,"skill":0.4306,"coverage":1,"random":0.5177,"tracks":[],"combined":0.0216,"private_eq":0.533},"61":{"raw":0.8522,"skill":0.7044,"coverage":1,"random":0.5,"tracks":[],"combined":0.4621,"private_eq":0.7336},"62":{"raw":0.6936,"skill":0.5915,"coverage":1,"random":0.25,"tracks":[],"combined":0.1387,"private_eq":0.3606},"64":{"raw":0.5265,"skill":0.4081,"coverage":1,"random":0.2,"tracks":[],"combined":0.4968,"private_eq":0.6012}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #277).","Submitted public 0.3 run adopted (kit rescore 37.31, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 100.00% on 831 rows (bar 98.5%).","Uses the Crivo typed-decision inference code.","Training includes train splits of public datasets that are also sources of index benchmarks (author-declared)."],"scores":{"balanced_raw":53.5,"balanced_skill":18.52,"breadth_skill":32.84,"public_skill":37.31},"frozen_scores":{"balanced_raw":53.5,"balanced_skill":37.31,"breadth_skill":32.84},"categories":[{"id":"knowledge","raw":0.3401,"skill":0.1445,"combined":0.0913,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6054,"skill":0.4318,"combined":0.1581,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6583,"skill":0.5214,"combined":0.4251,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6551,"skill":0.604,"combined":0.4147,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3904,"skill":0.0935,"combined":0.1315,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-ettin-150m","pub":37.31,"same_skills":25.45,"new_domains":5.36,"v":18.52,"lab":false,"jev":false,"rank_v":199,"rank_pub":169,"name":"JEV Ettin 150M","eq":{"S":24.69,"O":9.27},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":14.82,"new_domains":3.71}},"new_in_edition":true},{"engine":"valen-0.8b-r2","name":"Valen-0.8B","long_name":"Valen-0.8B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":854305641,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Valen-Team/Valen-0.8B","code_url":null,"base_model":"Qwen/Qwen3.5-0.8B","base_url":"https://huggingface.co/Qwen/Qwen3.5-0.8B","weights_repo":null,"served_checkpoint":null,"org":{"id":"Valen-Team","name":"Valen","logo":"data/orgs/Valen-Team.png"},"pr":311,"replaces":"valen-0.8b","new":true},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.5279,"conf":0.6363,"ece":0.1084,"brier":0.5956,"over95":0.0225,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0536,"conf":0.0704,"acc":0.0358},{"lo":0.1,"w":0.044,"conf":0.1537,"acc":0.1087},{"lo":0.2,"w":0.0343,"conf":0.2478,"acc":0.2421},{"lo":0.3,"w":0.0625,"conf":0.3537,"acc":0.3027},{"lo":0.4,"w":0.0911,"conf":0.4519,"acc":0.4043},{"lo":0.5,"w":0.142,"conf":0.5469,"acc":0.4954},{"lo":0.6,"w":0.1208,"conf":0.65,"acc":0.5424},{"lo":0.7,"w":0.1096,"conf":0.751,"acc":0.5726},{"lo":0.8,"w":0.1258,"conf":0.8517,"acc":0.6657},{"lo":0.9,"w":0.2162,"conf":0.9634,"acc":0.8082}],"areas":{"knowledge":{"ece":0.1204,"acc":0.482,"conf":0.6011},"language":{"ece":0.0885,"acc":0.6475,"conf":0.736},"retrieval":{"ece":0.1065,"acc":0.4945,"conf":0.5475},"tools":{"ece":0.2016,"acc":0.5623,"conf":0.7311},"arts":{"ece":0.1307,"acc":0.4106,"conf":0.5413}}},"gaps":null,"latency":{"median":200.1,"p95":556.2,"mean":262},"results":{"1":{"score":0.6954,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":114},"2":{"score":0.5461,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":574.9},"3":{"score":0.4606,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":431},"4":{"score":0.6004,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":124.2},"5":{"score":0.1784,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":156.3},"6":{"score":0.7697,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":203.9},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":216.6},"10":{"score":0.5786,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":131.4},"11":{"score":0.7833,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":269.7},"12":{"score":0.4016,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":85},"20":{"score":0.5064,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":85.7},"21":{"score":0.551,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":81},"22":{"score":0.009,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":372.2},"23":{"score":0.5858,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":101.1},"24":{"score":0.5024,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":83.2},"25":{"score":0.2602,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":88.4},"26":{"score":0.8531,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":81.3},"27":{"score":0.7048,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":81},"28":{"score":0.5588,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":80.7},"29":{"score":0.4387,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":88.9},"30":{"score":0.2684,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0738,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":121.5},"32":{"score":0.5319,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":130.2},"33":{"score":0.1018,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":131.8},"36":{"score":0.285,"jev":null,"metric":"nDCG@10","requests":220,"answered":219,"unsupported":1,"errors":0,"median_ms":820.4},"37":{"score":0.2764,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":95.7},"38":{"score":0.1231,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":500},"39":{"score":0.7488,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":89.3},"40":{"score":0.1472,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":82},"41":{"score":0.4343,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":86.3},"42":{"score":0.6463,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":103.4},"43":{"score":0.3123,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":85.5},"44":{"score":0.548,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":85.1},"45":{"score":0.1238,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":90.1},"50":{"score":0.4487,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":123.4},"56":{"score":0.5115,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":128.2},"57":{"score":0.2893,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":88.4},"58":{"score":0.4614,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":85.9},"59":{"score":0.5309,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":123},"61":{"score":0.5473,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":101.8},"62":{"score":0.4014,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":114.1},"64":{"score":0.3769,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":86.3}},"benchmarks":{"1":{"raw":0.6954,"skill":0.5888,"coverage":1,"random":0.2592,"tracks":[],"combined":0.4765,"private_eq":0.6159},"2":{"raw":0.5461,"skill":0.4758,"coverage":1,"random":0.1341,"tracks":[],"combined":0.5681,"private_eq":0.6294},"3":{"raw":0.4606,"skill":0.4502,"coverage":1,"random":0.0189,"tracks":[],"combined":0.1351,"private_eq":0.1601},"4":{"raw":0.6004,"skill":0.5953,"coverage":1,"random":0.0127,"tracks":[],"combined":0.5728,"private_eq":0.582},"5":{"raw":0.1784,"skill":0.1734,"coverage":1,"random":0.006,"tracks":[],"combined":0.1348,"private_eq":0.1488},"6":{"raw":0.7697,"skill":0.46,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0},"10":{"raw":0.5786,"skill":0.2988,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7833,"skill":0.6866,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4016,"skill":0.1037,"coverage":1,"random":0.3324,"tracks":[],"combined":0.0813,"private_eq":0.393},"20":{"raw":0.5052,"skill":0.0105,"coverage":1,"random":0.5,"tracks":[],"combined":0.0757,"private_eq":0.5426},"21":{"raw":0.551,"skill":0.102,"coverage":1,"random":0.5,"tracks":[],"combined":0.1258,"private_eq":0.5674},"22":{"raw":0.0088,"skill":0.001,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0078},"23":{"raw":0.5679,"skill":0.1358,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5024,"skill":0.3365,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2602,"skill":0.0136,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2851,"combined":0.0368},"26":{"raw":0.8531,"skill":0.8041,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7048,"skill":0.6063,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5588,"skill":0.1176,"coverage":1,"random":0.5,"tracks":[],"combined":0.0073,"private_eq":0.5088},"29":{"raw":0.4387,"skill":0.2516,"coverage":1,"random":0.25,"tracks":[],"combined":0.3693,"private_eq":0.5316},"30":{"raw":0.2684,"skill":0.1166,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3654,"random":0.25},{"track":"10-choice","raw":0.1713,"random":0.1}],"combined":0.2733,"private_eq":0.4065},"31":{"raw":0.0738,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0051,"private_eq":0.0961},"32":{"raw":0.5319,"skill":0.2558,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1018,"skill":0.0899,"coverage":1,"random":0.0131,"tracks":[],"combined":0.143,"private_eq":0.1629},"36":{"raw":0.2837,"skill":0.1897,"coverage":0.9955,"random":0.116,"tracks":[],"combined":0.16,"private_eq":0.265},"37":{"raw":0.2764,"skill":0.0923,"coverage":1,"random":0.2027,"tracks":[],"combined":0.0797,"private_eq":0.2738},"38":{"raw":0.1231,"skill":0.095,"coverage":1,"random":0.031,"tracks":[],"combined":0.1608,"private_eq":0.1951},"39":{"raw":0.7488,"skill":0.6305,"coverage":1,"random":0.3201,"tracks":[],"combined":0.465,"private_eq":0.6397},"40":{"raw":0.1472,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2129,"headline":false},{"track":"A · English","score":0.1472,"headline":true},{"track":"C · Arabic pairs","score":0.48,"headline":false},{"track":"C · English pairs","score":0.515,"headline":false}],"combined":0,"private_eq":0.2227},"41":{"raw":0.4343,"skill":0.1515,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3137,"private_eq":0.547},"42":{"raw":0.6463,"skill":0.3119,"coverage":1,"random":0.486,"tracks":[],"combined":0.2103,"private_eq":0.5982},"43":{"raw":0.3123,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2527,"private_eq":0.5337},"44":{"raw":0.548,"skill":0.096,"coverage":1,"random":0.5,"tracks":[],"combined":0.0747,"private_eq":0.5421},"45":{"raw":0.1238,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4487,"skill":0.1999,"coverage":1,"random":0.311,"tracks":[],"combined":0.1379,"private_eq":0.4121},"56":{"raw":0.5115,"skill":0.023,"coverage":1,"random":0.5,"tracks":[],"combined":0.1703,"private_eq":0.5894},"57":{"raw":0.2893,"skill":0.2007,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3487,"combined":0.26},"58":{"raw":0.4614,"skill":0.2193,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3352,"combined":0.0263},"59":{"raw":0.5309,"skill":0.0274,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5473,"skill":0.0946,"coverage":1,"random":0.5,"tracks":[],"combined":0.2011,"private_eq":0.6046},"62":{"raw":0.4014,"skill":0.2019,"coverage":1,"random":0.25,"tracks":[],"combined":0.1728,"private_eq":0.3859},"64":{"raw":0.3769,"skill":0.2211,"coverage":1,"random":0.2,"tracks":[],"combined":0.2349,"private_eq":0.3941}},"coverage":0.9998,"pending":0.0002,"notes":["New in the 2026-10-27 PR batch (PR #311).","Replaces the earlier board entry valen-0.8b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 20.79, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.69% on 827 rows (bar 98.5%).","Same weights as the earlier Valen-0.8B entry, served with the author's updated engine."],"scores":{"balanced_raw":40.41,"balanced_skill":16.55,"breadth_skill":19.5,"public_skill":20.79},"frozen_scores":{"balanced_raw":40.41,"balanced_skill":20.79,"breadth_skill":19.5},"categories":[{"id":"knowledge","raw":0.3224,"skill":0.1016,"combined":0.1051,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4705,"skill":0.2478,"combined":0.1987,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.3955,"skill":0.2061,"combined":0.2261,"coverage":0.9992,"pending":0,"n":6},{"id":"tools","raw":0.4324,"skill":0.3564,"combined":0.2712,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4097,"skill":0.1117,"combined":0.1127,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"valen-0.8b-r2","pub":20.79,"same_skills":19.6,"new_domains":8.57,"v":16.55,"lab":false,"jev":false,"rank_v":208,"rank_pub":211,"name":"Valen-0.8B","eq":{"S":18.77,"O":13.23},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":11.26,"new_domains":5.29}},"new_in_edition":true},{"engine":"valen-2b-r2","name":"Valen-2B","long_name":"Valen-2B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2214825577,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Valen-Team/Valen-2B","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"org":{"id":"Valen-Team","name":"Valen","logo":"data/orgs/Valen-Team.png"},"pr":311,"replaces":"valen-2b","new":true},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.5763,"conf":0.6658,"ece":0.1053,"brier":0.5528,"over95":0.0271,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.0565,"conf":0.0769,"acc":0.0449},{"lo":0.1,"w":0.0452,"conf":0.1316,"acc":0.2583},{"lo":0.2,"w":0.0211,"conf":0.254,"acc":0.3554},{"lo":0.3,"w":0.0485,"conf":0.3555,"acc":0.3409},{"lo":0.4,"w":0.0763,"conf":0.4509,"acc":0.3685},{"lo":0.5,"w":0.1321,"conf":0.5495,"acc":0.5116},{"lo":0.6,"w":0.1222,"conf":0.6486,"acc":0.5639},{"lo":0.7,"w":0.1062,"conf":0.7492,"acc":0.5983},{"lo":0.8,"w":0.1096,"conf":0.8521,"acc":0.6634},{"lo":0.9,"w":0.2822,"conf":0.9699,"acc":0.8403}],"areas":{"knowledge":{"ece":0.1197,"acc":0.5257,"conf":0.6454},"language":{"ece":0.0976,"acc":0.6746,"conf":0.7692},"retrieval":{"ece":0.2307,"acc":0.5634,"conf":0.5648},"tools":{"ece":0.0976,"acc":0.7015,"conf":0.7903},"arts":{"ece":0.0913,"acc":0.4356,"conf":0.5268}}},"gaps":null,"latency":{"median":160.7,"p95":421.8,"mean":223.3},"results":{"1":{"score":0.8477,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":112.5},"2":{"score":0.5803,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":588.4},"3":{"score":0.5276,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":442.8},"4":{"score":0.5743,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":126.2},"5":{"score":0.397,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":157.4},"6":{"score":0.7817,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":205.7},"9":{"score":0.0455,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":214.7},"10":{"score":0.5462,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":130.6},"11":{"score":0.7838,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":278.4},"12":{"score":0.4828,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":86.8},"20":{"score":0.673,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":87.1},"21":{"score":0.5525,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":82.7},"22":{"score":0.003,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":388.9},"23":{"score":0.5906,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":102.1},"24":{"score":0.5793,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":84.4},"25":{"score":0.3367,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":88},"26":{"score":0.9364,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":82.8},"27":{"score":0.8217,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":83.4},"28":{"score":0.5722,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":83},"29":{"score":0.5763,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":89.3},"30":{"score":0.2873,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":123},"32":{"score":0.5931,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":131.5},"33":{"score":0.1048,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":131.9},"36":{"score":0.3402,"jev":null,"metric":"nDCG@10","requests":220,"answered":219,"unsupported":1,"errors":0,"median_ms":854.3},"37":{"score":0.3108,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":97.8},"38":{"score":0.1791,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":519.4},"39":{"score":0.6593,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":91.9},"40":{"score":0.262,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":84.7},"41":{"score":0.5046,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":88.5},"42":{"score":0.7002,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":105.9},"43":{"score":0.3596,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":87.7},"44":{"score":0.5792,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":87.4},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":92.1},"50":{"score":0.4135,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":125.7},"56":{"score":0.5055,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":130.2},"57":{"score":0.3721,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":89.9},"58":{"score":0.5124,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":87.7},"59":{"score":0.5008,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":124.8},"61":{"score":0.5617,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":102.3},"62":{"score":0.4751,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":115},"64":{"score":0.4375,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":86.9}},"benchmarks":{"1":{"raw":0.8477,"skill":0.7944,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6808,"private_eq":0.7655},"2":{"raw":0.5803,"skill":0.5153,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6258,"private_eq":0.6788},"3":{"raw":0.5276,"skill":0.5185,"coverage":1,"random":0.0189,"tracks":[],"combined":0.156,"private_eq":0.1804},"4":{"raw":0.5743,"skill":0.5688,"coverage":1,"random":0.0127,"tracks":[],"combined":0.4285,"private_eq":0.4412},"5":{"raw":0.397,"skill":0.3934,"coverage":1,"random":0.006,"tracks":[],"combined":0.1016,"private_eq":0.1162},"6":{"raw":0.7817,"skill":0.4882,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0455,"skill":0.0455,"coverage":1,"random":0,"tracks":[],"combined":0,"private_eq":0.0075},"10":{"raw":0.5462,"skill":0.2449,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7838,"skill":0.6873,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4828,"skill":0.2253,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3683,"private_eq":0.5824},"20":{"raw":0.6698,"skill":0.3397,"coverage":1,"random":0.5,"tracks":[],"combined":0.4588,"private_eq":0.732},"21":{"raw":0.5525,"skill":0.105,"coverage":1,"random":0.5,"tracks":[],"combined":0,"private_eq":0.5},"22":{"raw":0.0051,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"combined":0,"private_eq":0.0144},"23":{"raw":0.5873,"skill":0.1745,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.5793,"skill":0.4391,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3367,"skill":0.1156,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2945,"combined":0.0495},"26":{"raw":0.9364,"skill":0.9152,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8217,"skill":0.7622,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5722,"skill":0.1444,"coverage":1,"random":0.5,"tracks":[],"combined":0.0779,"private_eq":0.5437},"29":{"raw":0.5763,"skill":0.4351,"coverage":1,"random":0.25,"tracks":[],"combined":0.3396,"private_eq":0.5096},"30":{"raw":0.2873,"skill":0.1402,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3882,"random":0.25},{"track":"10-choice","raw":0.1865,"random":0.1}],"combined":0.4402,"private_eq":0.5426},"31":{"raw":0.1,"skill":0.0198,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0127,"private_eq":0.103},"32":{"raw":0.5931,"skill":0.3531,"coverage":1,"random":0.371,"tracks":[],"combined":0.0418,"private_eq":0.4035},"33":{"raw":0.1048,"skill":0.0929,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2304,"private_eq":0.2481},"36":{"raw":0.3387,"skill":0.2519,"coverage":0.9955,"random":0.116,"tracks":[],"combined":0.23,"private_eq":0.3262},"37":{"raw":0.3108,"skill":0.1356,"coverage":1,"random":0.2027,"tracks":[],"combined":0.1813,"private_eq":0.3539},"38":{"raw":0.1791,"skill":0.1528,"coverage":1,"random":0.031,"tracks":[],"combined":0.2432,"private_eq":0.274},"39":{"raw":0.6593,"skill":0.4989,"coverage":1,"random":0.3201,"tracks":[],"combined":0.3788,"private_eq":0.5818},"40":{"raw":0.262,"skill":0.0506,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3285,"headline":false},{"track":"A · English","score":0.262,"headline":true},{"track":"C · Arabic pairs","score":0.425,"headline":false},{"track":"C · English pairs","score":0.675,"headline":false}],"combined":0.0434,"private_eq":0.2641},"41":{"raw":0.5046,"skill":0.2569,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1723,"private_eq":0.4538},"42":{"raw":0.7002,"skill":0.4167,"coverage":1,"random":0.486,"tracks":[],"combined":0.6381,"private_eq":0.8156},"43":{"raw":0.3596,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2652,"private_eq":0.5415},"44":{"raw":0.5792,"skill":0.1584,"coverage":1,"random":0.5,"tracks":[],"combined":0.155,"private_eq":0.5818},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4135,"skill":0.1488,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.5055,"skill":0.011,"coverage":1,"random":0.5,"tracks":[],"combined":0.0146,"private_eq":0.5124},"57":{"raw":0.3721,"skill":0.2938,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3633,"combined":0.2766},"58":{"raw":0.5124,"skill":0.2932,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3429,"combined":0.0376},"59":{"raw":0.5008,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"combined":0,"private_eq":0.5177},"61":{"raw":0.5617,"skill":0.1234,"coverage":1,"random":0.5,"tracks":[],"combined":0.1443,"private_eq":0.5765},"62":{"raw":0.4751,"skill":0.3001,"coverage":1,"random":0.25,"tracks":[],"combined":0.2892,"private_eq":0.4722},"64":{"raw":0.4375,"skill":0.2969,"coverage":1,"random":0.2,"tracks":[],"combined":0.3319,"private_eq":0.4708}},"coverage":0.9998,"pending":0.0002,"notes":["New in the 2026-10-27 PR batch (PR #311).","Replaces the earlier board entry valen-2b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 26.95, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.74% on 827 rows (bar 98.5%).","Same weights as the earlier Valen-2B entry, served with the author's updated engine."],"scores":{"balanced_raw":45.37,"balanced_skill":22.2,"breadth_skill":25.56,"public_skill":26.95},"frozen_scores":{"balanced_raw":45.37,"balanced_skill":26.95,"breadth_skill":25.56},"categories":[{"id":"knowledge","raw":0.3598,"skill":0.1484,"combined":0.1551,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5171,"skill":0.3038,"combined":0.2756,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.447,"skill":0.2617,"combined":0.1897,"coverage":0.9992,"pending":0,"n":6},{"id":"tools","raw":0.5095,"skill":0.4512,"combined":0.3549,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4443,"skill":0.1775,"combined":0.193,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"valen-2b-r2","pub":26.95,"same_skills":24.29,"new_domains":14.23,"v":22.2,"lab":false,"jev":false,"rank_v":189,"rank_pub":201,"name":"Valen-2B","eq":{"S":23.51,"O":20.22},"rank_tie":43,"tied":true,"parts":{"public":0,"same_skills":14.11,"new_domains":8.09}},"new_in_edition":true},{"engine":"valen-4b-r2","name":"Valen-4B","long_name":"Valen-4B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4540981545,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Valen-Team/Valen-4B","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"org":{"id":"Valen-Team","name":"Valen","logo":"data/orgs/Valen-Team.png"},"pr":311,"replaces":"valen-4b","new":true},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.6411,"conf":0.748,"ece":0.107,"brier":0.4815,"over95":0.0352,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.005,"conf":0.0859,"acc":0.0359},{"lo":0.1,"w":0.0321,"conf":0.1474,"acc":0.0541},{"lo":0.2,"w":0.0262,"conf":0.2519,"acc":0.1294},{"lo":0.3,"w":0.041,"conf":0.3568,"acc":0.2298},{"lo":0.4,"w":0.062,"conf":0.4514,"acc":0.4158},{"lo":0.5,"w":0.1068,"conf":0.5493,"acc":0.4795},{"lo":0.6,"w":0.1029,"conf":0.6489,"acc":0.5388},{"lo":0.7,"w":0.1072,"conf":0.7501,"acc":0.5849},{"lo":0.8,"w":0.1206,"conf":0.8523,"acc":0.7199},{"lo":0.9,"w":0.3962,"conf":0.9716,"acc":0.8691}],"areas":{"knowledge":{"ece":0.1128,"acc":0.5855,"conf":0.6983},"language":{"ece":0.0735,"acc":0.7534,"conf":0.8267},"retrieval":{"ece":0.1567,"acc":0.6362,"conf":0.771},"tools":{"ece":0.1268,"acc":0.827,"conf":0.9446},"arts":{"ece":0.1272,"acc":0.4423,"conf":0.5695}}},"gaps":null,"latency":{"median":236.3,"p95":714.8,"mean":347.1},"results":{"1":{"score":0.8725,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":148.8},"2":{"score":0.6156,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":858.9},"3":{"score":0.7087,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":635.4},"4":{"score":0.5965,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":159.7},"5":{"score":0.7126,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":196.4},"6":{"score":0.7987,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":276.8},"9":{"score":0.0795,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":283.9},"10":{"score":0.5223,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":166.2},"11":{"score":0.7361,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":388.3},"12":{"score":0.5807,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":114.7},"20":{"score":0.5044,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":115.1},"21":{"score":0.575,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":110},"22":{"score":0.095,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":554.6},"23":{"score":0.592,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":134},"24":{"score":0.6913,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":112.8},"25":{"score":0.4286,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":119.6},"26":{"score":0.9655,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":110.6},"27":{"score":0.8993,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":111.6},"28":{"score":0.6969,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":110.3},"29":{"score":0.7705,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":119.3},"30":{"score":0.4503,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1058,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":157.3},"32":{"score":0.5439,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":165.3},"33":{"score":0.323,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":165.9},"36":{"score":0.3758,"jev":null,"metric":"nDCG@10","requests":220,"answered":219,"unsupported":1,"errors":0,"median_ms":1193.7},"37":{"score":0.4051,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":127.6},"38":{"score":0.1888,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":736.9},"39":{"score":0.8868,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":120.2},"40":{"score":0.4506,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":112.9},"41":{"score":0.4565,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":117.2},"42":{"score":0.7731,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":136.5},"43":{"score":0.5368,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":117.1},"44":{"score":0.6208,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":117},"45":{"score":0.1118,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":123.2},"50":{"score":0.463,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":161.2},"56":{"score":0.6855,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":163.6},"57":{"score":0.4975,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":118.7},"58":{"score":0.6268,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":114.6},"59":{"score":0.7197,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":158.9},"61":{"score":0.6125,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":131.6},"62":{"score":0.5036,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":145.9},"64":{"score":0.5322,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":116.7}},"benchmarks":{"1":{"raw":0.8725,"skill":0.8279,"coverage":1,"random":0.2592,"tracks":[],"combined":0.7421,"private_eq":0.8104},"2":{"raw":0.6156,"skill":0.556,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6614,"private_eq":0.7093},"3":{"raw":0.7087,"skill":0.7031,"coverage":1,"random":0.0189,"tracks":[],"combined":0.7543,"private_eq":0.7607},"4":{"raw":0.5965,"skill":0.5913,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7383,"private_eq":0.7436},"5":{"raw":0.7126,"skill":0.7109,"coverage":1,"random":0.006,"tracks":[],"combined":0.5404,"private_eq":0.5474},"6":{"raw":0.7987,"skill":0.528,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0795,"skill":0.0795,"coverage":1,"random":0,"tracks":[],"combined":0.0299,"private_eq":0.04},"10":{"raw":0.5223,"skill":0.2052,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7361,"skill":0.6183,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5807,"skill":0.3719,"coverage":1,"random":0.3324,"tracks":[],"combined":0.459,"private_eq":0.6423},"20":{"raw":0.5031,"skill":0.0062,"coverage":1,"random":0.5,"tracks":[],"combined":0.1335,"private_eq":0.5712},"21":{"raw":0.575,"skill":0.1499,"coverage":1,"random":0.5,"tracks":[],"combined":0.136,"private_eq":0.5724},"22":{"raw":0.0932,"skill":0.0861,"coverage":1,"random":0.0078,"tracks":[],"combined":0.1156,"private_eq":0.1315},"23":{"raw":0.5823,"skill":0.1646,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6913,"skill":0.5884,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4286,"skill":0.2381,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3505,"combined":0.125},"26":{"raw":0.9655,"skill":0.954,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8993,"skill":0.8657,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6969,"skill":0.3938,"coverage":1,"random":0.5,"tracks":[],"combined":0.214,"private_eq":0.611},"29":{"raw":0.7705,"skill":0.694,"coverage":1,"random":0.25,"tracks":[],"combined":0.6614,"private_eq":0.7482},"30":{"raw":0.4503,"skill":0.3387,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5451,"random":0.25},{"track":"10-choice","raw":0.3556,"random":0.1}],"combined":0.5307,"private_eq":0.6164},"31":{"raw":0.1058,"skill":0.0261,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0333,"private_eq":0.1217},"32":{"raw":0.5439,"skill":0.2749,"coverage":1,"random":0.371,"tracks":[],"combined":0.0418,"private_eq":0.4035},"33":{"raw":0.323,"skill":0.314,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3331,"private_eq":0.3483},"36":{"raw":0.3741,"skill":0.2919,"coverage":0.9955,"random":0.116,"tracks":[],"combined":0.2837,"private_eq":0.3731},"37":{"raw":0.4051,"skill":0.2539,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3781,"private_eq":0.509},"38":{"raw":0.1888,"skill":0.1628,"coverage":1,"random":0.031,"tracks":[],"combined":0.283,"private_eq":0.3122},"39":{"raw":0.8868,"skill":0.8335,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5687,"private_eq":0.7094},"40":{"raw":0.4506,"skill":0.2932,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2256,"headline":false},{"track":"A · English","score":0.4506,"headline":true},{"track":"C · Arabic pairs","score":0.48,"headline":false},{"track":"C · English pairs","score":0.74,"headline":false}],"combined":0.2058,"private_eq":0.3889},"41":{"raw":0.4565,"skill":0.1847,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4243,"private_eq":0.6199},"42":{"raw":0.7731,"skill":0.5586,"coverage":1,"random":0.486,"tracks":[],"combined":0.6039,"private_eq":0.7982},"43":{"raw":0.5368,"skill":0.2652,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4241,"private_eq":0.6405},"44":{"raw":0.6208,"skill":0.2416,"coverage":1,"random":0.5,"tracks":[],"combined":0.3852,"private_eq":0.6956},"45":{"raw":0.1118,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.463,"skill":0.2206,"coverage":1,"random":0.311,"tracks":[],"combined":0.2322,"private_eq":0.4763},"56":{"raw":0.6855,"skill":0.371,"coverage":1,"random":0.5,"tracks":[],"combined":0.2569,"private_eq":0.6322},"57":{"raw":0.4975,"skill":0.4348,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4267,"combined":0.3488},"58":{"raw":0.6268,"skill":0.4591,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3981,"combined":0.1185},"59":{"raw":0.7197,"skill":0.4188,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2953,"private_eq":0.6635},"61":{"raw":0.6125,"skill":0.225,"coverage":1,"random":0.5,"tracks":[],"combined":0.2917,"private_eq":0.6494},"62":{"raw":0.5036,"skill":0.3381,"coverage":1,"random":0.25,"tracks":[],"combined":0.3508,"private_eq":0.5179},"64":{"raw":0.5322,"skill":0.4152,"coverage":1,"random":0.2,"tracks":[],"combined":0.361,"private_eq":0.4938}},"coverage":0.9998,"pending":0.0002,"notes":["New in the 2026-10-27 PR batch (PR #311).","Replaces the earlier board entry valen-4b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 38.68, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.74% on 827 rows (bar 98.5%).","Same weights as the earlier Valen-4B entry, served with the author's updated engine."],"scores":{"balanced_raw":54.06,"balanced_skill":37.88,"breadth_skill":37.23,"public_skill":38.68},"frozen_scores":{"balanced_raw":54.06,"balanced_skill":38.68,"breadth_skill":37.23},"categories":[{"id":"knowledge","raw":0.4512,"skill":0.2743,"combined":0.2515,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6205,"skill":0.4627,"combined":0.4435,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5641,"skill":0.4186,"combined":0.4245,"coverage":0.9992,"pending":0,"n":6},{"id":"tools","raw":0.5734,"skill":0.5205,"combined":0.5256,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4581,"skill":0.1738,"combined":0.1956,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"valen-4b-r2","pub":38.68,"same_skills":39.13,"new_domains":27.73,"v":37.88,"lab":false,"jev":false,"rank_v":141,"rank_pub":159,"name":"Valen-4B","eq":{"S":38.53,"O":36.9},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":23.12,"new_domains":14.76}},"new_in_edition":true},{"engine":"vela3-fast","name":"vLLM-SR vela3-fast (Vela 3.0, 0.4B)","long_name":"vLLM-SR vela3-fast (Vela 3.0, 0.4B)","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":380436225,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/vela3-fast","code_url":null,"base_model":"dmis-lab/UltraBERT-380M","base_url":"https://huggingface.co/dmis-lab/UltraBERT-380M","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pr":305,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6758,"conf":0.7925,"ece":0.1206,"brier":0.4579,"over95":0.0416,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0016,"conf":0.0812,"acc":0.0729},{"lo":0.1,"w":0.0057,"conf":0.1585,"acc":0.0727},{"lo":0.2,"w":0.017,"conf":0.2619,"acc":0.2804},{"lo":0.3,"w":0.0357,"conf":0.353,"acc":0.3994},{"lo":0.4,"w":0.0597,"conf":0.4544,"acc":0.3685},{"lo":0.5,"w":0.1051,"conf":0.5493,"acc":0.4255},{"lo":0.6,"w":0.0985,"conf":0.6512,"acc":0.5178},{"lo":0.7,"w":0.0941,"conf":0.7497,"acc":0.557},{"lo":0.8,"w":0.1144,"conf":0.8534,"acc":0.6834},{"lo":0.9,"w":0.4681,"conf":0.9767,"acc":0.8714}],"areas":{"knowledge":{"ece":0.1954,"acc":0.5051,"conf":0.6908},"language":{"ece":0.0895,"acc":0.7824,"conf":0.8717},"retrieval":{"ece":0.0713,"acc":0.724,"conf":0.7856},"tools":{"ece":0.0761,"acc":0.8891,"conf":0.9635},"arts":{"ece":0.1125,"acc":0.659,"conf":0.7564}}},"gaps":null,"latency":{"median":54.3,"p95":161,"mean":74.5},"results":{"1":{"score":0.9174,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":118.9},"2":{"score":0.6041,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":566.6},"3":{"score":0.7303,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":168},"4":{"score":0.926,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":47.8},"5":{"score":0.9402,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":52},"6":{"score":0.542,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":101.6},"9":{"score":0.7159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":155.8},"10":{"score":0.4133,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":54.6},"11":{"score":0.7776,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":261},"12":{"score":0.5609,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":46.5},"20":{"score":0.9098,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46},"21":{"score":0.6339,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":47.5},"22":{"score":0.669,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":140.2},"23":{"score":0.5264,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.1},"24":{"score":0.4198,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":46.9},"25":{"score":0.2908,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":48.2},"26":{"score":0.7382,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":47.5},"27":{"score":0.5819,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":47.4},"28":{"score":0.7206,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":47.5},"29":{"score":0.7156,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":46.6},"30":{"score":0.3563,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1788,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":47},"32":{"score":0.5545,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":51.5},"33":{"score":0.2358,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":89.3},"36":{"score":0.2648,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":380.1},"37":{"score":0.5473,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":48.8},"38":{"score":0.4876,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":84.7},"39":{"score":0.8768,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":46.4},"40":{"score":0.5067,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":46.5},"41":{"score":0.7693,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":46.4},"42":{"score":0.6868,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":48.5},"43":{"score":0.4509,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":45.9},"44":{"score":0.6848,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46.7},"45":{"score":0.1537,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":47.3},"50":{"score":0.5597,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":52},"56":{"score":0.9075,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":54.1},"57":{"score":0.2412,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":48},"58":{"score":0.5518,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":47.4},"59":{"score":0.7365,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":49.6},"61":{"score":0.8648,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":46.6},"62":{"score":0.8384,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":50},"64":{"score":0.5852,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":44.4}},"benchmarks":{"1":{"raw":0.9174,"skill":0.8884,"coverage":1,"random":0.2592,"tracks":[],"combined":0.8034,"private_eq":0.8553},"2":{"raw":0.6041,"skill":0.5427,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6743,"private_eq":0.7203},"3":{"raw":0.7303,"skill":0.7251,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6179,"private_eq":0.6284},"4":{"raw":0.926,"skill":0.925,"coverage":1,"random":0.0127,"tracks":[],"combined":0.704,"private_eq":0.7101},"5":{"raw":0.9402,"skill":0.9398,"coverage":1,"random":0.006,"tracks":[],"combined":0.6918,"private_eq":0.6961},"6":{"raw":0.542,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7159,"skill":0.7159,"coverage":1,"random":0,"tracks":[],"combined":0.4219,"private_eq":0.4275},"10":{"raw":0.4133,"skill":0.0238,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7776,"skill":0.6784,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5609,"skill":0.3423,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4333,"private_eq":0.6253},"20":{"raw":0.9037,"skill":0.8073,"coverage":1,"random":0.5,"tracks":[],"combined":0.7263,"private_eq":0.8642},"21":{"raw":0.6339,"skill":0.2679,"coverage":1,"random":0.5,"tracks":[],"combined":0.2466,"private_eq":0.6271},"22":{"raw":0.6598,"skill":0.6572,"coverage":1,"random":0.0078,"tracks":[],"combined":0.577,"private_eq":0.5841},"23":{"raw":0.5049,"skill":0.0098,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4198,"skill":0.2264,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2908,"skill":0.0544,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2965,"combined":0.0522},"26":{"raw":0.7382,"skill":0.6508,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.5819,"skill":0.4424,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7206,"skill":0.4412,"coverage":1,"random":0.5,"tracks":[],"combined":0.4006,"private_eq":0.7032},"29":{"raw":0.7156,"skill":0.6208,"coverage":1,"random":0.25,"tracks":[],"combined":0.3396,"private_eq":0.5096},"30":{"raw":0.3563,"skill":0.2263,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4738,"random":0.25},{"track":"10-choice","raw":0.2388,"random":0.1}],"combined":0.112,"private_eq":0.2749},"31":{"raw":0.1788,"skill":0.1056,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1091,"private_eq":0.1905},"32":{"raw":0.5545,"skill":0.2917,"coverage":1,"random":0.371,"tracks":[],"combined":0.0418,"private_eq":0.4035},"33":{"raw":0.2358,"skill":0.2257,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1275,"private_eq":0.1478},"36":{"raw":0.2648,"skill":0.1683,"coverage":1,"random":0.116,"tracks":[],"combined":0.2328,"private_eq":0.3286},"37":{"raw":0.5473,"skill":0.4321,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4424,"private_eq":0.5597},"38":{"raw":0.4876,"skill":0.4712,"coverage":1,"random":0.031,"tracks":[],"combined":0.5261,"private_eq":0.545},"39":{"raw":0.8768,"skill":0.8188,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4961,"private_eq":0.6606},"40":{"raw":0.5067,"skill":0.3653,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2726,"headline":false},{"track":"A · English","score":0.5067,"headline":true},{"track":"C · Arabic pairs","score":0.565,"headline":false},{"track":"C · English pairs","score":0.915,"headline":false}],"combined":0.4446,"private_eq":0.5724},"41":{"raw":0.7693,"skill":0.6539,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3955,"private_eq":0.6009},"42":{"raw":0.6868,"skill":0.3907,"coverage":1,"random":0.486,"tracks":[],"combined":0.0048,"private_eq":0.4938},"43":{"raw":0.4509,"skill":0.1288,"coverage":1,"random":0.3697,"tracks":[],"combined":0.006,"private_eq":0.38},"44":{"raw":0.6848,"skill":0.3696,"coverage":1,"random":0.5,"tracks":[],"combined":0.4174,"private_eq":0.7115},"45":{"raw":0.1537,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5597,"skill":0.3609,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.9075,"skill":0.815,"coverage":1,"random":0.5,"tracks":[],"combined":0.4819,"private_eq":0.7434},"57":{"raw":0.2412,"skill":0.1466,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.374,"combined":0.2888},"58":{"raw":0.5518,"skill":0.3503,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3332,"combined":0.0234},"59":{"raw":0.7365,"skill":0.4537,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1189,"private_eq":0.5794},"61":{"raw":0.8648,"skill":0.7296,"coverage":1,"random":0.5,"tracks":[],"combined":0.5586,"private_eq":0.7813},"62":{"raw":0.8384,"skill":0.7845,"coverage":1,"random":0.25,"tracks":[],"combined":0.3919,"private_eq":0.5484},"64":{"raw":0.5852,"skill":0.4815,"coverage":1,"random":0.2,"tracks":[],"combined":0.5161,"private_eq":0.6165}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #305).","Submitted public 0.3 run adopted (kit rescore 50.36, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.90% on 831 rows (bar 98.5%).","Distilled from soft labels of vLLM-SR's d3 models, which are on the board (author-declared).","Training includes train splits of public datasets that are also sources of index benchmarks (author-declared)."],"scores":{"balanced_raw":63.09,"balanced_skill":29.27,"breadth_skill":46.69,"public_skill":50.36},"frozen_scores":{"balanced_raw":63.09,"balanced_skill":50.36,"breadth_skill":46.69},"categories":[{"id":"knowledge","raw":0.4009,"skill":0.2116,"combined":0.1603,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.678,"skill":0.5306,"combined":0.3468,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7389,"skill":0.6692,"combined":0.5208,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7659,"skill":0.7369,"combined":0.5914,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6412,"skill":0.4308,"combined":0.4654,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vela3-fast","pub":50.36,"same_skills":40.22,"new_domains":8.96,"v":29.27,"lab":false,"jev":false,"rank_v":171,"rank_pub":117,"name":"vLLM-SR vela3-fast (Vela 3.0, 0.4B)","eq":{"S":39.63,"O":13.71},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":23.78,"new_domains":5.49}},"new_in_edition":true},{"engine":"vela3-fast-vision-text","name":"vLLM-SR vela3-fast-vision (Vela 3.0, 0.25B)","long_name":"vLLM-SR vela3-fast-vision (Vela 3.0, 0.25B)","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":254088960,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/vllm-sr/vela3-fast-vision","code_url":null,"base_model":"athrael-soju/UltraVBERT","base_url":"https://huggingface.co/athrael-soju/UltraVBERT","weights_repo":null,"served_checkpoint":null,"org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pr":305,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.618,"conf":0.7297,"ece":0.1119,"brier":0.516,"over95":0.0239,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.002,"conf":0.0743,"acc":0.1151},{"lo":0.1,"w":0.0064,"conf":0.1586,"acc":0.114},{"lo":0.2,"w":0.0279,"conf":0.2627,"acc":0.2241},{"lo":0.3,"w":0.0675,"conf":0.3536,"acc":0.3363},{"lo":0.4,"w":0.0872,"conf":0.4517,"acc":0.3391},{"lo":0.5,"w":0.1306,"conf":0.5498,"acc":0.485},{"lo":0.6,"w":0.1051,"conf":0.6485,"acc":0.5001},{"lo":0.7,"w":0.104,"conf":0.7501,"acc":0.5952},{"lo":0.8,"w":0.1276,"conf":0.8514,"acc":0.6633},{"lo":0.9,"w":0.3417,"conf":0.9697,"acc":0.8665}],"areas":{"knowledge":{"ece":0.1524,"acc":0.4626,"conf":0.6102},"language":{"ece":0.0973,"acc":0.7256,"conf":0.8226},"retrieval":{"ece":0.1358,"acc":0.6473,"conf":0.7802},"tools":{"ece":0.093,"acc":0.733,"conf":0.8261},"arts":{"ece":0.0607,"acc":0.6426,"conf":0.7033}}},"gaps":null,"latency":{"median":36.8,"p95":216.4,"mean":90.1},"results":{"1":{"score":0.8158,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":185.8},"2":{"score":0.5649,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2822.5},"3":{"score":0.374,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":161.1},"4":{"score":0.7125,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":92.5},"5":{"score":0.7658,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":102.4},"6":{"score":0.6534,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":210.8},"9":{"score":0.5795,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1570.3},"10":{"score":0.3973,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":93.7},"11":{"score":0.4724,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1744.2},"12":{"score":0.4737,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":84.5},"20":{"score":0.9142,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.3},"21":{"score":0.6104,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":74.1},"22":{"score":0.639,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":147.2},"23":{"score":0.5224,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.2},"24":{"score":0.3685,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":79.3},"25":{"score":0.3214,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":88.1},"26":{"score":0.6473,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":72.5},"27":{"score":0.4616,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":73.5},"28":{"score":0.6196,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":74.4},"29":{"score":0.5966,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":79.7},"30":{"score":0.271,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1556,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":86.6},"32":{"score":0.4415,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":90.9},"33":{"score":0.1594,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":654.9},"36":{"score":0.2337,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2684.7},"37":{"score":0.434,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":82.8},"38":{"score":0.3095,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":4696.5},"39":{"score":0.8321,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":135.6},"40":{"score":0.4357,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":86.4},"41":{"score":0.7363,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":78.6},"42":{"score":0.6201,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":82.6},"43":{"score":0.4632,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":81.7},"44":{"score":0.658,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77.7},"45":{"score":0.1657,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":81.6},"50":{"score":0.5543,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":84.5},"56":{"score":0.894,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":715.1},"57":{"score":0.202,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":79.6},"58":{"score":0.3893,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":73.9},"59":{"score":0.6207,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":84.1},"61":{"score":0.8097,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":82.2},"62":{"score":0.5386,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":84.3},"64":{"score":0.5701,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":80.8}},"benchmarks":{"1":{"raw":0.8158,"skill":0.7514,"coverage":1,"random":0.2592,"tracks":[],"combined":0.6876,"private_eq":0.7705},"2":{"raw":0.5649,"skill":0.4975,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6204,"private_eq":0.6742},"3":{"raw":0.374,"skill":0.3619,"coverage":1,"random":0.0189,"tracks":[],"combined":0.2977,"private_eq":0.3178},"4":{"raw":0.7125,"skill":0.7088,"coverage":1,"random":0.0127,"tracks":[],"combined":0.5785,"private_eq":0.5876},"5":{"raw":0.7658,"skill":0.7644,"coverage":1,"random":0.006,"tracks":[],"combined":0.5743,"private_eq":0.5807},"6":{"raw":0.6534,"skill":0.1873,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5795,"skill":0.5795,"coverage":1,"random":0,"tracks":[],"combined":0.3309,"private_eq":0.3375},"10":{"raw":0.3973,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.4724,"skill":0.237,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4737,"skill":0.2117,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3396,"private_eq":0.5635},"20":{"raw":0.9087,"skill":0.8174,"coverage":1,"random":0.5,"tracks":[],"combined":0.6903,"private_eq":0.8464},"21":{"raw":0.6104,"skill":0.2207,"coverage":1,"random":0.5,"tracks":[],"combined":0.2165,"private_eq":0.6122},"22":{"raw":0.6374,"skill":0.6345,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5143,"private_eq":0.5226},"23":{"raw":0.5019,"skill":0.0039,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3685,"skill":0.158,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2602,"combined":0.0032},"26":{"raw":0.6473,"skill":0.5296,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.4616,"skill":0.2819,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6196,"skill":0.2392,"coverage":1,"random":0.5,"tracks":[],"combined":0.315,"private_eq":0.6609},"29":{"raw":0.5966,"skill":0.4621,"coverage":1,"random":0.25,"tracks":[],"combined":0.2653,"private_eq":0.4545},"30":{"raw":0.271,"skill":0.1202,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3715,"random":0.25},{"track":"10-choice","raw":0.1706,"random":0.1}],"combined":0.1165,"private_eq":0.2786},"31":{"raw":0.1556,"skill":0.0803,"coverage":1,"random":0.0819,"tracks":[],"combined":0.1113,"private_eq":0.1925},"32":{"raw":0.4415,"skill":0.1121,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.371},"33":{"raw":0.1594,"skill":0.1482,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1635,"private_eq":0.1829},"36":{"raw":0.2337,"skill":0.1331,"coverage":1,"random":0.116,"tracks":[],"combined":0.2214,"private_eq":0.3187},"37":{"raw":0.434,"skill":0.2901,"coverage":1,"random":0.2027,"tracks":[],"combined":0.3124,"private_eq":0.4572},"38":{"raw":0.3095,"skill":0.2874,"coverage":1,"random":0.031,"tracks":[],"combined":0.36,"private_eq":0.3859},"39":{"raw":0.8321,"skill":0.7531,"coverage":1,"random":0.3201,"tracks":[],"combined":0.445,"private_eq":0.6263},"40":{"raw":0.4357,"skill":0.2741,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0,"headline":false},{"track":"A · English","score":0.4357,"headline":true},{"track":"C · Arabic pairs","score":0.455,"headline":false},{"track":"C · English pairs","score":0.89,"headline":false}],"combined":0.2127,"private_eq":0.3942},"41":{"raw":0.7363,"skill":0.6044,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.6201,"skill":0.2609,"coverage":1,"random":0.486,"tracks":[],"combined":0.1768,"private_eq":0.5812},"43":{"raw":0.4632,"skill":0.1482,"coverage":1,"random":0.3697,"tracks":[],"combined":0.0646,"private_eq":0.4165},"44":{"raw":0.658,"skill":0.316,"coverage":1,"random":0.5,"tracks":[],"combined":0.2675,"private_eq":0.6374},"45":{"raw":0.1657,"skill":0.0019,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5543,"skill":0.3531,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.894,"skill":0.788,"coverage":1,"random":0.5,"tracks":[],"combined":0.179,"private_eq":0.5937},"57":{"raw":0.202,"skill":0.1025,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3893,"skill":0.1148,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3328,"combined":0.0228},"59":{"raw":0.6207,"skill":0.2136,"coverage":1,"random":0.5177,"tracks":[],"combined":0.0356,"private_eq":0.5397},"61":{"raw":0.8097,"skill":0.6194,"coverage":1,"random":0.5,"tracks":[],"combined":0.5244,"private_eq":0.7644},"62":{"raw":0.5386,"skill":0.3848,"coverage":1,"random":0.25,"tracks":[],"combined":0.3166,"private_eq":0.4925},"64":{"raw":0.5701,"skill":0.4626,"coverage":1,"random":0.2,"tracks":[],"combined":0.4968,"private_eq":0.6012}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #305).","Submitted public 0.3 run adopted (kit rescore 37.50, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.46% on 831 rows (bar 98.5%).","Text requests run through the same network as its vision entry, without images.","Distilled from soft labels of vLLM-SR's d3 models, which are on the board (author-declared).","Training includes train splits of public datasets that are also sources of index benchmarks (author-declared)."],"scores":{"balanced_raw":53.18,"balanced_skill":22.11,"breadth_skill":34.35,"public_skill":37.5},"frozen_scores":{"balanced_raw":53.18,"balanced_skill":37.5,"breadth_skill":34.35},"categories":[{"id":"knowledge","raw":0.3489,"skill":0.1324,"combined":0.1027,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.565,"skill":0.3659,"combined":0.2316,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6352,"skill":0.5493,"combined":0.4038,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5761,"skill":0.5181,"combined":0.4537,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6305,"skill":0.4154,"combined":0.4285,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vela3-fast-vision-text","pub":37.5,"same_skills":30.47,"new_domains":6.45,"v":22.11,"lab":false,"jev":false,"rank_v":190,"rank_pub":167,"name":"vLLM-SR vela3-fast-vision (Vela 3.0, 0.25B)","eq":{"S":29.77,"O":10.62},"rank_tie":43,"tied":true,"parts":{"public":0,"same_skills":17.86,"new_domains":4.25}},"new_in_edition":true},{"engine":"sieve-31b-v3","name":"Sieve 31B v3","long_name":"Sieve 31B v3","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":31273086512,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/sthanika-ai/Sieve-31B-v3","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"},"pr":317,"replaces":"sieve-31b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.806,"conf":0.8597,"ece":0.0537,"brier":0.2674,"over95":0.0145,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0012,"conf":0.089,"acc":0.0645},{"lo":0.1,"w":0.0125,"conf":0.1558,"acc":0.1371},{"lo":0.2,"w":0.0123,"conf":0.2494,"acc":0.192},{"lo":0.3,"w":0.0186,"conf":0.3576,"acc":0.2715},{"lo":0.4,"w":0.0332,"conf":0.4527,"acc":0.385},{"lo":0.5,"w":0.0627,"conf":0.5485,"acc":0.4835},{"lo":0.6,"w":0.0625,"conf":0.6494,"acc":0.5495},{"lo":0.7,"w":0.0693,"conf":0.7529,"acc":0.6263},{"lo":0.8,"w":0.0838,"conf":0.8523,"acc":0.71},{"lo":0.9,"w":0.6439,"conf":0.9851,"acc":0.9574}],"areas":{"knowledge":{"ece":0.0672,"acc":0.7467,"conf":0.8139},"language":{"ece":0.0197,"acc":0.894,"conf":0.9126},"retrieval":{"ece":0.1391,"acc":0.7216,"conf":0.8595},"tools":{"ece":0.0374,"acc":0.9516,"conf":0.9871},"arts":{"ece":0.0418,"acc":0.7464,"conf":0.7799}}},"gaps":null,"latency":{"median":231.8,"p95":1520.2,"mean":663},"results":{"1":{"score":0.9787,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":509.9},"2":{"score":0.7048,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":8805.9},"3":{"score":0.8878,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":4966.3},"4":{"score":0.9132,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":656.6},"5":{"score":0.9682,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1083},"6":{"score":0.7978,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1029},"9":{"score":0.9659,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2104},"10":{"score":0.3656,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":662.1},"11":{"score":0.798,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2285},"12":{"score":0.809,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":184.2},"20":{"score":0.9922,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":176.6},"21":{"score":0.6758,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":107.1},"22":{"score":0.7925,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":4823.1},"23":{"score":0.665,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":349.6},"24":{"score":0.8968,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":124.3},"25":{"score":0.5459,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":213.5},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":115.8},"27":{"score":0.9778,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":118.2},"28":{"score":0.9487,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":111.8},"29":{"score":0.9753,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":216.5},"30":{"score":0.9098,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2554,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":470.4},"32":{"score":0.8085,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":1084.2},"33":{"score":0.3939,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":659.5},"36":{"score":0.5063,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":14201.2},"37":{"score":0.6445,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.7},"38":{"score":0.5013,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":6974.8},"39":{"score":0.9417,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":219},"40":{"score":0.683,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":107.8},"41":{"score":0.7967,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":191.6},"42":{"score":0.8419,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":288.9},"43":{"score":0.9316,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":175.1},"44":{"score":0.9228,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":184.5},"45":{"score":0.1297,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":219.2},"50":{"score":0.6217,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":471.5},"56":{"score":0.949,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":585.8},"57":{"score":0.7581,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":180.7},"58":{"score":0.896,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":175.5},"59":{"score":0.8214,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":230.3},"61":{"score":0.8912,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":277.4},"62":{"score":0.8803,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":433.9},"64":{"score":0.8731,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":179}},"benchmarks":{"1":{"raw":0.9787,"skill":0.9713,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.7048,"skill":0.6591,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7115,"private_eq":0.7522},"3":{"raw":0.8878,"skill":0.8856,"coverage":1,"random":0.0189,"tracks":[],"combined":0.938,"private_eq":0.9389},"4":{"raw":0.9132,"skill":0.9121,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9124,"private_eq":0.9135},"5":{"raw":0.9682,"skill":0.968,"coverage":1,"random":0.006,"tracks":[],"combined":0.8392,"private_eq":0.841},"6":{"raw":0.7978,"skill":0.5259,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9659,"skill":0.9659,"coverage":1,"random":0,"tracks":[],"combined":0.9252,"private_eq":0.925},"10":{"raw":0.3656,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.798,"skill":0.7079,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.809,"skill":0.7139,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5168,"private_eq":0.6804},"20":{"raw":0.9921,"skill":0.9842,"coverage":1,"random":0.5,"tracks":[],"combined":0.9939,"private_eq":0.9965},"21":{"raw":0.6758,"skill":0.3516,"coverage":1,"random":0.5,"tracks":[],"combined":0.3371,"private_eq":0.6718},"22":{"raw":0.7884,"skill":0.7867,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6727,"private_eq":0.678},"23":{"raw":0.6601,"skill":0.3202,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8968,"skill":0.8624,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5459,"skill":0.3946,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5584,"combined":0.4054},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9778,"skill":0.9704,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9487,"skill":0.8974,"coverage":1,"random":0.5,"tracks":[],"combined":0.6984,"private_eq":0.8504},"29":{"raw":0.9753,"skill":0.9671,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.9098,"skill":0.8926,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9356,"random":0.25},{"track":"10-choice","raw":0.884,"random":0.1}],"combined":0.8505,"private_eq":0.8772},"31":{"raw":0.2554,"skill":0.189,"coverage":1,"random":0.0819,"tracks":[],"combined":0.095,"private_eq":0.1777},"32":{"raw":0.8085,"skill":0.6955,"coverage":1,"random":0.371,"tracks":[],"combined":0.4518,"private_eq":0.6585},"33":{"raw":0.3939,"skill":0.3859,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4923,"private_eq":0.5037},"36":{"raw":0.5063,"skill":0.4415,"coverage":1,"random":0.116,"tracks":[],"combined":0.4363,"private_eq":0.5065},"37":{"raw":0.6445,"skill":0.554,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5916,"private_eq":0.6773},"38":{"raw":0.5013,"skill":0.4853,"coverage":1,"random":0.031,"tracks":[],"combined":0.7049,"private_eq":0.7163},"39":{"raw":0.9417,"skill":0.9143,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5926,"private_eq":0.7255},"40":{"raw":0.683,"skill":0.5922,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.6295,"headline":false},{"track":"A · English","score":0.683,"headline":true},{"track":"C · Arabic pairs","score":0.995,"headline":false},{"track":"C · English pairs","score":0.99,"headline":false}],"combined":0.6265,"private_eq":0.7122},"41":{"raw":0.7967,"skill":0.6951,"coverage":1,"random":0.3333,"tracks":[],"combined":0.6539,"private_eq":0.7712},"42":{"raw":0.8419,"skill":0.6924,"coverage":1,"random":0.486,"tracks":[],"combined":0.9741,"private_eq":0.9863},"43":{"raw":0.9316,"skill":0.8914,"coverage":1,"random":0.3697,"tracks":[],"combined":0.7,"private_eq":0.8124},"44":{"raw":0.9228,"skill":0.8456,"coverage":1,"random":0.5,"tracks":[],"combined":0.7868,"private_eq":0.8941},"45":{"raw":0.1297,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6217,"skill":0.451,"coverage":1,"random":0.311,"tracks":[],"combined":0.2684,"private_eq":0.501},"56":{"raw":0.949,"skill":0.898,"coverage":1,"random":0.5,"tracks":[],"combined":0.9144,"private_eq":0.9572},"57":{"raw":0.7581,"skill":0.7279,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6309,"combined":0.5811},"58":{"raw":0.896,"skill":0.8493,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5743,"combined":0.3769},"59":{"raw":0.8214,"skill":0.6297,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4048,"private_eq":0.7157},"61":{"raw":0.8912,"skill":0.7824,"coverage":1,"random":0.5,"tracks":[],"combined":0.6662,"private_eq":0.8345},"62":{"raw":0.8803,"skill":0.8404,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.8731,"skill":0.8414,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #317).","Replaces the earlier board entry sieve-31b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 71.68, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 100.00% on 831 rows (bar 98.5%).","Released as 0.3 base + 0.7 fine-tuned weights; two option orders averaged at inference.","Training data undisclosed."],"scores":{"balanced_raw":78.39,"balanced_skill":71.1,"breadth_skill":71.18,"public_skill":71.68},"frozen_scores":{"balanced_raw":78.39,"balanced_skill":71.68,"breadth_skill":71.18},"categories":[{"id":"knowledge","raw":0.6797,"skill":0.6119,"combined":0.5417,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8005,"skill":0.7164,"combined":0.6878,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.8106,"skill":0.7607,"combined":0.7269,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8872,"skill":0.8692,"combined":0.8378,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7685,"skill":0.6225,"combined":0.6159,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"sieve-31b-v3","pub":71.68,"same_skills":68.53,"new_domains":58.86,"v":71.1,"lab":false,"jev":false,"rank_v":3,"rank_pub":7,"name":"Sieve 31B v3","eq":{"S":68.27,"O":75.33},"rank_tie":1,"tied":true,"parts":{"public":0,"same_skills":40.96,"new_domains":30.13}},"new_in_edition":true},{"engine":"ainvest-decision-27b-v1.0","name":"Ainvest-Decision-27B v1.0","long_name":"Ainvest-Decision-27B v1.0","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26086635760,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Ainvest-World/Ainvest-Decision-27b-v1.0","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"pr":313,"new":true},"completed":110201,"counts":{"ok":110199,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.8197,"conf":0.8578,"ece":0.0421,"brier":0.2435,"over95":0.0162,"n":34406,"benchmarks":31,"rel":[{"lo":0,"w":0.013,"conf":0.0692,"acc":0.1627},{"lo":0.1,"w":0.0108,"conf":0.1494,"acc":0.223},{"lo":0.2,"w":0.0078,"conf":0.2537,"acc":0.2451},{"lo":0.3,"w":0.022,"conf":0.3565,"acc":0.2632},{"lo":0.4,"w":0.029,"conf":0.4598,"acc":0.3126},{"lo":0.5,"w":0.0565,"conf":0.5489,"acc":0.4772},{"lo":0.6,"w":0.0577,"conf":0.6483,"acc":0.5798},{"lo":0.7,"w":0.0668,"conf":0.7513,"acc":0.6804},{"lo":0.8,"w":0.0892,"conf":0.8563,"acc":0.8146},{"lo":0.9,"w":0.6472,"conf":0.9844,"acc":0.9578}],"areas":{"knowledge":{"ece":0.0771,"acc":0.7198,"conf":0.7844},"language":{"ece":0.0161,"acc":0.9024,"conf":0.9059},"retrieval":{"ece":0.0387,"acc":0.8546,"conf":0.8917},"tools":{"ece":0.037,"acc":0.9566,"conf":0.9922},"arts":{"ece":0.0514,"acc":0.7606,"conf":0.81}}},"gaps":null,"latency":{"median":99.1,"p95":1067.7,"mean":287.2},"results":{"1":{"score":0.9764,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":180},"2":{"score":0.6935,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":1315.8},"3":{"score":0.876,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":901.3},"4":{"score":0.9364,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":102.7},"5":{"score":0.9644,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":173.2},"6":{"score":0.7976,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":496.6},"9":{"score":0.9545,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2082.3},"10":{"score":0.8575,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":99.3},"11":{"score":0.8674,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":3688},"12":{"score":0.8001,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":21.4},"20":{"score":0.9886,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.7},"21":{"score":0.6929,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":13.2},"22":{"score":0.8609,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":765.1},"23":{"score":0.7016,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":58.3},"24":{"score":0.8833,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":16.7},"25":{"score":0.5612,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":30.1},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":13.6},"27":{"score":0.9735,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":15.1},"28":{"score":0.9645,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":12},"29":{"score":0.9437,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":31.7},"30":{"score":0.7839,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2532,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":107.6},"32":{"score":0.7261,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":110.4},"33":{"score":0.4655,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":403.2},"36":{"score":0.5015,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2020.1},"37":{"score":0.6106,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":43.5},"38":{"score":0.5813,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1262.7},"39":{"score":0.9401,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":42},"40":{"score":0.6926,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":12.5},"41":{"score":0.8055,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":23.5},"42":{"score":0.8759,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":60.9},"43":{"score":0.886,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":22.4},"44":{"score":0.862,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.1},"45":{"score":0.1497,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":31.5},"50":{"score":0.651,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":97.6},"56":{"score":0.9405,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":259.5},"57":{"score":0.7319,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":27.2},"58":{"score":0.8876,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":23.6},"59":{"score":0.8605,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":92.5},"61":{"score":0.894,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":52},"62":{"score":0.9189,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":90.1},"64":{"score":0.8258,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":25.3}},"benchmarks":{"1":{"raw":0.9764,"skill":0.9681,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9874,"private_eq":0.99},"2":{"raw":0.6925,"skill":0.6449,"coverage":0.9985,"random":0.1341,"tracks":[],"combined":0.7033,"private_eq":0.7451},"3":{"raw":0.876,"skill":0.8736,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.9364,"skill":0.9356,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9018,"private_eq":0.9032},"5":{"raw":0.9644,"skill":0.9642,"coverage":1,"random":0.006,"tracks":[],"combined":0.8968,"private_eq":0.8976},"6":{"raw":0.7976,"skill":0.5254,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9545,"skill":0.9545,"coverage":1,"random":0,"tracks":[],"combined":0.8518,"private_eq":0.8525},"10":{"raw":0.8575,"skill":0.7629,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8674,"skill":0.8082,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.8001,"skill":0.7006,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5547,"private_eq":0.7054},"20":{"raw":0.9889,"skill":0.9777,"coverage":1,"random":0.5,"tracks":[],"combined":0.965,"private_eq":0.9822},"21":{"raw":0.6929,"skill":0.3858,"coverage":1,"random":0.5,"tracks":[],"combined":0.3472,"private_eq":0.6768},"22":{"raw":0.8602,"skill":0.8591,"coverage":0.9995,"random":0.0078,"tracks":[],"combined":0.608,"private_eq":0.6145},"23":{"raw":0.7214,"skill":0.4428,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8833,"skill":0.8444,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5612,"skill":0.415,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5428,"combined":0.3844},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9735,"skill":0.9647,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9645,"skill":0.929,"coverage":1,"random":0.5,"tracks":[],"combined":0.6227,"private_eq":0.813},"29":{"raw":0.9437,"skill":0.9249,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7839,"skill":0.7408,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8279,"random":0.25},{"track":"10-choice","raw":0.74,"random":0.1}],"combined":0.7549,"private_eq":0.7993},"31":{"raw":0.2532,"skill":0.1866,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0506,"private_eq":0.1374},"32":{"raw":0.7261,"skill":0.5645,"coverage":1,"random":0.371,"tracks":[],"combined":0.3434,"private_eq":0.5911},"33":{"raw":0.4655,"skill":0.4584,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4923,"private_eq":0.5037},"36":{"raw":0.5015,"skill":0.4361,"coverage":1,"random":0.116,"tracks":[],"combined":0.4227,"private_eq":0.4946},"37":{"raw":0.6106,"skill":0.5116,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6347,"private_eq":0.7113},"38":{"raw":0.5813,"skill":0.5679,"coverage":1,"random":0.031,"tracks":[],"combined":0.6478,"private_eq":0.6616},"39":{"raw":0.9401,"skill":0.9119,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5538,"private_eq":0.6994},"40":{"raw":0.6926,"skill":0.6046,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.359,"headline":false},{"track":"A · English","score":0.6926,"headline":true},{"track":"C · Arabic pairs","score":0.765,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"combined":0.6599,"private_eq":0.7379},"41":{"raw":0.8055,"skill":0.7083,"coverage":1,"random":0.3333,"tracks":[],"combined":0.2099,"private_eq":0.4786},"42":{"raw":0.8759,"skill":0.7586,"coverage":1,"random":0.486,"tracks":[],"combined":0.9158,"private_eq":0.9567},"43":{"raw":0.886,"skill":0.8191,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5955,"private_eq":0.7473},"44":{"raw":0.862,"skill":0.724,"coverage":1,"random":0.5,"tracks":[],"combined":0.6261,"private_eq":0.8147},"45":{"raw":0.1497,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.651,"skill":0.4934,"coverage":1,"random":0.311,"tracks":[],"combined":0.2612,"private_eq":0.4961},"56":{"raw":0.9405,"skill":0.881,"coverage":1,"random":0.5,"tracks":[],"combined":0.9059,"private_eq":0.953},"57":{"raw":0.7319,"skill":0.6985,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5528,"combined":0.4922},"58":{"raw":0.8876,"skill":0.8371,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5214,"combined":0.2993},"59":{"raw":0.8605,"skill":0.7108,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2965,"private_eq":0.6641},"61":{"raw":0.894,"skill":0.788,"coverage":1,"random":0.5,"tracks":[],"combined":0.6549,"private_eq":0.8289},"62":{"raw":0.9189,"skill":0.8919,"coverage":1,"random":0.25,"tracks":[],"combined":0.604,"private_eq":0.7056},"64":{"raw":0.8258,"skill":0.7822,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":0.9999,"pending":0.0001,"notes":["New in the 2026-10-27 PR batch (PR #313).","Submitted public 0.3 run adopted (kit rescore 71.88, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.32% on 824 rows (bar 98.5%).","Linear merge of four fine-tunes of Qwen3.8-27B with a 255-way readout; full-attention layers read the prompt bidirectionally.","Questions over the engine's 16,384-token input limit count as unsupported.","Training data undisclosed beyond crediting tasksource."],"scores":{"balanced_raw":78.57,"balanced_skill":68.23,"breadth_skill":71.26,"public_skill":71.88},"frozen_scores":{"balanced_raw":78.57,"balanced_skill":71.88,"breadth_skill":71.26},"categories":[{"id":"knowledge","raw":0.6609,"skill":0.5791,"combined":0.4649,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8208,"skill":0.7469,"combined":0.6132,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.8073,"skill":0.7551,"combined":0.7365,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8868,"skill":0.8706,"combined":0.8242,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.79,"skill":0.6568,"combined":0.5977,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"ainvest-decision-27b-v1.0","pub":71.88,"same_skills":64.58,"new_domains":57.91,"v":68.23,"lab":false,"jev":false,"rank_v":10,"rank_pub":6,"name":"Ainvest-Decision-27B v1.0","eq":{"S":64.27,"O":74.17},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.56,"new_domains":29.67}},"new_in_edition":true},{"engine":"oak-1.1","name":"MISA AI 2.0 Decision Plus · oak 1.1 (31B)","long_name":"MISA AI 2.0 Decision Plus · oak 1.1 (31B)","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/PNPM244/oak1.1","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"misa-jsc","name":"MISA JSC","logo":"data/orgs/misa-jsc.png"},"pr":295,"replaces":"linne-oak-1.0","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7836,"conf":0.8551,"ece":0.0716,"brier":0.2964,"over95":0.0248,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0024,"conf":0.1689,"acc":0.0167},{"lo":0.2,"w":0.0089,"conf":0.2512,"acc":0.1023},{"lo":0.3,"w":0.0273,"conf":0.3537,"acc":0.1587},{"lo":0.4,"w":0.032,"conf":0.4538,"acc":0.3035},{"lo":0.5,"w":0.0822,"conf":0.5471,"acc":0.4536},{"lo":0.6,"w":0.0659,"conf":0.6481,"acc":0.5506},{"lo":0.7,"w":0.0718,"conf":0.7498,"acc":0.5836},{"lo":0.8,"w":0.0894,"conf":0.8531,"acc":0.7685},{"lo":0.9,"w":0.6201,"conf":0.9845,"acc":0.9424}],"areas":{"knowledge":{"ece":0.1074,"acc":0.6912,"conf":0.7986},"language":{"ece":0.0177,"acc":0.8787,"conf":0.8943},"retrieval":{"ece":0.1034,"acc":0.7572,"conf":0.8598},"tools":{"ece":0.0484,"acc":0.9524,"conf":0.9862},"arts":{"ece":0.0984,"acc":0.7168,"conf":0.8152}}},"gaps":null,"latency":{"median":112.6,"p95":1611.7,"mean":407.4},"results":{"1":{"score":0.9799,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":266.5},"2":{"score":0.6673,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3241.4},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1461.7},"4":{"score":0.9047,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":190.7},"5":{"score":0.9615,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":275},"6":{"score":0.7975,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":766.9},"9":{"score":0.9545,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2671.6},"10":{"score":0.461,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":192.7},"11":{"score":0.8153,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":5768.8},"12":{"score":0.7607,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":56.1},"20":{"score":0.9756,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":57.4},"21":{"score":0.6948,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":50.3},"22":{"score":0.813,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1174.9},"23":{"score":0.6554,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":83.2},"24":{"score":0.8808,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":50.9},"25":{"score":0.5459,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":61},"26":{"score":0.9928,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":49.6},"27":{"score":0.9761,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":50},"28":{"score":0.9329,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":49.4},"29":{"score":0.9452,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":64.4},"30":{"score":0.8726,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2348,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":148.9},"32":{"score":0.6622,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":175.9},"33":{"score":0.3291,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":657.6},"36":{"score":0.494,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4672.5},"37":{"score":0.5858,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":71.3},"38":{"score":0.4139,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2673},"39":{"score":0.8975,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":115.8},"40":{"score":0.7847,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":50.1},"41":{"score":0.8263,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":57.9},"42":{"score":0.8518,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":90.4},"43":{"score":0.8877,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":54.1},"44":{"score":0.8332,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":53.6},"45":{"score":0.1098,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":63.7},"50":{"score":0.4839,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":142.1},"56":{"score":0.9,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":540.5},"57":{"score":0.7339,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":59.7},"58":{"score":0.8219,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":54.5},"59":{"score":0.824,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":131.8},"61":{"score":0.8522,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":79.3},"62":{"score":0.902,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":120.8},"64":{"score":0.7936,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":59.2}},"benchmarks":{"1":{"raw":0.9799,"skill":0.9729,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9669,"private_eq":0.975},"2":{"raw":0.6673,"skill":0.6158,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7008,"private_eq":0.743},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8908,"private_eq":0.8931},"4":{"raw":0.9047,"skill":0.9035,"coverage":1,"random":0.0127,"tracks":[],"combined":0.884,"private_eq":0.8858},"5":{"raw":0.9615,"skill":0.9613,"coverage":1,"random":0.006,"tracks":[],"combined":0.835,"private_eq":0.8369},"6":{"raw":0.7975,"skill":0.5252,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9545,"skill":0.9545,"coverage":1,"random":0,"tracks":[],"combined":0.7861,"private_eq":0.7875},"10":{"raw":0.461,"skill":0.1032,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8153,"skill":0.7329,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7607,"skill":0.6416,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4772,"private_eq":0.6543},"20":{"raw":0.9767,"skill":0.9534,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.6948,"skill":0.3896,"coverage":1,"random":0.5,"tracks":[],"combined":0.317,"private_eq":0.6619},"22":{"raw":0.8119,"skill":0.8105,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6779,"private_eq":0.6831},"23":{"raw":0.6509,"skill":0.3019,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8808,"skill":0.8411,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5459,"skill":0.3946,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5386,"combined":0.3787},"26":{"raw":0.9928,"skill":0.9904,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9761,"skill":0.9681,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9329,"skill":0.8658,"coverage":1,"random":0.5,"tracks":[],"combined":0.6278,"private_eq":0.8155},"29":{"raw":0.9452,"skill":0.9269,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8726,"skill":0.8485,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9105,"random":0.25},{"track":"10-choice","raw":0.8347,"random":0.1}],"combined":0.8009,"private_eq":0.8368},"31":{"raw":0.2348,"skill":0.1666,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0939,"private_eq":0.1767},"32":{"raw":0.6622,"skill":0.463,"coverage":1,"random":0.371,"tracks":[],"combined":0.4054,"private_eq":0.6296},"33":{"raw":0.3291,"skill":0.3202,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3228,"private_eq":0.3383},"36":{"raw":0.494,"skill":0.4276,"coverage":1,"random":0.116,"tracks":[],"combined":0.4006,"private_eq":0.4753},"37":{"raw":0.5858,"skill":0.4805,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6005,"private_eq":0.6843},"38":{"raw":0.4139,"skill":0.3951,"coverage":1,"random":0.031,"tracks":[],"combined":0.4965,"private_eq":0.5167},"39":{"raw":0.8975,"skill":0.8492,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6056,"private_eq":0.7342},"40":{"raw":0.7847,"skill":0.723,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3077,"headline":false},{"track":"A · English","score":0.7847,"headline":true},{"track":"C · Arabic pairs","score":0.935,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.8268,"private_eq":0.8661},"41":{"raw":0.8263,"skill":0.7395,"coverage":1,"random":0.3333,"tracks":[],"combined":0.1755,"private_eq":0.4559},"42":{"raw":0.8518,"skill":0.7117,"coverage":1,"random":0.486,"tracks":[],"combined":0.9597,"private_eq":0.979},"43":{"raw":0.8877,"skill":0.8219,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6499,"private_eq":0.7812},"44":{"raw":0.8332,"skill":0.6664,"coverage":1,"random":0.5,"tracks":[],"combined":0.6102,"private_eq":0.8068},"45":{"raw":0.1098,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4839,"skill":0.251,"coverage":1,"random":0.311,"tracks":[],"combined":0.2539,"private_eq":0.4911},"56":{"raw":0.9,"skill":0.8,"coverage":1,"random":0.5,"tracks":[],"combined":0.9144,"private_eq":0.9572},"57":{"raw":0.7339,"skill":0.7007,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1514,"combined":0.0356},"58":{"raw":0.8219,"skill":0.7418,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5389,"combined":0.325},"59":{"raw":0.824,"skill":0.6351,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3089,"private_eq":0.67},"61":{"raw":0.8522,"skill":0.7044,"coverage":1,"random":0.5,"tracks":[],"combined":0.6662,"private_eq":0.8345},"62":{"raw":0.902,"skill":0.8693,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.7936,"skill":0.742,"coverage":1,"random":0.2,"tracks":[],"combined":0.7489,"private_eq":0.8006}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #295).","Replaces the earlier board entry linne-oak-1.0 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 68.19, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.55% on 831 rows (bar 98.5%).","Full-attention layers read the prompt bidirectionally; the request's questions are listed before the state.","Training data undisclosed."],"scores":{"balanced_raw":75.9,"balanced_skill":67.62,"breadth_skill":67.42,"public_skill":68.19},"frozen_scores":{"balanced_raw":75.9,"balanced_skill":68.19,"breadth_skill":67.42},"categories":[{"id":"knowledge","raw":0.6327,"skill":0.5442,"combined":0.4188,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7937,"skill":0.7094,"combined":0.6127,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7834,"skill":0.7175,"combined":0.7159,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.875,"skill":0.8572,"combined":0.8051,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7353,"skill":0.5747,"combined":0.5896,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"oak-1.1","pub":68.19,"same_skills":62.55,"new_domains":59.17,"v":67.62,"lab":false,"jev":false,"rank_v":14,"rank_pub":17,"name":"MISA AI 2.0 Decision Plus · oak 1.1 (31B)","eq":{"S":62.22,"O":75.73},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.33,"new_domains":30.29}},"new_in_edition":true},{"engine":"wemm-jev-27b","name":"WeMM-JEV-27B","long_name":"WeMM-JEV-27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":26085330160,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/Rover00/WeMM-JEV","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"pr":273,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.8026,"conf":0.8512,"ece":0.0512,"brier":0.2728,"over95":0.0172,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0015,"conf":0.0786,"acc":0.0539},{"lo":0.1,"w":0.0119,"conf":0.1523,"acc":0.1757},{"lo":0.2,"w":0.0146,"conf":0.2523,"acc":0.3207},{"lo":0.3,"w":0.0235,"conf":0.3579,"acc":0.2789},{"lo":0.4,"w":0.0333,"conf":0.4527,"acc":0.3651},{"lo":0.5,"w":0.0701,"conf":0.5497,"acc":0.4893},{"lo":0.6,"w":0.0615,"conf":0.6487,"acc":0.5439},{"lo":0.7,"w":0.0686,"conf":0.7521,"acc":0.6553},{"lo":0.8,"w":0.0913,"conf":0.8542,"acc":0.7484},{"lo":0.9,"w":0.6237,"conf":0.9846,"acc":0.9556}],"areas":{"knowledge":{"ece":0.0802,"acc":0.7325,"conf":0.81},"language":{"ece":0.0084,"acc":0.9016,"conf":0.909},"retrieval":{"ece":0.0695,"acc":0.7958,"conf":0.8653},"tools":{"ece":0.0368,"acc":0.9566,"conf":0.9894},"arts":{"ece":0.0714,"acc":0.6779,"conf":0.7358}}},"gaps":null,"latency":{"median":98.3,"p95":1076.1,"mean":279.5},"results":{"1":{"score":0.9793,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":495.1},"2":{"score":0.6899,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":4730.9},"3":{"score":0.8819,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2150.2},"4":{"score":0.8483,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":291.4},"5":{"score":0.965,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":466.1},"6":{"score":0.798,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1283.1},"9":{"score":0.9886,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":5504.1},"10":{"score":0.6713,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":360.3},"11":{"score":0.8208,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":9131.1},"12":{"score":0.7695,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":110},"20":{"score":0.9648,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":110},"21":{"score":0.6218,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":109.3},"22":{"score":0.691,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1840.1},"23":{"score":0.6494,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":187.1},"24":{"score":0.8813,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":110.8},"25":{"score":0.5051,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":115.9},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":109.8},"27":{"score":0.9735,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":109.3},"28":{"score":0.9463,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":109.5},"29":{"score":0.9673,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":119},"30":{"score":0.8093,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.3162,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":307.5},"32":{"score":0.7367,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":313.7},"33":{"score":0.4079,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1014.9},"36":{"score":0.5142,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":7353.2},"37":{"score":0.6409,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":146},"38":{"score":0.652,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3409.4},"39":{"score":0.9415,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":146.9},"40":{"score":0.6667,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":111.1},"41":{"score":0.8052,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":112.6},"42":{"score":0.8679,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":185.5},"43":{"score":0.8526,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":111.5},"44":{"score":0.9644,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":111},"45":{"score":0.1796,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":118.6},"50":{"score":0.466,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":284.4},"56":{"score":0.778,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":774.9},"57":{"score":0.7389,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":116},"58":{"score":0.8974,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":115.1},"59":{"score":0.8601,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":260.3},"61":{"score":0.902,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":154.3},"62":{"score":0.845,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":257.4},"64":{"score":0.8125,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":113.6}},"benchmarks":{"1":{"raw":0.9793,"skill":0.9721,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9739,"private_eq":0.9801},"2":{"raw":0.6899,"skill":0.6418,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7162,"private_eq":0.7562},"3":{"raw":0.8819,"skill":0.8796,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9485,"private_eq":0.9491},"4":{"raw":0.8483,"skill":0.8463,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9001,"private_eq":0.9015},"5":{"raw":0.965,"skill":0.9648,"coverage":1,"random":0.006,"tracks":[],"combined":0.8672,"private_eq":0.8685},"6":{"raw":0.798,"skill":0.5264,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9886,"skill":0.9886,"coverage":1,"random":0,"tracks":[],"combined":0.9252,"private_eq":0.925},"10":{"raw":0.6713,"skill":0.4531,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8208,"skill":0.7409,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7695,"skill":0.6547,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5159,"private_eq":0.6798},"20":{"raw":0.9663,"skill":0.9326,"coverage":1,"random":0.5,"tracks":[],"combined":0.9577,"private_eq":0.9786},"21":{"raw":0.6218,"skill":0.2435,"coverage":1,"random":0.5,"tracks":[],"combined":0.2869,"private_eq":0.647},"22":{"raw":0.6836,"skill":0.6811,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6118,"private_eq":0.6182},"23":{"raw":0.6487,"skill":0.2974,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8813,"skill":0.8417,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5064,"combined":0.3353},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9735,"skill":0.9647,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9463,"skill":0.8926,"coverage":1,"random":0.5,"tracks":[],"combined":0.5924,"private_eq":0.798},"29":{"raw":0.9673,"skill":0.9564,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8093,"skill":0.7714,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8491,"random":0.25},{"track":"10-choice","raw":0.7695,"random":0.1}],"combined":0.756,"private_eq":0.8002},"31":{"raw":0.3162,"skill":0.2552,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0939,"private_eq":0.1767},"32":{"raw":0.7367,"skill":0.5814,"coverage":1,"random":0.371,"tracks":[],"combined":0.4054,"private_eq":0.6296},"33":{"raw":0.4079,"skill":0.4,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4565,"private_eq":0.4687},"36":{"raw":0.5142,"skill":0.4504,"coverage":1,"random":0.116,"tracks":[],"combined":0.4242,"private_eq":0.4959},"37":{"raw":0.6409,"skill":0.5496,"coverage":1,"random":0.2027,"tracks":[],"combined":0.612,"private_eq":0.6934},"38":{"raw":0.652,"skill":0.6409,"coverage":1,"random":0.031,"tracks":[],"combined":0.7394,"private_eq":0.7494},"39":{"raw":0.9415,"skill":0.914,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5333,"private_eq":0.6856},"40":{"raw":0.6667,"skill":0.5712,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5557,"headline":false},{"track":"A · English","score":0.6667,"headline":true},{"track":"C · Arabic pairs","score":0.97,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.5619,"private_eq":0.6626},"41":{"raw":0.8052,"skill":0.7078,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3438,"private_eq":0.5668},"42":{"raw":0.8679,"skill":0.743,"coverage":1,"random":0.486,"tracks":[],"combined":0.93,"private_eq":0.9639},"43":{"raw":0.8526,"skill":0.7662,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6372,"private_eq":0.7733},"44":{"raw":0.9644,"skill":0.9288,"coverage":1,"random":0.5,"tracks":[],"combined":0.8511,"private_eq":0.9259},"45":{"raw":0.1796,"skill":0.0185,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.466,"skill":0.225,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.778,"skill":0.556,"coverage":1,"random":0.5,"tracks":[],"combined":0.6549,"private_eq":0.8289},"57":{"raw":0.7389,"skill":0.7063,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6865,"combined":0.6444},"58":{"raw":0.8974,"skill":0.8513,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.558,"combined":0.353},"59":{"raw":0.8601,"skill":0.7099,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3334,"private_eq":0.6817},"61":{"raw":0.902,"skill":0.804,"coverage":1,"random":0.5,"tracks":[],"combined":0.6605,"private_eq":0.8317},"62":{"raw":0.845,"skill":0.7933,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.8125,"skill":0.7656,"coverage":1,"random":0.2,"tracks":[],"combined":0.8167,"private_eq":0.8542}},"coverage":1,"pending":0,"notes":["New in the 2026-10-22 PR batch (PR #273).","Submitted public 0.3 run adopted (kit rescore 69.37, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.80% on 831 rows (bar 98.5%).","Likely derived from vLLM-SR d3 v3.1.0: the d3 runtime files are byte-identical and inference/NOTICE credits d3; the card names only Qwen3.8-27B as base."],"scores":{"balanced_raw":77.03,"balanced_skill":68.49,"breadth_skill":68.66,"public_skill":69.37},"frozen_scores":{"balanced_raw":77.03,"balanced_skill":69.37,"breadth_skill":68.66},"categories":[{"id":"knowledge","raw":0.6673,"skill":0.5895,"combined":0.5102,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.819,"skill":0.7405,"combined":0.6264,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7748,"skill":0.7005,"combined":0.6905,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8809,"skill":0.8603,"combined":0.8481,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6998,"skill":0.5242,"combined":0.5839,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"wemm-jev-27b","pub":69.37,"same_skills":65.39,"new_domains":57.45,"v":68.49,"lab":false,"jev":false,"rank_v":8,"rank_pub":14,"name":"WeMM-JEV-27B","eq":{"S":65.09,"O":73.6},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":39.05,"new_domains":29.44}},"new_in_edition":true},{"engine":"knowline-31b-gen1","name":"KnowLine 31B Gen1","long_name":"KnowLine 31B Gen1","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":31273088876,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/PelaAI/KnowLine-31B-Gen1","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"PelaAI","name":"PelaAI","logo":"data/orgs/PelaAI.png"},"pr":283,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7973,"conf":0.828,"ece":0.0321,"brier":0.2713,"over95":0.0089,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0017,"conf":0.0875,"acc":0.1183},{"lo":0.1,"w":0.0121,"conf":0.1525,"acc":0.1816},{"lo":0.2,"w":0.0135,"conf":0.2547,"acc":0.2801},{"lo":0.3,"w":0.0284,"conf":0.3577,"acc":0.2924},{"lo":0.4,"w":0.0444,"conf":0.4551,"acc":0.3527},{"lo":0.5,"w":0.079,"conf":0.5475,"acc":0.4895},{"lo":0.6,"w":0.0803,"conf":0.6514,"acc":0.5736},{"lo":0.7,"w":0.0728,"conf":0.7512,"acc":0.7185},{"lo":0.8,"w":0.102,"conf":0.8535,"acc":0.8136},{"lo":0.9,"w":0.566,"conf":0.9805,"acc":0.967}],"areas":{"knowledge":{"ece":0.0658,"acc":0.7284,"conf":0.7908},"language":{"ece":0.0081,"acc":0.8893,"conf":0.8892},"retrieval":{"ece":0.0703,"acc":0.7382,"conf":0.8005},"tools":{"ece":0.0382,"acc":0.9503,"conf":0.9869},"arts":{"ece":0.0178,"acc":0.7253,"conf":0.7186}}},"gaps":null,"latency":{"median":203.7,"p95":1822,"mean":459.3},"results":{"1":{"score":0.9776,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6939,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8898,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8287,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9315,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7993,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.8182,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.5578,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.7938,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.8027,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9748,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6933,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.7435,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6918,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8966,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5459,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9787,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.94,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9628,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.9261,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.316,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6902,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3933,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.5036,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.6092,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.4744,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9299,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6847,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.8199,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8772,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.9263,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8942,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1537,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.5537,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.9795,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7661,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.8771,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8262,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8578,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.9118,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8485,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9776,"skill":0.9697,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9739,"private_eq":0.9801},"2":{"raw":0.6939,"skill":0.6465,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7057,"private_eq":0.7472},"3":{"raw":0.8898,"skill":0.8877,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9433,"private_eq":0.944},"4":{"raw":0.8287,"skill":0.8265,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9149,"private_eq":0.9159},"5":{"raw":0.9315,"skill":0.9311,"coverage":1,"random":0.006,"tracks":[],"combined":0.8248,"private_eq":0.8268},"6":{"raw":0.7993,"skill":0.5294,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8182,"skill":0.8182,"coverage":1,"random":0,"tracks":[],"combined":0.7835,"private_eq":0.785},"10":{"raw":0.5578,"skill":0.2642,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7938,"skill":0.7018,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.8027,"skill":0.7045,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4875,"private_eq":0.6611},"20":{"raw":0.9751,"skill":0.9502,"coverage":1,"random":0.5,"tracks":[],"combined":0.9794,"private_eq":0.9893},"21":{"raw":0.6933,"skill":0.3866,"coverage":1,"random":0.5,"tracks":[],"combined":0.3672,"private_eq":0.6867},"22":{"raw":0.7392,"skill":0.7372,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5981,"private_eq":0.6048},"23":{"raw":0.6964,"skill":0.3927,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8966,"skill":0.8621,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5459,"skill":0.3946,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5594,"combined":0.4068},"26":{"raw":0.9899,"skill":0.9865,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9787,"skill":0.9716,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.94,"skill":0.88,"coverage":1,"random":0.5,"tracks":[],"combined":0.8244,"private_eq":0.9127},"29":{"raw":0.9628,"skill":0.9504,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.9261,"skill":0.9119,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9462,"random":0.25},{"track":"10-choice","raw":0.906,"random":0.1}],"combined":0.8903,"private_eq":0.9097},"31":{"raw":0.316,"skill":0.255,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0983,"private_eq":0.1807},"32":{"raw":0.6902,"skill":0.5075,"coverage":1,"random":0.371,"tracks":[],"combined":0.444,"private_eq":0.6536},"33":{"raw":0.3933,"skill":0.3852,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4512,"private_eq":0.4636},"36":{"raw":0.5036,"skill":0.4384,"coverage":1,"random":0.116,"tracks":[],"combined":0.4147,"private_eq":0.4876},"37":{"raw":0.6092,"skill":0.5099,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6228,"private_eq":0.7019},"38":{"raw":0.4744,"skill":0.4576,"coverage":1,"random":0.031,"tracks":[],"combined":0.6897,"private_eq":0.7018},"39":{"raw":0.9299,"skill":0.8969,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5877,"private_eq":0.7222},"40":{"raw":0.6847,"skill":0.5944,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.6214,"headline":false},{"track":"A · English","score":0.6847,"headline":true},{"track":"C · Arabic pairs","score":0.93,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.6717,"private_eq":0.7469},"41":{"raw":0.8199,"skill":0.7298,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5788,"private_eq":0.7217},"42":{"raw":0.8772,"skill":0.7611,"coverage":1,"random":0.486,"tracks":[],"combined":0.895,"private_eq":0.9461},"43":{"raw":0.9263,"skill":0.8831,"coverage":1,"random":0.3697,"tracks":[],"combined":0.7125,"private_eq":0.8202},"44":{"raw":0.8942,"skill":0.7884,"coverage":1,"random":0.5,"tracks":[],"combined":0.7279,"private_eq":0.865},"45":{"raw":0.1537,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5537,"skill":0.3523,"coverage":1,"random":0.311,"tracks":[],"combined":0.3047,"private_eq":0.5257},"56":{"raw":0.9795,"skill":0.959,"coverage":1,"random":0.5,"tracks":[],"combined":0.6377,"private_eq":0.8204},"57":{"raw":0.7661,"skill":0.7369,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6055,"combined":0.5522},"58":{"raw":0.8771,"skill":0.8219,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5968,"combined":0.4098},"59":{"raw":0.8262,"skill":0.6396,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4366,"private_eq":0.7309},"61":{"raw":0.8578,"skill":0.7156,"coverage":1,"random":0.5,"tracks":[],"combined":0.6834,"private_eq":0.843},"62":{"raw":0.9118,"skill":0.8824,"coverage":1,"random":0.25,"tracks":[],"combined":0.652,"private_eq":0.7412},"64":{"raw":0.8485,"skill":0.8106,"coverage":1,"random":0.2,"tracks":[],"combined":0.8264,"private_eq":0.8619}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #283).","Submitted public 0.3 run adopted (kit rescore 70.13, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.65% on 831 rows (bar 98.5%).","Read twice: short text-only requests are written a second time in the same prompt before the answer is read; one prefill per question.","Training includes train splits of Decision Index benchmarks in request format (author-declared).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":77.07,"balanced_skill":71.86,"breadth_skill":69.64,"public_skill":70.13},"frozen_scores":{"balanced_raw":77.07,"balanced_skill":70.13,"breadth_skill":69.64},"categories":[{"id":"knowledge","raw":0.6738,"skill":0.5943,"combined":0.5515,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8005,"skill":0.7199,"combined":0.6721,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7823,"skill":0.7303,"combined":0.6862,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8638,"skill":0.8474,"combined":0.8225,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.751,"skill":0.6049,"combined":0.6152,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"knowline-31b-gen1","pub":70.13,"same_skills":67.22,"new_domains":62.02,"v":71.86,"lab":false,"jev":false,"rank_v":1,"rank_pub":11,"name":"KnowLine 31B Gen1","eq":{"S":66.94,"O":79.24},"rank_tie":1,"tied":true,"parts":{"public":0,"same_skills":40.17,"new_domains":31.7}},"new_in_edition":true},{"engine":"h2o-lightning-31b-v1.1","name":"H2O Lightning 31B v1.1","long_name":"H2O Lightning 31B v1.1","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":31273086512,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/h2oai/h2o-lightning-31b","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"h2oai","name":"H2O.ai","logo":"data/orgs/h2oai.png"},"pr":276,"replaces":"h2o-lightning-31b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7852,"conf":0.8552,"ece":0.0743,"brier":0.326,"over95":0.0456,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0094,"conf":0.0778,"acc":0.1213},{"lo":0.1,"w":0.0124,"conf":0.1407,"acc":0.2281},{"lo":0.2,"w":0.007,"conf":0.2546,"acc":0.3482},{"lo":0.3,"w":0.0106,"conf":0.3607,"acc":0.3082},{"lo":0.4,"w":0.0253,"conf":0.4584,"acc":0.3659},{"lo":0.5,"w":0.0611,"conf":0.5489,"acc":0.507},{"lo":0.6,"w":0.0502,"conf":0.6503,"acc":0.5749},{"lo":0.7,"w":0.0586,"conf":0.7528,"acc":0.6342},{"lo":0.8,"w":0.0758,"conf":0.8555,"acc":0.6876},{"lo":0.9,"w":0.6896,"conf":0.9577,"acc":0.8949}],"areas":{"knowledge":{"ece":0.0817,"acc":0.724,"conf":0.7922},"language":{"ece":0.0203,"acc":0.8887,"conf":0.8948},"retrieval":{"ece":0.2636,"acc":0.6542,"conf":0.9134},"tools":{"ece":0.0181,"acc":0.9483,"conf":0.9594},"arts":{"ece":0.0728,"acc":0.728,"conf":0.8008}}},"gaps":null,"latency":{"median":100,"p95":701.6,"mean":207.1},"results":{"1":{"score":0.977,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6916,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8445,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.9138,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9516,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.7863,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":1,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.0338,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.8304,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7784,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.9836,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6872,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.654,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.658,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.8848,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.5612,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9928,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.977,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.9376,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9579,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.8825,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2484,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.6902,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3006,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4278,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.5838,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.4653,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9169,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6525,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.806,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8543,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.8649,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8912,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1337,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.6372,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.82,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.7324,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.8756,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.7841,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.819,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8828,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.8428,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.977,"skill":0.9689,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9669,"private_eq":0.975},"2":{"raw":0.6916,"skill":0.6439,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7006,"private_eq":0.7428},"3":{"raw":0.8445,"skill":0.8415,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.9138,"skill":0.9127,"coverage":1,"random":0.0127,"tracks":[],"combined":0.88,"private_eq":0.8819},"5":{"raw":0.9516,"skill":0.9513,"coverage":1,"random":0.006,"tracks":[],"combined":0.8267,"private_eq":0.8287},"6":{"raw":0.7863,"skill":0.4989,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":1,"skill":1,"coverage":1,"random":0,"tracks":[],"combined":0.8796,"private_eq":0.88},"10":{"raw":0.0338,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8304,"skill":0.7547,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7784,"skill":0.6681,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5772,"private_eq":0.7203},"20":{"raw":0.9835,"skill":0.967,"coverage":1,"random":0.5,"tracks":[],"combined":0.9867,"private_eq":0.9929},"21":{"raw":0.6872,"skill":0.3744,"coverage":1,"random":0.5,"tracks":[],"combined":0.317,"private_eq":0.6619},"22":{"raw":0.6476,"skill":0.6448,"coverage":1,"random":0.0078,"tracks":[],"combined":0.3783,"private_eq":0.3892},"23":{"raw":0.6608,"skill":0.3215,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8848,"skill":0.8464,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5612,"skill":0.415,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5199,"combined":0.3535},"26":{"raw":0.9928,"skill":0.9904,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.977,"skill":0.9693,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9376,"skill":0.8752,"coverage":1,"random":0.5,"tracks":[],"combined":0.6379,"private_eq":0.8205},"29":{"raw":0.9579,"skill":0.9439,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8825,"skill":0.8603,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.9181,"random":0.25},{"track":"10-choice","raw":0.8469,"random":0.1}],"combined":0.8251,"private_eq":0.8565},"31":{"raw":0.2484,"skill":0.1814,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0626,"private_eq":0.1483},"32":{"raw":0.6902,"skill":0.5075,"coverage":1,"random":0.371,"tracks":[],"combined":0.3899,"private_eq":0.62},"33":{"raw":0.3006,"skill":0.2913,"coverage":1,"random":0.0131,"tracks":[],"combined":0.405,"private_eq":0.4185},"36":{"raw":0.4278,"skill":0.3527,"coverage":1,"random":0.116,"tracks":[],"combined":0.4006,"private_eq":0.4753},"37":{"raw":0.5838,"skill":0.4779,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6157,"private_eq":0.6963},"38":{"raw":0.4653,"skill":0.4482,"coverage":1,"random":0.031,"tracks":[],"combined":0.6065,"private_eq":0.6221},"39":{"raw":0.9169,"skill":0.8778,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5944,"private_eq":0.7267},"40":{"raw":0.6525,"skill":0.553,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.605,"headline":false},{"track":"A · English","score":0.6525,"headline":true},{"track":"C · Arabic pairs","score":0.91,"headline":false},{"track":"C · English pairs","score":0.975,"headline":false}],"combined":0.5376,"private_eq":0.6439},"41":{"raw":0.806,"skill":0.709,"coverage":1,"random":0.3333,"tracks":[],"combined":0.6603,"private_eq":0.7754},"42":{"raw":0.8543,"skill":0.7165,"coverage":1,"random":0.486,"tracks":[],"combined":0.9628,"private_eq":0.9806},"43":{"raw":0.8649,"skill":0.7857,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6791,"private_eq":0.7994},"44":{"raw":0.8912,"skill":0.7824,"coverage":1,"random":0.5,"tracks":[],"combined":0.6905,"private_eq":0.8465},"45":{"raw":0.1337,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6372,"skill":0.4735,"coverage":1,"random":0.311,"tracks":[],"combined":0.283,"private_eq":0.5109},"56":{"raw":0.82,"skill":0.64,"coverage":1,"random":0.5,"tracks":[],"combined":0.7848,"private_eq":0.8931},"57":{"raw":0.7324,"skill":0.699,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6504,"combined":0.6033},"58":{"raw":0.8756,"skill":0.8197,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.528,"combined":0.309},"59":{"raw":0.7841,"skill":0.5524,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3645,"private_eq":0.6965},"61":{"raw":0.819,"skill":0.638,"coverage":1,"random":0.5,"tracks":[],"combined":0.6662,"private_eq":0.8345},"62":{"raw":0.8828,"skill":0.8437,"coverage":1,"random":0.25,"tracks":[],"combined":0.6725,"private_eq":0.7564},"64":{"raw":0.8428,"skill":0.8035,"coverage":1,"random":0.2,"tracks":[],"combined":0.7585,"private_eq":0.8082}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #276).","Replaces the earlier board entry h2o-lightning-31b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 67.75, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8 at load); decision agreement with the submitted rows 98.12% on 831 rows (bar 96%).","Served in FP8 at load as in the submitted run, so the quantized agreement bar applies.","Same weights as H2O Lightning 31B; v1.1 changes inference: a short text question whose first read is under 90% confident is read a second time and the two reads are averaged.","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":75.7,"balanced_skill":68.64,"breadth_skill":67.13,"public_skill":67.75},"frozen_scores":{"balanced_raw":75.7,"balanced_skill":67.75,"breadth_skill":67.13},"categories":[{"id":"knowledge","raw":0.6472,"skill":0.5649,"combined":0.4941,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7865,"skill":0.6964,"combined":0.6691,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7537,"skill":0.6691,"combined":0.6963,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8815,"skill":0.863,"combined":0.837,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7432,"skill":0.5974,"combined":0.5447,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"h2o-lightning-31b-v1.1","pub":67.75,"same_skills":65.6,"new_domains":57.49,"v":68.64,"lab":false,"jev":false,"rank_v":7,"rank_pub":20,"name":"H2O Lightning 31B v1.1","eq":{"S":65.3,"O":73.64},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":39.18,"new_domains":29.46}},"new_in_edition":true},{"engine":"oneshot-27b","name":"OneShot 27B","long_name":"OneShot 27B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/OneShotOrg/OneShot","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"OneShotOrg","name":"OneShot","logo":"data/orgs/OneShotOrg.png"},"pr":275,"new":true},"completed":110201,"counts":{"ok":110184,"unsupported":0,"error":17},"lifted":false,"calibration":{"acc":0.769,"conf":0.7783,"ece":0.0231,"brier":0.3166,"over95":0.0049,"n":34219,"benchmarks":31,"rel":[{"lo":0,"w":0.005,"conf":0.0796,"acc":0.2265},{"lo":0.1,"w":0.0171,"conf":0.1517,"acc":0.2404},{"lo":0.2,"w":0.0244,"conf":0.2536,"acc":0.2745},{"lo":0.3,"w":0.0371,"conf":0.3539,"acc":0.3017},{"lo":0.4,"w":0.0505,"conf":0.4497,"acc":0.472},{"lo":0.5,"w":0.0932,"conf":0.5491,"acc":0.581},{"lo":0.6,"w":0.0924,"conf":0.6477,"acc":0.6213},{"lo":0.7,"w":0.0968,"conf":0.7506,"acc":0.7306},{"lo":0.8,"w":0.1385,"conf":0.8516,"acc":0.7946},{"lo":0.9,"w":0.445,"conf":0.97,"acc":0.9656}],"areas":{"knowledge":{"ece":0.0551,"acc":0.6921,"conf":0.7433},"language":{"ece":0.0416,"acc":0.8892,"conf":0.8475},"retrieval":{"ece":0.1608,"acc":0.6583,"conf":0.8131},"tools":{"ece":0.0218,"acc":0.9457,"conf":0.9435},"arts":{"ece":0.0922,"acc":0.6888,"conf":0.5966}}},"gaps":null,"latency":{"median":127.8,"p95":1245.2,"mean":361},"results":{"1":{"score":0.9699,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":598.3},"2":{"score":0.6566,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":5947.1},"3":{"score":0.8681,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2333.2},"4":{"score":0.8604,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":553.8},"5":{"score":0.9507,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":933.8},"6":{"score":0.7958,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1470.7},"9":{"score":0.7841,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":6618.6},"10":{"score":0.0719,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":439.9},"11":{"score":0.8289,"jev":null,"metric":"macro-F1","requests":123,"answered":122,"unsupported":0,"errors":1,"median_ms":10181.4},"12":{"score":0.8046,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":144.3},"20":{"score":0.9536,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":144.3},"21":{"score":0.6465,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":118.1},"22":{"score":0.5545,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2221.1},"23":{"score":0.6642,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":246.6},"24":{"score":0.8769,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":122.7},"25":{"score":0.5051,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":149.7},"26":{"score":0.992,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":120.3},"27":{"score":0.9693,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":120.9},"28":{"score":0.9408,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":117.2},"29":{"score":0.9705,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":173.7},"30":{"score":0.7885,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2174,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":435.7},"32":{"score":0.7128,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":368},"33":{"score":0.3073,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1288.7},"36":{"score":0.4861,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":9526.9},"37":{"score":0.6288,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":183.1},"38":{"score":0.4279,"jev":null,"metric":"per-review F1","requests":1565,"answered":1544,"unsupported":0,"errors":21,"median_ms":5494.4},"39":{"score":0.9256,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":227.7},"40":{"score":0.635,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":118},"41":{"score":0.7982,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":146.1},"42":{"score":0.8648,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":226.8},"43":{"score":0.8842,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":144.4},"44":{"score":0.8052,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":143.6},"45":{"score":0.1497,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":174.8},"50":{"score":0.6492,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":333.5},"56":{"score":0.7265,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1079.2},"57":{"score":0.7022,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":174.3},"58":{"score":0.8402,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":145.8},"59":{"score":0.8399,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":317.9},"61":{"score":0.818,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":219.2},"62":{"score":0.8823,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":321.5},"64":{"score":0.7367,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":149.3}},"benchmarks":{"1":{"raw":0.9699,"skill":0.9594,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.6566,"skill":0.6035,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7125,"private_eq":0.753},"3":{"raw":0.8681,"skill":0.8656,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9171,"private_eq":0.9186},"4":{"raw":0.8604,"skill":0.8586,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8845,"private_eq":0.8863},"5":{"raw":0.9507,"skill":0.9504,"coverage":1,"random":0.006,"tracks":[],"combined":0.8595,"private_eq":0.8609},"6":{"raw":0.7958,"skill":0.5212,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7841,"skill":0.7841,"coverage":1,"random":0,"tracks":[],"combined":0.7583,"private_eq":0.76},"10":{"raw":0.0719,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8152,"skill":0.7327,"coverage":0.9919,"random":0.3085,"tracks":[]},"12":{"raw":0.8046,"skill":0.7073,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5639,"private_eq":0.7115},"20":{"raw":0.955,"skill":0.9099,"coverage":1,"random":0.5,"tracks":[],"combined":0.9215,"private_eq":0.9607},"21":{"raw":0.6465,"skill":0.293,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.5501,"skill":0.5466,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4996,"private_eq":0.5082},"23":{"raw":0.6692,"skill":0.3383,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8769,"skill":0.8359,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5179,"combined":0.3508},"26":{"raw":0.992,"skill":0.9893,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9693,"skill":0.9591,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9408,"skill":0.8816,"coverage":1,"random":0.5,"tracks":[],"combined":0.6075,"private_eq":0.8055},"29":{"raw":0.9705,"skill":0.9607,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7885,"skill":0.7463,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8317,"random":0.25},{"track":"10-choice","raw":0.7453,"random":0.1}],"combined":0.7161,"private_eq":0.7676},"31":{"raw":0.2174,"skill":0.1476,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0745,"private_eq":0.1591},"32":{"raw":0.7128,"skill":0.5434,"coverage":1,"random":0.371,"tracks":[],"combined":0.3666,"private_eq":0.6055},"33":{"raw":0.3073,"skill":0.2981,"coverage":1,"random":0.0131,"tracks":[],"combined":0.364,"private_eq":0.3785},"36":{"raw":0.4861,"skill":0.4186,"coverage":1,"random":0.116,"tracks":[],"combined":0.4116,"private_eq":0.4849},"37":{"raw":0.6288,"skill":0.5344,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6056,"private_eq":0.6883},"38":{"raw":0.4222,"skill":0.4037,"coverage":0.9866,"random":0.031,"tracks":[],"combined":0.5518,"private_eq":0.5697},"39":{"raw":0.9256,"skill":0.8906,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6283,"private_eq":0.7495},"40":{"raw":0.635,"skill":0.5305,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4706,"headline":false},{"track":"A · English","score":0.635,"headline":true},{"track":"C · Arabic pairs","score":0.87,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.5979,"private_eq":0.6902},"41":{"raw":0.7982,"skill":0.6973,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4545,"private_eq":0.6398},"42":{"raw":0.8648,"skill":0.737,"coverage":1,"random":0.486,"tracks":[],"combined":0.9123,"private_eq":0.9549},"43":{"raw":0.8842,"skill":0.8163,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5746,"private_eq":0.7343},"44":{"raw":0.8052,"skill":0.6104,"coverage":1,"random":0.5,"tracks":[],"combined":0.6583,"private_eq":0.8306},"45":{"raw":0.1497,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6492,"skill":0.4908,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.7265,"skill":0.453,"coverage":1,"random":0.5,"tracks":[],"combined":0.4991,"private_eq":0.7519},"57":{"raw":0.7022,"skill":0.6651,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4542,"combined":0.3801},"58":{"raw":0.8402,"skill":0.7684,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5295,"combined":0.3112},"59":{"raw":0.8399,"skill":0.668,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3618,"private_eq":0.6952},"61":{"raw":0.818,"skill":0.636,"coverage":1,"random":0.5,"tracks":[],"combined":0.6435,"private_eq":0.8233},"62":{"raw":0.8823,"skill":0.8431,"coverage":1,"random":0.25,"tracks":[],"combined":0.6656,"private_eq":0.7513},"64":{"raw":0.7367,"skill":0.6709,"coverage":1,"random":0.2,"tracks":[],"combined":0.6713,"private_eq":0.7392}},"coverage":0.9995,"pending":0.0005,"notes":["New in the 2026-10-27 PR batch (PR #275).","Submitted public 0.3 run adopted (kit rescore 65.41, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.85% on 831 rows (bar 98.5%).","Base model not declared; Qwen3.8-27B inferred from the tokenizer.","Full-attention layers read the prompt bidirectionally at inference.","Training data undisclosed."],"scores":{"balanced_raw":73.85,"balanced_skill":65.74,"breadth_skill":64.69,"public_skill":65.41},"frozen_scores":{"balanced_raw":73.85,"balanced_skill":65.41,"breadth_skill":64.69},"categories":[{"id":"knowledge","raw":0.6234,"skill":0.529,"combined":0.4383,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7905,"skill":0.7087,"combined":0.641,"coverage":0.9977,"pending":0,"n":9},{"id":"retrieval","raw":0.747,"skill":0.651,"combined":0.6568,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8386,"skill":0.8186,"combined":0.8113,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7011,"skill":0.5416,"combined":0.5289,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"oneshot-27b","pub":65.41,"same_skills":62.05,"new_domains":55.98,"v":65.74,"lab":false,"jev":false,"rank_v":28,"rank_pub":35,"name":"OneShot 27B","eq":{"S":61.71,"O":71.78},"rank_tie":6,"tied":true,"parts":{"public":0,"same_skills":37.03,"new_domains":28.71}},"new_in_edition":true},{"engine":"turion-jev-31b","name":"Turion Jev 31B","long_name":"Turion Jev 31B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":31273086512,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/turion-ai/turion-jev-31b","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"turion-ai","name":"Turion.AI","logo":"data/orgs/turion-ai.png"},"pr":286,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7684,"conf":0.7674,"ece":0.0193,"brier":0.3077,"over95":0.0064,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0105,"conf":0.0818,"acc":0.0606},{"lo":0.1,"w":0.0329,"conf":0.1426,"acc":0.2228},{"lo":0.2,"w":0.0215,"conf":0.2541,"acc":0.2968},{"lo":0.3,"w":0.0397,"conf":0.3558,"acc":0.3909},{"lo":0.4,"w":0.0458,"conf":0.4567,"acc":0.4419},{"lo":0.5,"w":0.0845,"conf":0.5519,"acc":0.5307},{"lo":0.6,"w":0.0994,"conf":0.6512,"acc":0.5942},{"lo":0.7,"w":0.0947,"conf":0.752,"acc":0.7436},{"lo":0.8,"w":0.1334,"conf":0.8543,"acc":0.887},{"lo":0.9,"w":0.4376,"conf":0.9707,"acc":0.9727}],"areas":{"knowledge":{"ece":0.0286,"acc":0.7166,"conf":0.7317},"language":{"ece":0.0544,"acc":0.8896,"conf":0.8352},"retrieval":{"ece":0.1543,"acc":0.6472,"conf":0.7912},"tools":{"ece":0.0358,"acc":0.934,"conf":0.9542},"arts":{"ece":0.0665,"acc":0.6515,"conf":0.5857}}},"gaps":null,"latency":{"median":143.8,"p95":814.8,"mean":299.2},"results":{"1":{"score":0.9758,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":182.1},"2":{"score":0.7036,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3318.4},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1211.7},"4":{"score":0.8267,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":322.2},"5":{"score":0.9244,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":476.7},"6":{"score":0.7997,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":392},"9":{"score":0.6477,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":545.5},"10":{"score":0.0465,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":372.1},"11":{"score":0.8263,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":669.3},"12":{"score":0.8022,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":88.7},"20":{"score":0.9858,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":108.6},"21":{"score":0.6655,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":76.1},"22":{"score":0.3885,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1163.2},"23":{"score":0.6664,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":223.5},"24":{"score":0.8974,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":51.7},"25":{"score":0.5561,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":101.2},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":47},"27":{"score":0.9795,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":50.8},"28":{"score":0.9092,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":84.7},"29":{"score":0.9246,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":102},"30":{"score":0.8275,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2338,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":282.1},"32":{"score":0.7181,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":502.1},"33":{"score":0.3339,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":203.6},"36":{"score":0.5152,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5334.5},"37":{"score":0.6193,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":105.6},"38":{"score":0.4421,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1801.8},"39":{"score":0.9373,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":65.9},"40":{"score":0.6838,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":80.1},"41":{"score":0.7927,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":93.1},"42":{"score":0.8317,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":318},"43":{"score":0.893,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":91.6},"44":{"score":0.878,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":125.3},"45":{"score":0.1577,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":104.1},"50":{"score":0.5274,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":283.5},"56":{"score":0.958,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":160.9},"57":{"score":0.7441,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":66.6},"58":{"score":0.8275,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":83.2},"59":{"score":0.813,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":178.6},"61":{"score":0.8095,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":227.6},"62":{"score":0.8206,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":191.8},"64":{"score":0.7992,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":64.2}},"benchmarks":{"1":{"raw":0.9758,"skill":0.9673,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9534,"private_eq":0.9651},"2":{"raw":0.7036,"skill":0.6577,"coverage":1,"random":0.1341,"tracks":[],"combined":0.717,"private_eq":0.7569},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9224,"private_eq":0.9237},"4":{"raw":0.8267,"skill":0.8245,"coverage":1,"random":0.0127,"tracks":[],"combined":0.9102,"private_eq":0.9114},"5":{"raw":0.9244,"skill":0.9239,"coverage":1,"random":0.006,"tracks":[],"combined":0.8208,"private_eq":0.8229},"6":{"raw":0.7997,"skill":0.5304,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6477,"skill":0.6477,"coverage":1,"random":0,"tracks":[],"combined":0.6596,"private_eq":0.6625},"10":{"raw":0.0465,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8263,"skill":0.7488,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.8022,"skill":0.7037,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4477,"private_eq":0.6348},"20":{"raw":0.986,"skill":0.972,"coverage":1,"random":0.5,"tracks":[],"combined":0.9867,"private_eq":0.9929},"21":{"raw":0.6655,"skill":0.3311,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.3848,"skill":0.38,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4132,"private_eq":0.4234},"23":{"raw":0.6647,"skill":0.3293,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8974,"skill":0.8632,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5561,"skill":0.4082,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5449,"combined":0.3872},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9795,"skill":0.9727,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9092,"skill":0.8184,"coverage":1,"random":0.5,"tracks":[],"combined":0.7842,"private_eq":0.8928},"29":{"raw":0.9246,"skill":0.8995,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8275,"skill":0.7951,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.881,"random":0.25},{"track":"10-choice","raw":0.7741,"random":0.1}],"combined":0.8077,"private_eq":0.8423},"31":{"raw":0.2338,"skill":0.1655,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0886,"private_eq":0.1719},"32":{"raw":0.7181,"skill":0.5518,"coverage":1,"random":0.371,"tracks":[],"combined":0.4904,"private_eq":0.6825},"33":{"raw":0.3339,"skill":0.3251,"coverage":1,"random":0.0131,"tracks":[],"combined":0.4204,"private_eq":0.4335},"36":{"raw":0.5152,"skill":0.4516,"coverage":1,"random":0.116,"tracks":[],"combined":0.4307,"private_eq":0.5016},"37":{"raw":0.6193,"skill":0.5224,"coverage":1,"random":0.2027,"tracks":[],"combined":0.6208,"private_eq":0.7003},"38":{"raw":0.4421,"skill":0.4243,"coverage":1,"random":0.031,"tracks":[],"combined":0.6009,"private_eq":0.6167},"39":{"raw":0.9373,"skill":0.9078,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5355,"private_eq":0.6871},"40":{"raw":0.6838,"skill":0.5932,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5654,"headline":false},{"track":"A · English","score":0.6838,"headline":true},{"track":"C · Arabic pairs","score":0.915,"headline":false},{"track":"C · English pairs","score":0.985,"headline":false}],"combined":0.6599,"private_eq":0.7379},"41":{"raw":0.7927,"skill":0.689,"coverage":1,"random":0.3333,"tracks":[],"combined":0.5588,"private_eq":0.7085},"42":{"raw":0.8317,"skill":0.6726,"coverage":1,"random":0.486,"tracks":[],"combined":0.9882,"private_eq":0.9935},"43":{"raw":0.893,"skill":0.8302,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6749,"private_eq":0.7968},"44":{"raw":0.878,"skill":0.756,"coverage":1,"random":0.5,"tracks":[],"combined":0.6959,"private_eq":0.8492},"45":{"raw":0.1577,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5274,"skill":0.3142,"coverage":1,"random":0.311,"tracks":[],"combined":0.2466,"private_eq":0.4861},"56":{"raw":0.958,"skill":0.916,"coverage":1,"random":0.5,"tracks":[],"combined":0.8885,"private_eq":0.9444},"57":{"raw":0.7441,"skill":0.7122,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5947,"combined":0.5399},"58":{"raw":0.8275,"skill":0.75,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5645,"combined":0.3625},"59":{"raw":0.813,"skill":0.6123,"coverage":1,"random":0.5177,"tracks":[],"combined":0.4001,"private_eq":0.7135},"61":{"raw":0.8095,"skill":0.619,"coverage":1,"random":0.5,"tracks":[],"combined":0.6209,"private_eq":0.8121},"62":{"raw":0.8206,"skill":0.7608,"coverage":1,"random":0.25,"tracks":[],"combined":0.5698,"private_eq":0.6803},"64":{"raw":0.7992,"skill":0.749,"coverage":1,"random":0.2,"tracks":[],"combined":0.8264,"private_eq":0.8619}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #286).","Submitted public 0.3 run adopted (kit rescore 66.16, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving (FP8 at load); decision agreement with the submitted rows 98.59% on 829 rows (bar 96%).","Served in FP8 at load as in the submitted run, so the quantized agreement bar applies.","Training includes a distillation corpus of TypeSafe Jev teacher distributions and synthetic labels (author-declared)."],"scores":{"balanced_raw":73.96,"balanced_skill":70.47,"breadth_skill":65.56,"public_skill":66.16},"frozen_scores":{"balanced_raw":73.96,"balanced_skill":66.16,"breadth_skill":65.56},"categories":[{"id":"knowledge","raw":0.6432,"skill":0.5542,"combined":0.5247,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7871,"skill":0.6991,"combined":0.6526,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7737,"skill":0.7117,"combined":0.7158,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8089,"skill":0.7868,"combined":0.7773,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6713,"skill":0.5126,"combined":0.56,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"turion-jev-31b","pub":66.16,"same_skills":65.34,"new_domains":61.5,"v":70.47,"lab":false,"jev":false,"rank_v":4,"rank_pub":28,"name":"Turion Jev 31B","eq":{"S":65.04,"O":78.6},"rank_tie":2,"tied":true,"parts":{"public":0,"same_skills":39.02,"new_domains":31.44}},"new_in_edition":true},{"engine":"verdin-4b-v1.1","name":"Verdin-4B v1.1","long_name":"Verdin-4B v1.1","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4205751296,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/victorbennett/Verdin-4B-v1.1","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"pr":300,"replaces":"verdin-4b","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7435,"conf":0.805,"ece":0.0627,"brier":0.3389,"over95":0.0166,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0093,"conf":0.0825,"acc":0.1097},{"lo":0.1,"w":0.0211,"conf":0.1397,"acc":0.156},{"lo":0.2,"w":0.0124,"conf":0.2568,"acc":0.1914},{"lo":0.3,"w":0.0387,"conf":0.3577,"acc":0.2879},{"lo":0.4,"w":0.0415,"conf":0.4586,"acc":0.3652},{"lo":0.5,"w":0.0908,"conf":0.5487,"acc":0.4945},{"lo":0.6,"w":0.078,"conf":0.6477,"acc":0.56},{"lo":0.7,"w":0.0743,"conf":0.7501,"acc":0.6298},{"lo":0.8,"w":0.099,"conf":0.855,"acc":0.7022},{"lo":0.9,"w":0.535,"conf":0.9805,"acc":0.9451}],"areas":{"knowledge":{"ece":0.078,"acc":0.6503,"conf":0.7248},"language":{"ece":0.0211,"acc":0.864,"conf":0.8809},"retrieval":{"ece":0.1831,"acc":0.6451,"conf":0.8232},"tools":{"ece":0.0412,"acc":0.9289,"conf":0.9687},"arts":{"ece":0.0394,"acc":0.6803,"conf":0.7161}}},"gaps":null,"latency":{"median":199.9,"p95":505.4,"mean":294.2},"results":{"1":{"score":0.9664,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":1590.7},"2":{"score":0.6777,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":24359},"3":{"score":0.8425,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":8863.9},"4":{"score":0.8758,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1718},"5":{"score":0.9264,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":2464.7},"6":{"score":0.7968,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":3345.7},"9":{"score":0.7159,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2687.9},"10":{"score":0.0925,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":1630.1},"11":{"score":0.8425,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1447.9},"12":{"score":0.7355,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":407.5},"20":{"score":0.9548,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":393.8},"21":{"score":0.6286,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":1494},"22":{"score":0.786,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":5580.6},"23":{"score":0.651,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":872.5},"24":{"score":0.7966,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":347.4},"25":{"score":0.4133,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":338.5},"26":{"score":0.9848,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":241.8},"27":{"score":0.9548,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":235.8},"28":{"score":0.8374,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":241.3},"29":{"score":0.9341,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":610.1},"30":{"score":0.652,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.202,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":1426.7},"32":{"score":0.6875,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":1585.8},"33":{"score":0.3012,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1569.8},"36":{"score":0.4563,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4136.8},"37":{"score":0.5845,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":1568.8},"38":{"score":0.1245,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":3125.5},"39":{"score":0.9342,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":1441},"40":{"score":0.5584,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":397.6},"41":{"score":0.7802,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":480.7},"42":{"score":0.8231,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":701.8},"43":{"score":0.6281,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":568.7},"44":{"score":0.879,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":539.1},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":774.1},"50":{"score":0.4469,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":865.5},"56":{"score":0.7945,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":703.1},"57":{"score":0.5994,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":485.5},"58":{"score":0.7908,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":446.7},"59":{"score":0.8079,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":930.4},"61":{"score":0.8612,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":796},"62":{"score":0.9069,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":950.6},"64":{"score":0.7102,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":855.6}},"benchmarks":{"1":{"raw":0.9664,"skill":0.9546,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9125,"private_eq":0.9352},"2":{"raw":0.6777,"skill":0.6278,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6847,"private_eq":0.7292},"3":{"raw":0.8425,"skill":0.8395,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8804,"private_eq":0.883},"4":{"raw":0.8758,"skill":0.8742,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8639,"private_eq":0.8662},"5":{"raw":0.9264,"skill":0.926,"coverage":1,"random":0.006,"tracks":[],"combined":0.7968,"private_eq":0.7993},"6":{"raw":0.7968,"skill":0.5236,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.7159,"skill":0.7159,"coverage":1,"random":0,"tracks":[],"combined":0.6596,"private_eq":0.6625},"10":{"raw":0.0925,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8425,"skill":0.7722,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7355,"skill":0.6038,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5054,"private_eq":0.6729},"20":{"raw":0.9561,"skill":0.9121,"coverage":1,"random":0.5,"tracks":[],"combined":0.9504,"private_eq":0.975},"21":{"raw":0.6286,"skill":0.2572,"coverage":1,"random":0.5,"tracks":[],"combined":0.3672,"private_eq":0.6867},"22":{"raw":0.7775,"skill":0.7757,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5971,"private_eq":0.6038},"23":{"raw":0.6439,"skill":0.2879,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7966,"skill":0.7288,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4133,"skill":0.2177,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3838,"combined":0.1699},"26":{"raw":0.9848,"skill":0.9797,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9548,"skill":0.9397,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8374,"skill":0.6748,"coverage":1,"random":0.5,"tracks":[],"combined":0.5017,"private_eq":0.7532},"29":{"raw":0.9341,"skill":0.9121,"coverage":1,"random":0.25,"tracks":[],"combined":0.9831,"private_eq":0.9867},"30":{"raw":0.652,"skill":0.5828,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7255,"random":0.25},{"track":"10-choice","raw":0.5785,"random":0.1}],"combined":0.6127,"private_eq":0.6833},"31":{"raw":0.202,"skill":0.1309,"coverage":1,"random":0.0819,"tracks":[],"combined":0.079,"private_eq":0.1631},"32":{"raw":0.6875,"skill":0.5032,"coverage":1,"random":0.371,"tracks":[],"combined":0.2119,"private_eq":0.5093},"33":{"raw":0.3012,"skill":0.2919,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2921,"private_eq":0.3083},"36":{"raw":0.4563,"skill":0.385,"coverage":1,"random":0.116,"tracks":[],"combined":0.3465,"private_eq":0.428},"37":{"raw":0.5845,"skill":0.4788,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5084,"private_eq":0.6117},"38":{"raw":0.1245,"skill":0.0965,"coverage":1,"random":0.031,"tracks":[],"combined":0.0095,"private_eq":0.0502},"39":{"raw":0.9342,"skill":0.9032,"coverage":1,"random":0.3201,"tracks":[],"combined":0.56,"private_eq":0.7036},"40":{"raw":0.5584,"skill":0.4319,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4788,"headline":false},{"track":"A · English","score":0.5584,"headline":true},{"track":"C · Arabic pairs","score":0.785,"headline":false},{"track":"C · English pairs","score":0.92,"headline":false}],"combined":0.4524,"private_eq":0.5784},"41":{"raw":0.7802,"skill":0.6702,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3283,"private_eq":0.5566},"42":{"raw":0.8231,"skill":0.6558,"coverage":1,"random":0.486,"tracks":[],"combined":0.7043,"private_eq":0.8492},"43":{"raw":0.6281,"skill":0.4099,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4491,"private_eq":0.6561},"44":{"raw":0.879,"skill":0.758,"coverage":1,"random":0.5,"tracks":[],"combined":0.7334,"private_eq":0.8677},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4469,"skill":0.1973,"coverage":1,"random":0.311,"tracks":[],"combined":0.3846,"private_eq":0.5801},"56":{"raw":0.7945,"skill":0.589,"coverage":1,"random":0.5,"tracks":[],"combined":0.0923,"private_eq":0.5508},"57":{"raw":0.5994,"skill":0.5494,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5674,"combined":0.5089},"58":{"raw":0.7908,"skill":0.6968,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4541,"combined":0.2006},"59":{"raw":0.8079,"skill":0.6017,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3399,"private_eq":0.6848},"61":{"raw":0.8612,"skill":0.7224,"coverage":1,"random":0.5,"tracks":[],"combined":0.5869,"private_eq":0.7953},"62":{"raw":0.9069,"skill":0.8759,"coverage":1,"random":0.25,"tracks":[],"combined":0.6177,"private_eq":0.7158},"64":{"raw":0.7102,"skill":0.6377,"coverage":1,"random":0.2,"tracks":[],"combined":0.6519,"private_eq":0.7239}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #300).","Replaces the earlier board entry verdin-4b (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 60.96, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.85% on 831 rows (bar 98.5%).","Three LoRA seeds averaged and merged into Qwen3.5-4B, with a pointer head.","Training includes train splits of public datasets that are also sources of index benchmarks (author-declared)."],"scores":{"balanced_raw":70.34,"balanced_skill":51.99,"breadth_skill":59.77,"public_skill":60.96},"frozen_scores":{"balanced_raw":70.34,"balanced_skill":60.96,"breadth_skill":59.77},"categories":[{"id":"knowledge","raw":0.555,"skill":0.4369,"combined":0.3736,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7313,"skill":0.633,"combined":0.4977,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7501,"skill":0.6686,"combined":0.5449,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.828,"skill":0.8097,"combined":0.7618,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6939,"skill":0.5113,"combined":0.5902,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"verdin-4b-v1.1","pub":60.96,"same_skills":54.13,"new_domains":37.88,"v":51.99,"lab":false,"jev":false,"rank_v":89,"rank_pub":55,"name":"Verdin-4B v1.1","eq":{"S":53.7,"O":49.43},"rank_tie":19,"tied":false,"parts":{"public":0,"same_skills":32.22,"new_domains":19.77}},"new_in_edition":true},{"engine":"vera-large","name":"vera-large","long_name":"vera-large","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25628527110,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/scar-ai/vera-large","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"pr":281,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7898,"conf":0.8383,"ece":0.0507,"brier":0.2864,"over95":0.0162,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0022,"conf":0.0872,"acc":0.1453},{"lo":0.1,"w":0.0173,"conf":0.1544,"acc":0.1467},{"lo":0.2,"w":0.0202,"conf":0.2494,"acc":0.2975},{"lo":0.3,"w":0.0291,"conf":0.3483,"acc":0.3086},{"lo":0.4,"w":0.0338,"conf":0.4559,"acc":0.3479},{"lo":0.5,"w":0.0641,"conf":0.5487,"acc":0.4763},{"lo":0.6,"w":0.0652,"conf":0.6498,"acc":0.5374},{"lo":0.7,"w":0.0751,"conf":0.7555,"acc":0.6583},{"lo":0.8,"w":0.0896,"conf":0.8539,"acc":0.7656},{"lo":0.9,"w":0.6032,"conf":0.9847,"acc":0.9558}],"areas":{"knowledge":{"ece":0.1031,"acc":0.6726,"conf":0.7755},"language":{"ece":0.0144,"acc":0.909,"conf":0.9187},"retrieval":{"ece":0.061,"acc":0.7992,"conf":0.8561},"tools":{"ece":0.0441,"acc":0.9483,"conf":0.9881},"arts":{"ece":0.0597,"acc":0.7072,"conf":0.7153}}},"gaps":null,"latency":{"median":159.7,"p95":1265.2,"mean":407.4},"results":{"1":{"score":0.9752,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":299.3},"2":{"score":0.685,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3254},"3":{"score":0.8504,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1155.6},"4":{"score":0.934,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":220.9},"5":{"score":0.9677,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":316.7},"6":{"score":0.7979,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":721.7},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2966.4},"10":{"score":0.6322,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":246.8},"11":{"score":0.861,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":4880.5},"12":{"score":0.802,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":140.9},"20":{"score":0.9952,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":138.9},"21":{"score":0.6872,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":140.9},"22":{"score":0.601,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":987.6},"23":{"score":0.6638,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":152.9},"24":{"score":0.8701,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":133.8},"25":{"score":0.5051,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":132.9},"26":{"score":0.9928,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":131.9},"27":{"score":0.9676,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":133.7},"28":{"score":0.9534,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":132.9},"29":{"score":0.9767,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":143.8},"30":{"score":0.8207,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1776,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":222.7},"32":{"score":0.6649,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":224.8},"33":{"score":0.2758,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":582.3},"36":{"score":0.4928,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5599.1},"37":{"score":0.6199,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":142.9},"38":{"score":0.6351,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2173.4},"39":{"score":0.9455,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":146.8},"40":{"score":0.6847,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":136.9},"41":{"score":0.7966,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":136},"42":{"score":0.8827,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":154.6},"43":{"score":0.8439,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":134.9},"44":{"score":0.7808,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":136.7},"45":{"score":0.1218,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":139.9},"50":{"score":0.6193,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":205.9},"56":{"score":0.7415,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":448.9},"57":{"score":0.7068,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":137.9},"58":{"score":0.8338,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":135.9},"59":{"score":0.8585,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":196.8},"61":{"score":0.8952,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":146.9},"62":{"score":0.8957,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":195.3},"64":{"score":0.8011,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":137.8}},"benchmarks":{"1":{"raw":0.9752,"skill":0.9665,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9602,"private_eq":0.9701},"2":{"raw":0.685,"skill":0.6362,"coverage":1,"random":0.1341,"tracks":[],"combined":0.7057,"private_eq":0.7472},"3":{"raw":0.8504,"skill":0.8475,"coverage":1,"random":0.0189,"tracks":[],"combined":0.9066,"private_eq":0.9084},"4":{"raw":0.934,"skill":0.9332,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8964,"private_eq":0.8979},"5":{"raw":0.9677,"skill":0.9675,"coverage":1,"random":0.006,"tracks":[],"combined":0.8411,"private_eq":0.8428},"6":{"raw":0.7979,"skill":0.5261,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.776,"private_eq":0.7775},"10":{"raw":0.6322,"skill":0.388,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.861,"skill":0.7989,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.802,"skill":0.7034,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4946,"private_eq":0.6658},"20":{"raw":0.9952,"skill":0.9904,"coverage":1,"random":0.5,"tracks":[],"combined":0.9794,"private_eq":0.9893},"21":{"raw":0.6872,"skill":0.3744,"coverage":1,"random":0.5,"tracks":[],"combined":0.3874,"private_eq":0.6967},"22":{"raw":0.6005,"skill":0.5974,"coverage":1,"random":0.0078,"tracks":[],"combined":0.4829,"private_eq":0.4918},"23":{"raw":0.6741,"skill":0.3482,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8701,"skill":0.8268,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5051,"skill":0.3401,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5043,"combined":0.3325},"26":{"raw":0.9928,"skill":0.9904,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9676,"skill":0.9568,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9534,"skill":0.9068,"coverage":1,"random":0.5,"tracks":[],"combined":0.6632,"private_eq":0.833},"29":{"raw":0.9767,"skill":0.9689,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.8207,"skill":0.7849,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8575,"random":0.25},{"track":"10-choice","raw":0.7839,"random":0.1}],"combined":0.7482,"private_eq":0.7938},"31":{"raw":0.1776,"skill":0.1043,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0875,"private_eq":0.1709},"32":{"raw":0.6649,"skill":0.4672,"coverage":1,"random":0.371,"tracks":[],"combined":0.3589,"private_eq":0.6007},"33":{"raw":0.2758,"skill":0.2662,"coverage":1,"random":0.0131,"tracks":[],"combined":0.143,"private_eq":0.1629},"36":{"raw":0.4928,"skill":0.4263,"coverage":1,"random":0.116,"tracks":[],"combined":0.4261,"private_eq":0.4976},"37":{"raw":0.6199,"skill":0.5233,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5972,"private_eq":0.6817},"38":{"raw":0.6351,"skill":0.6234,"coverage":1,"random":0.031,"tracks":[],"combined":0.7573,"private_eq":0.7665},"39":{"raw":0.9455,"skill":0.9198,"coverage":1,"random":0.3201,"tracks":[],"combined":0.6334,"private_eq":0.7529},"40":{"raw":0.6847,"skill":0.5944,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.6059,"headline":false},{"track":"A · English","score":0.6847,"headline":true},{"track":"C · Arabic pairs","score":0.96,"headline":false},{"track":"C · English pairs","score":0.98,"headline":false}],"combined":0.478,"private_eq":0.5981},"41":{"raw":0.7966,"skill":0.6949,"coverage":1,"random":0.3333,"tracks":[],"combined":0,"private_eq":0.3333},"42":{"raw":0.8827,"skill":0.7718,"coverage":1,"random":0.486,"tracks":[],"combined":0.9365,"private_eq":0.9672},"43":{"raw":0.8439,"skill":0.7523,"coverage":1,"random":0.3697,"tracks":[],"combined":0.6122,"private_eq":0.7577},"44":{"raw":0.7808,"skill":0.5616,"coverage":1,"random":0.5,"tracks":[],"combined":0.5406,"private_eq":0.7724},"45":{"raw":0.1218,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6193,"skill":0.4475,"coverage":1,"random":0.311,"tracks":[],"combined":0.2248,"private_eq":0.4713},"56":{"raw":0.7415,"skill":0.483,"coverage":1,"random":0.5,"tracks":[],"combined":0.6723,"private_eq":0.8375},"57":{"raw":0.7068,"skill":0.6702,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6084,"combined":0.5555},"58":{"raw":0.8338,"skill":0.7591,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5249,"combined":0.3044},"59":{"raw":0.8585,"skill":0.7066,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2649,"private_eq":0.649},"61":{"raw":0.8952,"skill":0.7904,"coverage":1,"random":0.5,"tracks":[],"combined":0.7174,"private_eq":0.8598},"62":{"raw":0.8957,"skill":0.8609,"coverage":1,"random":0.25,"tracks":[],"combined":0.6246,"private_eq":0.7209},"64":{"raw":0.8011,"skill":0.7514,"coverage":1,"random":0.2,"tracks":[],"combined":0.8071,"private_eq":0.8466}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #281).","Submitted public 0.3 run adopted (kit rescore 67.75, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.70% on 831 rows (bar 98.5%).","Full-attention projections, DeltaNet gates and norms trained in full plus a merged LoRA, with an answer-code and option-pointer head.","Training data undisclosed."],"scores":{"balanced_raw":75.55,"balanced_skill":65.26,"breadth_skill":66.84,"public_skill":67.75},"frozen_scores":{"balanced_raw":75.55,"balanced_skill":67.75,"breadth_skill":66.84},"categories":[{"id":"knowledge","raw":0.6091,"skill":0.5123,"combined":0.4367,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8296,"skill":0.7571,"combined":0.5779,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7773,"skill":0.6953,"combined":0.6944,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8478,"skill":0.8298,"combined":0.8049,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7296,"skill":0.5849,"combined":0.5763,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"vera-large","pub":67.75,"same_skills":61.56,"new_domains":55.61,"v":65.26,"lab":false,"jev":false,"rank_v":29,"rank_pub":21,"name":"vera-large","eq":{"S":61.21,"O":71.33},"rank_tie":6,"tied":true,"parts":{"public":0,"same_skills":36.73,"new_domains":28.53}},"new_in_edition":true},{"engine":"amber-hybrid-4b","name":"Amber Hybrid 4B","long_name":"Amber Hybrid 4B","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4610000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/MLukauskas/amber-hybrid-4b","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null,"pr":298,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7472,"conf":0.7031,"ece":0.0461,"brier":0.3455,"over95":0.0039,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0262,"conf":0.0684,"acc":0.069},{"lo":0.1,"w":0.0098,"conf":0.1414,"acc":0.1981},{"lo":0.2,"w":0.0286,"conf":0.2627,"acc":0.2535},{"lo":0.3,"w":0.0501,"conf":0.3565,"acc":0.358},{"lo":0.4,"w":0.0885,"conf":0.4543,"acc":0.5751},{"lo":0.5,"w":0.1485,"conf":0.5478,"acc":0.6367},{"lo":0.6,"w":0.1315,"conf":0.6492,"acc":0.7602},{"lo":0.7,"w":0.1056,"conf":0.7474,"acc":0.7841},{"lo":0.8,"w":0.0995,"conf":0.852,"acc":0.8731},{"lo":0.9,"w":0.3118,"conf":0.9749,"acc":0.9724}],"areas":{"knowledge":{"ece":0.0974,"acc":0.6145,"conf":0.5451},"language":{"ece":0.0226,"acc":0.8419,"conf":0.8216},"retrieval":{"ece":0.0243,"acc":0.7885,"conf":0.7872},"tools":{"ece":0.0416,"acc":0.9074,"conf":0.8913},"arts":{"ece":0.0998,"acc":0.7127,"conf":0.6255}}},"gaps":null,"latency":{"median":274.4,"p95":1535.3,"mean":481.7},"results":{"1":{"score":0.9734,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":342.8},"2":{"score":0.7108,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":2777},"3":{"score":0.8051,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1568.8},"4":{"score":0.9025,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":233.2},"5":{"score":0.9496,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":343.5},"6":{"score":0.7224,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":866.3},"9":{"score":0.9432,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":3848.4},"10":{"score":0.6376,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":299.3},"11":{"score":0.782,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":6396.5},"12":{"score":0.6144,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":176.6},"20":{"score":0.942,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":229.6},"21":{"score":0.6381,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":229.1},"22":{"score":0.8135,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1120.3},"23":{"score":0.6668,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":193.5},"24":{"score":0.7315,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":235.8},"25":{"score":0.3622,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":240.9},"26":{"score":0.9785,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":235.9},"27":{"score":0.9334,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":237.6},"28":{"score":0.8319,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":181.6},"29":{"score":0.9394,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":240.8},"30":{"score":0.5974,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0924,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":284.7},"32":{"score":0.5785,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":290.2},"33":{"score":0.4236,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":794.8},"36":{"score":0.4065,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":4079.9},"37":{"score":0.5508,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":184.8},"38":{"score":0.452,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2687},"39":{"score":0.9459,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":188.3},"40":{"score":0.4238,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":192.9},"41":{"score":0.7839,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":180.1},"42":{"score":0.7569,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":196},"43":{"score":0.5386,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":242.1},"44":{"score":0.8284,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":179.2},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":247.9},"50":{"score":0.5996,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":302.7},"56":{"score":0.8825,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":630.7},"57":{"score":0.4955,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":243.5},"58":{"score":0.7167,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":186.2},"59":{"score":0.7785,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":239.3},"61":{"score":0.8652,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":203.1},"62":{"score":0.8921,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":295.5},"64":{"score":0.7083,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":220.6}},"benchmarks":{"1":{"raw":0.9734,"skill":0.9641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.7108,"skill":0.666,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6922,"private_eq":0.7356},"3":{"raw":0.8051,"skill":0.8013,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6337,"private_eq":0.6437},"4":{"raw":0.9025,"skill":0.9012,"coverage":1,"random":0.0127,"tracks":[],"combined":0.781,"private_eq":0.7852},"5":{"raw":0.9496,"skill":0.9493,"coverage":1,"random":0.006,"tracks":[],"combined":0.686,"private_eq":0.6904},"6":{"raw":0.7224,"skill":0.3491,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.9432,"skill":0.9432,"coverage":1,"random":0,"tracks":[],"combined":0.6293,"private_eq":0.6325},"10":{"raw":0.6376,"skill":0.397,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.782,"skill":0.6848,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6144,"skill":0.4224,"coverage":1,"random":0.3324,"tracks":[],"combined":0.3646,"private_eq":0.58},"20":{"raw":0.9385,"skill":0.877,"coverage":1,"random":0.5,"tracks":[],"combined":0.8493,"private_eq":0.925},"21":{"raw":0.6381,"skill":0.2763,"coverage":1,"random":0.5,"tracks":[],"combined":0.4378,"private_eq":0.7216},"22":{"raw":0.816,"skill":0.8146,"coverage":1,"random":0.0078,"tracks":[],"combined":0.5725,"private_eq":0.5797},"23":{"raw":0.6484,"skill":0.2967,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7315,"skill":0.642,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3622,"skill":0.1497,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3339,"combined":0.1026},"26":{"raw":0.9785,"skill":0.9713,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9334,"skill":0.9112,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8319,"skill":0.6638,"coverage":1,"random":0.5,"tracks":[],"combined":0.6328,"private_eq":0.818},"29":{"raw":0.9394,"skill":0.9192,"coverage":1,"random":0.25,"tracks":[],"combined":0.9088,"private_eq":0.9316},"30":{"raw":0.5974,"skill":0.5176,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6846,"random":0.25},{"track":"10-choice","raw":0.5102,"random":0.1}],"combined":0.5526,"private_eq":0.6343},"31":{"raw":0.0924,"skill":0.0115,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0269,"private_eq":0.1159},"32":{"raw":0.5785,"skill":0.3299,"coverage":1,"random":0.371,"tracks":[],"combined":0,"private_eq":0.3746},"33":{"raw":0.4236,"skill":0.4159,"coverage":1,"random":0.0131,"tracks":[],"combined":0.3433,"private_eq":0.3583},"36":{"raw":0.4065,"skill":0.3286,"coverage":1,"random":0.116,"tracks":[],"combined":0.2941,"private_eq":0.3822},"37":{"raw":0.5508,"skill":0.4366,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5949,"private_eq":0.6799},"38":{"raw":0.452,"skill":0.4345,"coverage":1,"random":0.031,"tracks":[],"combined":0.5242,"private_eq":0.5432},"39":{"raw":0.9459,"skill":0.9204,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4555,"private_eq":0.6333},"40":{"raw":0.4238,"skill":0.2587,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2878,"headline":false},{"track":"A · English","score":0.4238,"headline":true},{"track":"C · Arabic pairs","score":0.92,"headline":false},{"track":"C · English pairs","score":0.92,"headline":false}],"combined":0.211,"private_eq":0.3929},"41":{"raw":0.7839,"skill":0.6759,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3158,"private_eq":0.5484},"42":{"raw":0.7569,"skill":0.527,"coverage":1,"random":0.486,"tracks":[],"combined":0.6614,"private_eq":0.8274},"43":{"raw":0.5386,"skill":0.2679,"coverage":1,"random":0.3697,"tracks":[],"combined":0.2569,"private_eq":0.5363},"44":{"raw":0.8284,"skill":0.6568,"coverage":1,"random":0.5,"tracks":[],"combined":0.6154,"private_eq":0.8094},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5996,"skill":0.4189,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.8825,"skill":0.765,"coverage":1,"random":0.5,"tracks":[],"combined":0.5857,"private_eq":0.7947},"57":{"raw":0.4955,"skill":0.4326,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3281,"combined":0.2366},"58":{"raw":0.7167,"skill":0.5894,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4044,"combined":0.1278},"59":{"raw":0.7785,"skill":0.5407,"coverage":1,"random":0.5177,"tracks":[],"combined":0.2397,"private_eq":0.637},"61":{"raw":0.8652,"skill":0.7304,"coverage":1,"random":0.5,"tracks":[],"combined":0.5586,"private_eq":0.7813},"62":{"raw":0.8921,"skill":0.8561,"coverage":1,"random":0.25,"tracks":[],"combined":0.5287,"private_eq":0.6498},"64":{"raw":0.7083,"skill":0.6354,"coverage":1,"random":0.2,"tracks":[],"combined":0.681,"private_eq":0.7469}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #298).","Submitted public 0.3 run adopted (kit rescore 59.87, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.46% on 831 rows (bar 98.5%).","Ensemble: the unchanged Amber 0.4B encoder averaged with a Qwen3.5-4B LoRA decoder.","Amber 0.4B stays listed as its own entry."],"scores":{"balanced_raw":70.05,"balanced_skill":42.74,"breadth_skill":57.86,"public_skill":59.87},"frozen_scores":{"balanced_raw":70.05,"balanced_skill":59.87,"breadth_skill":57.86},"categories":[{"id":"knowledge","raw":0.5069,"skill":0.3669,"combined":0.288,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7221,"skill":0.6013,"combined":0.4685,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7589,"skill":0.6889,"combined":0.5837,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8667,"skill":0.8488,"combined":0.6938,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7248,"skill":0.5531,"combined":0.5473,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"amber-hybrid-4b","pub":59.87,"same_skills":50.42,"new_domains":23.71,"v":42.74,"lab":false,"jev":false,"rank_v":123,"rank_pub":60,"name":"Amber Hybrid 4B","eq":{"S":49.95,"O":31.93},"rank_tie":27,"tied":true,"parts":{"public":0,"same_skills":29.97,"new_domains":12.77}},"new_in_edition":true},{"engine":"jev-style-2b-v4","name":"Jev-Style 2B v4","long_name":"Jev-Style 2B v4","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2213241664,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/chaoliangUNSW/Jev-Style-2B-Decision-v4","code_url":null,"base_model":"Qwen/Qwen3.5-2B","base_url":"https://huggingface.co/Qwen/Qwen3.5-2B","weights_repo":null,"served_checkpoint":null,"pr":319,"replaces":"jev-style-2b-v3","new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6422,"conf":0.7518,"ece":0.1099,"brier":0.4845,"over95":0.0313,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0318,"conf":0.0553,"acc":0.0088},{"lo":0.1,"w":0.0079,"conf":0.1474,"acc":0.1634},{"lo":0.2,"w":0.0112,"conf":0.2568,"acc":0.1481},{"lo":0.3,"w":0.0399,"conf":0.3532,"acc":0.2968},{"lo":0.4,"w":0.057,"conf":0.4554,"acc":0.3293},{"lo":0.5,"w":0.1136,"conf":0.5501,"acc":0.4432},{"lo":0.6,"w":0.1072,"conf":0.6508,"acc":0.5671},{"lo":0.7,"w":0.1073,"conf":0.7495,"acc":0.6371},{"lo":0.8,"w":0.1331,"conf":0.8534,"acc":0.6836},{"lo":0.9,"w":0.391,"conf":0.9709,"acc":0.8638}],"areas":{"knowledge":{"ece":0.1415,"acc":0.56,"conf":0.7015},"language":{"ece":0.0876,"acc":0.7256,"conf":0.8132},"retrieval":{"ece":0.11,"acc":0.7434,"conf":0.8533},"tools":{"ece":0.1039,"acc":0.8144,"conf":0.9123},"arts":{"ece":0.0939,"acc":0.4719,"conf":0.5642}}},"gaps":null,"latency":{"median":39.8,"p95":92,"mean":73.7},"results":{"1":{"score":0.9227,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":264.5},"2":{"score":0.6261,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3336.5},"3":{"score":0.6654,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1487.1},"4":{"score":0.8768,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":291.2},"5":{"score":0.9421,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":430.1},"6":{"score":0.7717,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":603.7},"9":{"score":0.0682,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":522.1},"10":{"score":0.4659,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":167.5},"11":{"score":0.836,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":511.7},"12":{"score":0.5007,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":93.8},"20":{"score":0.7622,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":84.2},"21":{"score":0.6138,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":204.9},"22":{"score":0.018,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":703.6},"23":{"score":0.633,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":130.1},"24":{"score":0.6076,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":80.5},"25":{"score":0.352,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":83.2},"26":{"score":0.9428,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":75.6},"27":{"score":0.8575,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":75.5},"28":{"score":0.768,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":74.3},"29":{"score":0.8163,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":90.9},"30":{"score":0.3006,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.153,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":169.6},"32":{"score":0.5492,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":177.3},"33":{"score":0.2224,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":360.8},"36":{"score":0.4251,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1737.4},"37":{"score":0.556,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":187.4},"38":{"score":0.0932,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":464.8},"39":{"score":0.7975,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":163.2},"40":{"score":0.4697,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":99.7},"41":{"score":0.4358,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":69.3},"42":{"score":0.6912,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":71.7},"43":{"score":0.293,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":69.8},"44":{"score":0.7212,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":69.6},"45":{"score":0.1257,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":70.5},"50":{"score":0.4069,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":73.5},"56":{"score":0.778,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":152.4},"57":{"score":0.3328,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":70.8},"58":{"score":0.6301,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":70.7},"59":{"score":0.7024,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":93.7},"61":{"score":0.8585,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":77.3},"62":{"score":0.8749,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":88},"64":{"score":0.6458,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":75.8}},"benchmarks":{"1":{"raw":0.9227,"skill":0.8956,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9261,"private_eq":0.9451},"2":{"raw":0.6261,"skill":0.5682,"coverage":1,"random":0.1341,"tracks":[],"combined":0.676,"private_eq":0.7218},"3":{"raw":0.6654,"skill":0.659,"coverage":1,"random":0.0189,"tracks":[],"combined":0.6704,"private_eq":0.6793},"4":{"raw":0.8768,"skill":0.8752,"coverage":1,"random":0.0127,"tracks":[],"combined":0.7913,"private_eq":0.7953},"5":{"raw":0.9421,"skill":0.9418,"coverage":1,"random":0.006,"tracks":[],"combined":0.6592,"private_eq":0.6641},"6":{"raw":0.7717,"skill":0.4647,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.0682,"skill":0.0682,"coverage":1,"random":0,"tracks":[],"combined":0.0401,"private_eq":0.05},"10":{"raw":0.4659,"skill":0.1113,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.836,"skill":0.7629,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5007,"skill":0.2521,"coverage":1,"random":0.3324,"tracks":[],"combined":0.4489,"private_eq":0.6356},"20":{"raw":0.7655,"skill":0.5309,"coverage":1,"random":0.5,"tracks":[],"combined":0.6324,"private_eq":0.8178},"21":{"raw":0.6138,"skill":0.2275,"coverage":1,"random":0.5,"tracks":[],"combined":0.3269,"private_eq":0.6668},"22":{"raw":0.0208,"skill":0.0132,"coverage":1,"random":0.0078,"tracks":[],"combined":0.0074,"private_eq":0.0254},"23":{"raw":0.6363,"skill":0.2727,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6076,"skill":0.4768,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.352,"skill":0.1361,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.308,"combined":0.0677},"26":{"raw":0.9428,"skill":0.9237,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8575,"skill":0.8099,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.768,"skill":0.536,"coverage":1,"random":0.5,"tracks":[],"combined":0.3603,"private_eq":0.6833},"29":{"raw":0.8163,"skill":0.7551,"coverage":1,"random":0.25,"tracks":[],"combined":0.5723,"private_eq":0.6821},"30":{"raw":0.3006,"skill":0.1569,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4056,"random":0.25},{"track":"10-choice","raw":0.1956,"random":0.1}],"combined":0.2688,"private_eq":0.4028},"31":{"raw":0.153,"skill":0.0775,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0778,"private_eq":0.1621},"32":{"raw":0.5492,"skill":0.2833,"coverage":1,"random":0.371,"tracks":[],"combined":0.0262,"private_eq":0.3938},"33":{"raw":0.2224,"skill":0.2121,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1738,"private_eq":0.1929},"36":{"raw":0.4251,"skill":0.3496,"coverage":1,"random":0.116,"tracks":[],"combined":0.2682,"private_eq":0.3596},"37":{"raw":0.556,"skill":0.4431,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4003,"private_eq":0.5265},"38":{"raw":0.0932,"skill":0.0642,"coverage":1,"random":0.031,"tracks":[],"combined":0.1096,"private_eq":0.1461},"39":{"raw":0.7975,"skill":0.7022,"coverage":1,"random":0.3201,"tracks":[],"combined":0.4952,"private_eq":0.66},"40":{"raw":0.4697,"skill":0.3178,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.1813,"headline":false},{"track":"A · English","score":0.4697,"headline":true},{"track":"C · Arabic pairs","score":0.725,"headline":false},{"track":"C · English pairs","score":0.895,"headline":false}],"combined":0.3353,"private_eq":0.4884},"41":{"raw":0.4358,"skill":0.1537,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3759,"private_eq":0.588},"42":{"raw":0.6912,"skill":0.3992,"coverage":1,"random":0.486,"tracks":[],"combined":0.6123,"private_eq":0.8025},"43":{"raw":0.293,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"combined":0.3738,"private_eq":0.6092},"44":{"raw":0.7212,"skill":0.4424,"coverage":1,"random":0.5,"tracks":[],"combined":0.562,"private_eq":0.783},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4069,"skill":0.1392,"coverage":1,"random":0.311,"tracks":[],"combined":0.1887,"private_eq":0.4467},"56":{"raw":0.778,"skill":0.556,"coverage":1,"random":0.5,"tracks":[],"combined":0.3433,"private_eq":0.6749},"57":{"raw":0.3328,"skill":0.2496,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4542,"combined":0.3801},"58":{"raw":0.6301,"skill":0.4638,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3632,"combined":0.0673},"59":{"raw":0.7024,"skill":0.383,"coverage":1,"random":0.5177,"tracks":[],"combined":0.1428,"private_eq":0.5908},"61":{"raw":0.8585,"skill":0.717,"coverage":1,"random":0.5,"tracks":[],"combined":0.5982,"private_eq":0.8009},"62":{"raw":0.8749,"skill":0.8332,"coverage":1,"random":0.25,"tracks":[],"combined":0.5218,"private_eq":0.6447},"64":{"raw":0.6458,"skill":0.5573,"coverage":1,"random":0.2,"tracks":[],"combined":0.5744,"private_eq":0.6626}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #319).","Replaces the earlier board entry jev-style-2b-v3 (newer version of the same model).","Submitted public 0.3 run adopted (kit rescore 44.37, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.51% on 831 rows (bar 98.5%).","Training uses soft labels distilled from TypeSafe Jev 1.14 and train splits of index benchmarks (author-declared)."],"scores":{"balanced_raw":57.68,"balanced_skill":34.53,"breadth_skill":41.75,"public_skill":44.37},"frozen_scores":{"balanced_raw":57.68,"balanced_skill":44.37,"breadth_skill":41.75},"categories":[{"id":"knowledge","raw":0.4075,"skill":0.2362,"combined":0.2345,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5964,"skill":0.4246,"combined":0.3924,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7402,"skill":0.6539,"combined":0.5158,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6435,"skill":0.6176,"combined":0.584,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5149,"skill":0.2901,"combined":0.346,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jev-style-2b-v4","pub":44.37,"same_skills":41.87,"new_domains":17.59,"v":34.53,"lab":false,"jev":false,"rank_v":149,"rank_pub":137,"name":"Jev-Style 2B v4","eq":{"S":41.3,"O":24.37},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":24.78,"new_domains":9.75}},"new_in_edition":true},{"engine":"knowline-9b-gen1","name":"KnowLine 9B Gen1","long_name":"KnowLine 9B Gen1","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/PelaAI/KnowLine-9B-Gen1","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"org":{"id":"PelaAI","name":"PelaAI","logo":"data/orgs/PelaAI.png"},"pr":289,"new":true},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7796,"conf":0.8069,"ece":0.0283,"brier":0.2867,"over95":0.0106,"n":34411,"benchmarks":31,"rel":[{"lo":0,"w":0.0067,"conf":0.0829,"acc":0.0963},{"lo":0.1,"w":0.0167,"conf":0.145,"acc":0.15},{"lo":0.2,"w":0.0151,"conf":0.2531,"acc":0.2779},{"lo":0.3,"w":0.0332,"conf":0.3565,"acc":0.2363},{"lo":0.4,"w":0.0518,"conf":0.4513,"acc":0.3895},{"lo":0.5,"w":0.0897,"conf":0.5501,"acc":0.5016},{"lo":0.6,"w":0.0824,"conf":0.6502,"acc":0.621},{"lo":0.7,"w":0.0753,"conf":0.7519,"acc":0.7292},{"lo":0.8,"w":0.0897,"conf":0.8518,"acc":0.8158},{"lo":0.9,"w":0.5393,"conf":0.9807,"acc":0.9642}],"areas":{"knowledge":{"ece":0.0629,"acc":0.6748,"conf":0.7363},"language":{"ece":0.0166,"acc":0.8764,"conf":0.8854},"retrieval":{"ece":0.0378,"acc":0.7446,"conf":0.7571},"tools":{"ece":0.0308,"acc":0.9532,"conf":0.9802},"arts":{"ece":0.0212,"acc":0.7393,"conf":0.7427}}},"gaps":null,"latency":{"median":128.1,"p95":747.1,"mean":222.6},"results":{"1":{"score":0.9711,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":null},"2":{"score":0.6833,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":null},"3":{"score":0.8957,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":null},"4":{"score":0.8286,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":null},"5":{"score":0.9021,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"6":{"score":0.8001,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":null},"9":{"score":0.75,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":null},"10":{"score":0.6524,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":null},"11":{"score":0.804,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":null},"12":{"score":0.7365,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":null},"20":{"score":0.956,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"21":{"score":0.6514,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":null},"22":{"score":0.812,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"23":{"score":0.6878,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"24":{"score":0.84,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":null},"25":{"score":0.4847,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":null},"26":{"score":0.9891,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":null},"27":{"score":0.9616,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":null},"28":{"score":0.8919,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":null},"29":{"score":0.9533,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":null},"30":{"score":0.7426,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2336,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"32":{"score":0.641,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":null},"33":{"score":0.3552,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":null},"36":{"score":0.4936,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":null},"37":{"score":0.6073,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"38":{"score":0.4962,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":null},"39":{"score":0.9078,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":null},"40":{"score":0.6762,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":null},"41":{"score":0.7909,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":null},"42":{"score":0.8403,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":null},"43":{"score":0.6965,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":null},"44":{"score":0.8376,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":null},"45":{"score":0.1357,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":null},"50":{"score":0.608,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":null},"56":{"score":0.9635,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":null},"57":{"score":0.6513,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":null},"58":{"score":0.7937,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":null},"59":{"score":0.8186,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":null},"61":{"score":0.8632,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":null},"62":{"score":0.8927,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":null},"64":{"score":0.7557,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":null}},"benchmarks":{"1":{"raw":0.9711,"skill":0.961,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9465,"private_eq":0.9601},"2":{"raw":0.6833,"skill":0.6343,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6821,"private_eq":0.727},"3":{"raw":0.8957,"skill":0.8937,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8645,"private_eq":0.8676},"4":{"raw":0.8286,"skill":0.8264,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8439,"private_eq":0.8466},"5":{"raw":0.9021,"skill":0.9015,"coverage":1,"random":0.006,"tracks":[],"combined":0.7524,"private_eq":0.7557},"6":{"raw":0.8001,"skill":0.5313,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.75,"skill":0.75,"coverage":1,"random":0,"tracks":[],"combined":0.6723,"private_eq":0.675},"10":{"raw":0.6524,"skill":0.4216,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.804,"skill":0.7165,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7365,"skill":0.6053,"coverage":1,"random":0.3324,"tracks":[],"combined":0.5266,"private_eq":0.6869},"20":{"raw":0.9574,"skill":0.9148,"coverage":1,"random":0.5,"tracks":[],"combined":0.9361,"private_eq":0.9679},"21":{"raw":0.6514,"skill":0.3029,"coverage":1,"random":0.5,"tracks":[],"combined":0.4176,"private_eq":0.7116},"22":{"raw":0.8173,"skill":0.8159,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6457,"private_eq":0.6515},"23":{"raw":0.713,"skill":0.4261,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.84,"skill":0.7867,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4847,"skill":0.3129,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4378,"combined":0.2428},"26":{"raw":0.9891,"skill":0.9855,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9616,"skill":0.9488,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8919,"skill":0.7838,"coverage":1,"random":0.5,"tracks":[],"combined":0.8196,"private_eq":0.9103},"29":{"raw":0.9533,"skill":0.9377,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.7426,"skill":0.6919,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8006,"random":0.25},{"track":"10-choice","raw":0.6846,"random":0.1}],"combined":0.6706,"private_eq":0.7305},"31":{"raw":0.2336,"skill":0.1653,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0799,"private_eq":0.164},"32":{"raw":0.641,"skill":0.4293,"coverage":1,"random":0.371,"tracks":[],"combined":0.3899,"private_eq":0.62},"33":{"raw":0.3552,"skill":0.3466,"coverage":1,"random":0.0131,"tracks":[],"combined":0.2921,"private_eq":0.3083},"36":{"raw":0.4936,"skill":0.4272,"coverage":1,"random":0.116,"tracks":[],"combined":0.3796,"private_eq":0.4569},"37":{"raw":0.6073,"skill":0.5075,"coverage":1,"random":0.2027,"tracks":[],"combined":0.5078,"private_eq":0.6112},"38":{"raw":0.4962,"skill":0.4801,"coverage":1,"random":0.031,"tracks":[],"combined":0.6907,"private_eq":0.7027},"39":{"raw":0.9078,"skill":0.8644,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5647,"private_eq":0.7067},"40":{"raw":0.6762,"skill":0.5834,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3588,"headline":false},{"track":"A · English","score":0.6762,"headline":true},{"track":"C · Arabic pairs","score":0.88,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"combined":0.5219,"private_eq":0.6318},"41":{"raw":0.7909,"skill":0.6863,"coverage":1,"random":0.3333,"tracks":[],"combined":0.4909,"private_eq":0.6638},"42":{"raw":0.8403,"skill":0.6893,"coverage":1,"random":0.486,"tracks":[],"combined":0.8833,"private_eq":0.9402},"43":{"raw":0.6965,"skill":0.5185,"coverage":1,"random":0.3697,"tracks":[],"combined":0.5035,"private_eq":0.69},"44":{"raw":0.8376,"skill":0.6752,"coverage":1,"random":0.5,"tracks":[],"combined":0.6423,"private_eq":0.8227},"45":{"raw":0.1357,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.608,"skill":0.4311,"coverage":1,"random":0.311,"tracks":[],"combined":0.1959,"private_eq":0.4516},"56":{"raw":0.9635,"skill":0.927,"coverage":1,"random":0.5,"tracks":[],"combined":0.6377,"private_eq":0.8204},"57":{"raw":0.6513,"skill":0.6078,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5137,"combined":0.4478},"58":{"raw":0.7937,"skill":0.701,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4767,"combined":0.2338},"59":{"raw":0.8186,"skill":0.6239,"coverage":1,"random":0.5177,"tracks":[],"combined":0.3691,"private_eq":0.6987},"61":{"raw":0.8632,"skill":0.7264,"coverage":1,"random":0.5,"tracks":[],"combined":0.6322,"private_eq":0.8177},"62":{"raw":0.8927,"skill":0.8569,"coverage":1,"random":0.25,"tracks":[],"combined":0.6315,"private_eq":0.726},"64":{"raw":0.7557,"skill":0.6946,"coverage":1,"random":0.2,"tracks":[],"combined":0.681,"private_eq":0.7469}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #289).","Submitted public 0.3 run adopted (kit rescore 65.67, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.70% on 831 rows (bar 98.5%).","Read twice: short text-only requests are written a second time in the same prompt before the answer is read; one prefill per question.","Training includes train splits of Decision Index benchmarks in request format (author-declared).","Per-benchmark times are not shown: the board run sent requests in batches, so its timings are batch wall-clock. The latency shown is the single-request measurement."],"scores":{"balanced_raw":73.88,"balanced_skill":58.44,"breadth_skill":64.63,"public_skill":65.67},"frozen_scores":{"balanced_raw":73.88,"balanced_skill":65.67,"breadth_skill":64.63},"categories":[{"id":"knowledge","raw":0.5879,"skill":0.4761,"combined":0.4325,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7832,"skill":0.691,"combined":0.6374,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7732,"skill":0.7192,"combined":0.6286,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8456,"skill":0.8272,"combined":0.7702,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.7505,"skill":0.5976,"combined":0.5752,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"knowline-9b-gen1","pub":65.67,"same_skills":60.82,"new_domains":42.7,"v":58.44,"lab":false,"jev":false,"rank_v":63,"rank_pub":29,"name":"KnowLine 9B Gen1","eq":{"S":60.48,"O":55.38},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":36.29,"new_domains":22.15}},"new_in_edition":true},{"engine":"seb-9b-v2","name":"Seb-9B v2","long_name":"Seb-9B v2","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/ironbcc/Seb-9B-v2","code_url":null,"base_model":"Qwen/Qwen3.5-9B","base_url":"https://huggingface.co/Qwen/Qwen3.5-9B","weights_repo":null,"served_checkpoint":null,"pr":294,"new":true},"completed":110201,"counts":{"ok":110200,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.7383,"conf":0.8487,"ece":0.1107,"brier":0.3721,"over95":0.0265,"n":34410,"benchmarks":31,"rel":[{"lo":0,"w":0.0019,"conf":0.0882,"acc":0.0423},{"lo":0.1,"w":0.0145,"conf":0.1578,"acc":0.1333},{"lo":0.2,"w":0.0116,"conf":0.2457,"acc":0.2282},{"lo":0.3,"w":0.0167,"conf":0.3633,"acc":0.3713},{"lo":0.4,"w":0.0266,"conf":0.4499,"acc":0.2498},{"lo":0.5,"w":0.0658,"conf":0.5488,"acc":0.4534},{"lo":0.6,"w":0.0742,"conf":0.6469,"acc":0.4393},{"lo":0.7,"w":0.0802,"conf":0.7538,"acc":0.53},{"lo":0.8,"w":0.1112,"conf":0.854,"acc":0.6193},{"lo":0.9,"w":0.5973,"conf":0.9807,"acc":0.9157}],"areas":{"knowledge":{"ece":0.1423,"acc":0.6332,"conf":0.7755},"language":{"ece":0.047,"acc":0.8474,"conf":0.8943},"retrieval":{"ece":0.1955,"acc":0.6801,"conf":0.8734},"tools":{"ece":0.0293,"acc":0.9394,"conf":0.9687},"arts":{"ece":0.1446,"acc":0.6779,"conf":0.8211}}},"gaps":null,"latency":{"median":89.4,"p95":748.8,"mean":274},"results":{"1":{"score":0.9734,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":509.9},"2":{"score":0.6731,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":4117},"3":{"score":0.8543,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1567},"4":{"score":0.9244,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":150},"5":{"score":0.9311,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":318.4},"6":{"score":0.7997,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1937},"9":{"score":0.8068,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":3822.5},"10":{"score":0.265,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":554.1},"11":{"score":0.8481,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":7539.5},"12":{"score":0.6376,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":52.1},"20":{"score":0.9154,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":95.8},"21":{"score":0.6187,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":91.5},"22":{"score":0.798,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1825.6},"23":{"score":0.6474,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":153.8},"24":{"score":0.7697,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":117.4},"25":{"score":0.4235,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":132.4},"26":{"score":0.9844,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":96.6},"27":{"score":0.9573,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":103},"28":{"score":0.884,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":91.8},"29":{"score":0.9518,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":128.8},"30":{"score":0.5516,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.222,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":271.5},"32":{"score":0.6782,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":251.1},"33":{"score":0.3006,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1100.2},"36":{"score":0.4433,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5312.2},"37":{"score":0.5601,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":166.8},"38":{"score":0.2248,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":5515.6},"39":{"score":0.8854,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":180.2},"40":{"score":0.5726,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":125.9},"41":{"score":0.7413,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":103},"42":{"score":0.822,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":151.2},"43":{"score":0.5561,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":98.5},"44":{"score":0.8092,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":93.7},"45":{"score":0.084,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":136.2},"50":{"score":0.4529,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":222.8},"56":{"score":0.6545,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":450.9},"57":{"score":0.5318,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":96.3},"58":{"score":0.7175,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":76.1},"59":{"score":0.828,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":124.4},"61":{"score":0.8808,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":81.5},"62":{"score":0.911,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":160.2},"64":{"score":0.6553,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":97.9}},"benchmarks":{"1":{"raw":0.9734,"skill":0.9641,"coverage":1,"random":0.2592,"tracks":[],"combined":0.9397,"private_eq":0.9551},"2":{"raw":0.6731,"skill":0.6225,"coverage":1,"random":0.1341,"tracks":[],"combined":0.6846,"private_eq":0.7291},"3":{"raw":0.8543,"skill":0.8515,"coverage":1,"random":0.0189,"tracks":[],"combined":0.8751,"private_eq":0.8779},"4":{"raw":0.9244,"skill":0.9234,"coverage":1,"random":0.0127,"tracks":[],"combined":0.8298,"private_eq":0.8329},"5":{"raw":0.9311,"skill":0.9307,"coverage":1,"random":0.006,"tracks":[],"combined":0.697,"private_eq":0.7012},"6":{"raw":0.7997,"skill":0.5304,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8068,"skill":0.8068,"coverage":1,"random":0,"tracks":[],"combined":0.5863,"private_eq":0.59},"10":{"raw":0.265,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8481,"skill":0.7803,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6376,"skill":0.4572,"coverage":1,"random":0.3324,"tracks":[],"combined":0.566,"private_eq":0.7129},"20":{"raw":0.9145,"skill":0.829,"coverage":1,"random":0.5,"tracks":[],"combined":0.9432,"private_eq":0.9714},"21":{"raw":0.6187,"skill":0.2374,"coverage":1,"random":0.5,"tracks":[],"combined":0.2565,"private_eq":0.632},"22":{"raw":0.7935,"skill":0.7919,"coverage":1,"random":0.0078,"tracks":[],"combined":0.6117,"private_eq":0.6181},"23":{"raw":0.6509,"skill":0.3018,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7697,"skill":0.6929,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4235,"skill":0.2313,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3423,"combined":0.114},"26":{"raw":0.9844,"skill":0.9792,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9573,"skill":0.9431,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.884,"skill":0.768,"coverage":1,"random":0.5,"tracks":[],"combined":0.5974,"private_eq":0.8005},"29":{"raw":0.9518,"skill":0.9357,"coverage":1,"random":0.25,"tracks":[],"combined":1.001,"private_eq":1},"30":{"raw":0.5516,"skill":0.461,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6338,"random":0.25},{"track":"10-choice","raw":0.4693,"random":0.1}],"combined":0.5694,"private_eq":0.648},"31":{"raw":0.222,"skill":0.1526,"coverage":1,"random":0.0819,"tracks":[],"combined":0.0831,"private_eq":0.1669},"32":{"raw":0.6782,"skill":0.4884,"coverage":1,"random":0.371,"tracks":[],"combined":0.3201,"private_eq":0.5766},"33":{"raw":0.3006,"skill":0.2913,"coverage":1,"random":0.0131,"tracks":[],"combined":0.1789,"private_eq":0.1979},"36":{"raw":0.4433,"skill":0.3702,"coverage":1,"random":0.116,"tracks":[],"combined":0.3562,"private_eq":0.4365},"37":{"raw":0.5601,"skill":0.4483,"coverage":1,"random":0.2027,"tracks":[],"combined":0.4507,"private_eq":0.5662},"38":{"raw":0.2248,"skill":0.2,"coverage":1,"random":0.031,"tracks":[],"combined":0.3313,"private_eq":0.3584},"39":{"raw":0.8854,"skill":0.8314,"coverage":1,"random":0.3201,"tracks":[],"combined":0.5764,"private_eq":0.7146},"40":{"raw":0.5726,"skill":0.4502,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2979,"headline":false},{"track":"A · English","score":0.5726,"headline":true},{"track":"C · Arabic pairs","score":0.66,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"combined":0.4524,"private_eq":0.5784},"41":{"raw":0.7413,"skill":0.6119,"coverage":1,"random":0.3333,"tracks":[],"combined":0.3395,"private_eq":0.564},"42":{"raw":0.822,"skill":0.6537,"coverage":1,"random":0.486,"tracks":[],"combined":0.755,"private_eq":0.875},"43":{"raw":0.5561,"skill":0.2958,"coverage":1,"random":0.3697,"tracks":[],"combined":0.4366,"private_eq":0.6483},"44":{"raw":0.8092,"skill":0.6184,"coverage":1,"random":0.5,"tracks":[],"combined":0.5887,"private_eq":0.7962},"45":{"raw":0.0838,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.4529,"skill":0.2059,"coverage":1,"random":0.311,"tracks":[],"combined":0.2031,"private_eq":0.4565},"56":{"raw":0.6545,"skill":0.309,"coverage":1,"random":0.5,"tracks":[],"combined":0.0751,"private_eq":0.5423},"57":{"raw":0.5318,"skill":0.4734,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4258,"combined":0.3478},"58":{"raw":0.7175,"skill":0.5905,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4319,"combined":0.1681},"59":{"raw":0.828,"skill":0.6434,"coverage":1,"random":0.5177,"tracks":[],"combined":0.233,"private_eq":0.6338},"61":{"raw":0.8808,"skill":0.7616,"coverage":1,"random":0.5,"tracks":[],"combined":0.6039,"private_eq":0.8037},"62":{"raw":0.911,"skill":0.8813,"coverage":1,"random":0.25,"tracks":[],"combined":0.6108,"private_eq":0.7107},"64":{"raw":0.6553,"skill":0.5691,"coverage":1,"random":0.2,"tracks":[],"combined":0.5647,"private_eq":0.6549}},"coverage":1,"pending":0,"notes":["New in the 2026-10-27 PR batch (PR #294).","Submitted public 0.3 run adopted (kit rescore 59.16, equal to the claim). Private tests run by the lab on one RTX PRO 6000 with the author's serving; decision agreement with the submitted rows 99.46% on 831 rows (bar 98.5%).","Questions over its declared 32,768-token capacity count as unsupported.","Training uses public Decision Index skill data (author-declared)."],"scores":{"balanced_raw":69.37,"balanced_skill":46.25,"breadth_skill":57.6,"public_skill":59.16},"frozen_scores":{"balanced_raw":69.37,"balanced_skill":59.16,"breadth_skill":57.6},"categories":[{"id":"knowledge","raw":0.5239,"skill":0.3986,"combined":0.3379,"coverage":0.9998,"pending":0,"n":11},{"id":"language","raw":0.7265,"skill":0.6215,"combined":0.5438,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7354,"skill":0.6346,"combined":0.5135,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8489,"skill":0.8314,"combined":0.7518,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.681,"skill":0.4892,"combined":0.5158,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"seb-9b-v2","pub":59.16,"same_skills":52.86,"new_domains":27.83,"v":46.25,"lab":false,"jev":false,"rank_v":105,"rank_pub":68,"name":"Seb-9B v2","eq":{"S":52.41,"O":37.01},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":31.45,"new_domains":14.8}},"new_in_edition":true},{"engine":"jpt-35b-a3b","name":"JPT-35B-A3B","long_name":"JPT-35B-A3B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":35950000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/kirp/jpt-35b-a3b","code_url":null,"base_model":"Qwen/Qwen3.5-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.5-35B-A3B","weights_repo":null,"served_checkpoint":null},"completed":100833,"counts":{"ok":100833,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7194,"conf":0.7686,"ece":0.0491,"brier":0.3696,"over95":0.0092,"n":34661,"benchmarks":31,"rel":[{"lo":0,"w":0.0146,"conf":0.0699,"acc":0.0598},{"lo":0.1,"w":0.0261,"conf":0.1454,"acc":0.1238},{"lo":0.2,"w":0.0224,"conf":0.2521,"acc":0.163},{"lo":0.3,"w":0.0309,"conf":0.3558,"acc":0.2815},{"lo":0.4,"w":0.0404,"conf":0.4514,"acc":0.2966},{"lo":0.5,"w":0.0895,"conf":0.5475,"acc":0.4818},{"lo":0.6,"w":0.0997,"conf":0.6507,"acc":0.5955},{"lo":0.7,"w":0.1034,"conf":0.7526,"acc":0.6581},{"lo":0.8,"w":0.1472,"conf":0.8565,"acc":0.7925},{"lo":0.9,"w":0.4258,"conf":0.9655,"acc":0.9483}],"areas":{"knowledge":{"ece":0.0891,"acc":0.6429,"conf":0.7315},"language":{"ece":0.009,"acc":0.8423,"conf":0.8453},"retrieval":{"ece":0.0694,"acc":0.7039,"conf":0.7707},"tools":{"ece":0.0328,"acc":0.913,"conf":0.921},"arts":{"ece":0.0638,"acc":0.5474,"conf":0.6113}}},"gaps":null,"latency":{"median":250.2,"p95":794.6,"mean":339.7},"results":{"1":{"score":0.9599,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":413.7},"2":{"score":0.6337,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":769.7},"3":{"score":0.8563,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":436.4},"4":{"score":0.7728,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":236.9},"5":{"score":0.8505,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":246.9},"6":{"score":0.8,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":547.8},"9":{"score":0.3409,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":586.3},"10":{"score":0.5982,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":242},"11":{"score":0.8173,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":663},"12":{"score":0.713,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":219.2},"20":{"score":0.895,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":214.4},"21":{"score":0.6172,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":214.1},"22":{"score":0.1225,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":425.2},"23":{"score":0.6232,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":224.1},"24":{"score":0.8461,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":218},"25":{"score":0.4796,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":213.3},"26":{"score":0.9895,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":213.2},"27":{"score":0.9625,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":213},"28":{"score":0.8303,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":223.9},"29":{"score":0.9182,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":218.5},"30":{"score":0.4079,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1684,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":233},"32":{"score":0.5878,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":233.4},"33":{"score":0.2242,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":521.9},"36":{"score":0.4467,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":953.6},"37":{"score":0.5306,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":226.3},"38":{"score":0.2107,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":824.5},"39":{"score":0.9338,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":389.5},"40":{"score":0.6103,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":288.4},"41":{"score":0.6826,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":218.5},"42":{"score":0.8241,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":228.3},"43":{"score":0.6702,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":214.4},"44":{"score":0.694,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":214.3},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":226.5},"50":{"score":0.4374,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":231},"56":{"score":0.7555,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":491.1},"57":{"score":0.6165,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":218.9},"58":{"score":0.7271,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":219.7},"59":{"score":0.757,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":230},"61":{"score":0.6767,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":225.7},"62":{"score":0.7453,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":234.3},"64":{"score":0.6913,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":221.1}},"benchmarks":{"1":{"raw":0.9599,"skill":0.9458,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.8803,"combined":0.8376},"2":{"raw":0.6337,"skill":0.577,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7225,"combined":0.6769},"3":{"raw":0.8563,"skill":0.8535,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.7964,"combined":0.7911},"4":{"raw":0.7728,"skill":0.7699,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8235,"combined":0.8202},"5":{"raw":0.8505,"skill":0.8496,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.6466,"combined":0.6414},"6":{"raw":0.8,"skill":0.5311,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3409,"skill":0.3409,"coverage":1,"random":0,"tracks":[],"private_eq":0.3625,"combined":0.3562},"10":{"raw":0.5982,"skill":0.3314,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8173,"skill":0.7358,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.713,"skill":0.5701,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6338,"combined":0.4462},"20":{"raw":0.8976,"skill":0.7952,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.9071,"combined":0.8131},"21":{"raw":0.6172,"skill":0.2344,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6668,"combined":0.3269},"22":{"raw":0.1208,"skill":0.1139,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.1151,"combined":0.0989},"23":{"raw":0.626,"skill":0.2521,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8461,"skill":0.7948,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4796,"skill":0.3061,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4649,"combined":0.2793},"26":{"raw":0.9895,"skill":0.986,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9625,"skill":0.95,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8303,"skill":0.6606,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7781,"combined":0.5521},"29":{"raw":0.9182,"skill":0.8909,"coverage":1,"random":0.25,"tracks":[],"private_eq":1,"combined":1.001},"30":{"raw":0.4079,"skill":0.2905,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5353,"random":0.25},{"track":"10-choice","raw":0.2805,"random":0.1}],"private_eq":0.6379,"combined":0.5571},"31":{"raw":0.1684,"skill":0.0943,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.166,"combined":0.0821},"32":{"raw":0.5878,"skill":0.3447,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.5574,"combined":0.2892},"33":{"raw":0.2242,"skill":0.2139,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.2631,"combined":0.2457},"36":{"raw":0.4467,"skill":0.374,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.4334,"combined":0.3527},"37":{"raw":0.5306,"skill":0.4112,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.6074,"combined":0.5029},"38":{"raw":0.2107,"skill":0.1854,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.3924,"combined":0.3668},"39":{"raw":0.9338,"skill":0.9026,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6866,"combined":0.5348},"40":{"raw":0.6103,"skill":0.4987,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3413,"headline":false},{"track":"A · English","score":0.6103,"headline":true},{"track":"C · Arabic pairs","score":0.755,"headline":false},{"track":"C · English pairs","score":0.955,"headline":false}],"private_eq":0.6121,"combined":0.4962},"41":{"raw":0.6826,"skill":0.5238,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.617,"combined":0.4199},"42":{"raw":0.8241,"skill":0.6578,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.9567,"combined":0.9158},"43":{"raw":0.6702,"skill":0.4767,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6822,"combined":0.491},"44":{"raw":0.694,"skill":0.388,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.73,"combined":0.4548},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4374,"skill":0.1834,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4467,"combined":0.1887},"56":{"raw":0.7555,"skill":0.511,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.769,"combined":0.5337},"57":{"raw":0.6165,"skill":0.5687,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4521,"combined":0.3777},"58":{"raw":0.7271,"skill":0.6044,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4868,"combined":0.2486},"59":{"raw":0.757,"skill":0.4962,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6839,"combined":0.3381},"61":{"raw":0.6767,"skill":0.3534,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7139,"combined":0.4222},"62":{"raw":0.7453,"skill":0.6604,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.7006,"combined":0.5972},"64":{"raw":0.6913,"skill":0.6141,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.7162,"combined":0.6422}},"coverage":0.9481,"pending":0.0519,"notes":["Public board run by the lab (results dataset gated)."],"scores":{"balanced_raw":64.1,"balanced_skill":54.09,"breadth_skill":51.32,"public_skill":52.68},"frozen_scores":{"balanced_raw":64.1,"balanced_skill":52.68,"breadth_skill":51.32},"categories":[{"id":"knowledge","raw":0.5061,"skill":0.3647,"combined":0.3582,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7227,"skill":0.6121,"combined":0.5724,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6738,"skill":0.5557,"combined":0.5509,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7221,"skill":0.6921,"combined":0.6638,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.565,"skill":0.3655,"combined":0.414,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jpt-35b-a3b","pub":52.68,"same_skills":52.25,"new_domains":44.43,"v":54.09,"lab":false,"jev":false,"rank_v":82,"rank_pub":108,"name":"JPT-35B-A3B","eq":{"S":51.8,"O":57.52},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":31.08,"new_domains":23.01}}},{"engine":"zev-latest","name":"Zev","long_name":"Zev","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":null,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/bhubbard/zev-rs","base_model":null,"base_url":null,"weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.2973,"conf":0.4342,"ece":0.1378,"brier":0.7765,"over95":0.0168,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.1552,"conf":0.0274,"acc":0.0182},{"lo":0.1,"w":0.0229,"conf":0.1229,"acc":0.143},{"lo":0.2,"w":0.1847,"conf":0.2458,"acc":0.2271},{"lo":0.3,"w":0.1476,"conf":0.3347,"acc":0.2516},{"lo":0.4,"w":0.0131,"conf":0.4609,"acc":0.1858},{"lo":0.5,"w":0.2381,"conf":0.5062,"acc":0.4877},{"lo":0.6,"w":0.0151,"conf":0.6506,"acc":0.2899},{"lo":0.7,"w":0.0234,"conf":0.7593,"acc":0.4167},{"lo":0.8,"w":0.1437,"conf":0.855,"acc":0.3086},{"lo":0.9,"w":0.0562,"conf":0.9833,"acc":0.6234}],"areas":{"knowledge":{"ece":0.1287,"acc":0.2826,"conf":0.4074},"language":{"ece":0.2113,"acc":0.2946,"conf":0.4842},"retrieval":{"ece":0.1275,"acc":0.1641,"conf":0.2916},"tools":{"ece":0.1188,"acc":0.3512,"conf":0.47},"arts":{"ece":0.09,"acc":0.4085,"conf":0.4957}}},"gaps":null,"latency":{"median":0.5,"p95":1.7,"mean":0.7},"results":{"1":{"score":0.3247,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":0.6},"2":{"score":0.1829,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1.1},"3":{"score":0.0098,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":1.8},"4":{"score":0.0003,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":0.5},"5":{"score":0.0001,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":0.7},"6":{"score":0.6863,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1.6},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1.2},"10":{"score":0.3999,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":0.6},"11":{"score":0.1077,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2.2},"12":{"score":0.1671,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":0.4},"20":{"score":0.4974,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.4},"21":{"score":0.4901,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":0.4},"22":{"score":0.0145,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1.3},"23":{"score":0.5022,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.4},"24":{"score":0.2295,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":0.4},"25":{"score":0.2449,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":0.5},"26":{"score":0.2601,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":0.4},"27":{"score":0.2321,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":0.4},"28":{"score":0.4957,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":0.4},"29":{"score":0.2502,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":0.4},"30":{"score":0.1785,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0792,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.5},"32":{"score":0.3577,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":0.5},"33":{"score":0,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":0.9},"36":{"score":0.1607,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1.5},"37":{"score":0.1338,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.6},"38":{"score":0.0263,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1.3},"39":{"score":0.1386,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":0.4},"40":{"score":0.2355,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":0.4},"41":{"score":0.1687,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":0.4},"42":{"score":0.3995,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":0.6},"43":{"score":0.3807,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":0.5},"44":{"score":0.499,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":0.4},"45":{"score":0.1477,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":0.4},"50":{"score":0.5334,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":2.2},"56":{"score":0.496,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":0.7},"57":{"score":0.1331,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":0.5},"58":{"score":0.3259,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":0.5},"59":{"score":0.4148,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":0.7},"61":{"score":0.4995,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":0.7},"62":{"score":0.1473,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":1.1},"64":{"score":0.2102,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":0.6}},"benchmarks":{"1":{"raw":0.3247,"skill":0.0883,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.3515,"combined":0.1155},"2":{"raw":0.1829,"skill":0.0564,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.1341,"combined":0},"3":{"raw":0.0098,"skill":0,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.0531,"combined":0.0247},"4":{"raw":0.0003,"skill":0,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.0127,"combined":0},"5":{"raw":0.0001,"skill":0,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.006,"combined":0},"6":{"raw":0.6863,"skill":0.2645,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0,"combined":0},"10":{"raw":0.3999,"skill":0.0015,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.1077,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.1671,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.3324,"combined":0},"20":{"raw":0.499,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"21":{"raw":0.4901,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5276,"combined":0.0453},"22":{"raw":0.014,"skill":0.0063,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0078,"combined":0},"23":{"raw":0.5088,"skill":0.0176,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2295,"skill":0,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2449,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.25,"combined":0},"26":{"raw":0.2601,"skill":0.0132,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.2321,"skill":0,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.4957,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5013,"combined":0},"29":{"raw":0.2502,"skill":0.0003,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3627,"combined":0.1415},"30":{"raw":0.1785,"skill":0.0057,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2585,"random":0.25},{"track":"10-choice","raw":0.0986,"random":0.1}],"private_eq":0.2062,"combined":0.0277},"31":{"raw":0.0792,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0824,"combined":0},"32":{"raw":0.3577,"skill":0,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0131,"combined":0},"36":{"raw":0.1607,"skill":0.0506,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.1179,"combined":0},"37":{"raw":0.1338,"skill":0,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2027,"combined":0},"38":{"raw":0.0263,"skill":0,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.031,"combined":0},"39":{"raw":0.1386,"skill":0,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.3201,"combined":0},"40":{"raw":0.2355,"skill":0.0164,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.25,"headline":false},{"track":"A · English","score":0.2355,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.555,"headline":false}],"private_eq":0.267,"combined":0.0471},"41":{"raw":0.1687,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3333,"combined":0},"42":{"raw":0.3995,"skill":0,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.486,"combined":0},"43":{"raw":0.3807,"skill":0.0174,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.3697,"combined":0},"44":{"raw":0.499,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"45":{"raw":0.1477,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.5334,"skill":0.3228,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.3379,"combined":0.029},"56":{"raw":0.496,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5081,"combined":0.0059},"57":{"raw":0.1331,"skill":0.025,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3259,"skill":0.0229,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3161,"combined":0},"59":{"raw":0.4148,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.4995,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5036,"combined":0},"62":{"raw":0.1473,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.25,"combined":0},"64":{"raw":0.2102,"skill":0.0127,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.21,"combined":0.0021}},"coverage":1,"pending":0,"notes":["Rule-based engine, no weights; its three submitted variants give identical rows."],"scores":{"balanced_raw":22.43,"balanced_skill":3.37,"breadth_skill":1.44,"public_skill":1.55},"frozen_scores":{"balanced_raw":22.43,"balanced_skill":1.55,"breadth_skill":1.44},"categories":[{"id":"knowledge","raw":0.2593,"skill":0.0067,"combined":0,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2119,"skill":0.0018,"combined":0.0181,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.2004,"skill":0.0092,"combined":0,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1355,"skill":0.0301,"combined":0.0253,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.3759,"skill":0.0599,"combined":0.0111,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"zev-latest","pub":1.55,"same_skills":1.79,"new_domains":3.76,"v":3.37,"lab":false,"jev":false,"rank_v":243,"rank_pub":243,"name":"Zev","eq":{"S":0.76,"O":7.29},"rank_tie":59,"tied":true,"parts":{"public":0,"same_skills":0.46,"new_domains":2.92}}},{"engine":"exaone-4.0-1.2b-jev-v0.3","name":"EXAONE-4.0-1.2B-JEV v0.3","long_name":"EXAONE-4.0-1.2B-JEV v0.3","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":1280000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/carrtesy/EXAONE-4.0-1.2B-JEV","code_url":null,"base_model":"LGAI-EXAONE/EXAONE-4.0-1.2B","base_url":"https://huggingface.co/LGAI-EXAONE/EXAONE-4.0-1.2B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.5646,"conf":0.644,"ece":0.0794,"brier":0.5485,"over95":0.0108,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0336,"conf":0.0749,"acc":0.0641},{"lo":0.1,"w":0.0393,"conf":0.1468,"acc":0.1063},{"lo":0.2,"w":0.0232,"conf":0.2597,"acc":0.2293},{"lo":0.3,"w":0.0649,"conf":0.3525,"acc":0.3265},{"lo":0.4,"w":0.0765,"conf":0.4515,"acc":0.3739},{"lo":0.5,"w":0.1946,"conf":0.546,"acc":0.5249},{"lo":0.6,"w":0.1373,"conf":0.6475,"acc":0.5717},{"lo":0.7,"w":0.1167,"conf":0.7491,"acc":0.6029},{"lo":0.8,"w":0.1107,"conf":0.8518,"acc":0.679},{"lo":0.9,"w":0.2031,"conf":0.9614,"acc":0.871}],"areas":{"knowledge":{"ece":0.0622,"acc":0.4912,"conf":0.5534},"language":{"ece":0.1131,"acc":0.6031,"conf":0.7162},"retrieval":{"ece":0.057,"acc":0.7623,"conf":0.8144},"tools":{"ece":0.1589,"acc":0.6657,"conf":0.8007},"arts":{"ece":0.0452,"acc":0.4379,"conf":0.4827}}},"gaps":null,"latency":{"median":148.5,"p95":1905.6,"mean":597.4},"results":{"1":{"score":0.8743,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":307.7},"2":{"score":0.6377,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":18397.3},"3":{"score":0.5,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3861.3},"4":{"score":0.8847,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1307.4},"5":{"score":0.9001,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1907.5},"6":{"score":0.6909,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":961.5},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":1081.4},"10":{"score":0.6274,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":300.6},"11":{"score":0.5197,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1320.4},"12":{"score":0.5142,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":88.4},"20":{"score":0.6636,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.6},"21":{"score":0.5776,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":61.7},"22":{"score":0.088,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":5364},"23":{"score":0.5338,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":119.1},"24":{"score":0.537,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":105.1},"25":{"score":0.3724,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":104.6},"26":{"score":0.8923,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":94.8},"27":{"score":0.7577,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":94.3},"28":{"score":0.6646,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":73.7},"29":{"score":0.7688,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":139},"30":{"score":0.2544,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0858,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":1116.4},"32":{"score":0.5399,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":325.8},"33":{"score":0.1715,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":388.1},"36":{"score":0.3645,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2008.7},"37":{"score":0.2734,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":178.7},"38":{"score":0.0365,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1781.8},"39":{"score":0.7074,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":515.5},"40":{"score":0.1724,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":97.4},"41":{"score":0.4359,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":96.3},"42":{"score":0.6352,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":95.9},"43":{"score":0.3807,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":95},"44":{"score":0.5548,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.6},"45":{"score":0.1198,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":165.8},"50":{"score":0.3604,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":295.6},"56":{"score":0.5,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":311.8},"57":{"score":0.298,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":161.9},"58":{"score":0.4807,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":108.2},"59":{"score":0.5258,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":175.3},"61":{"score":0.5687,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":152.5},"62":{"score":0.9173,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":196.2},"64":{"score":0.3447,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":168.8}},"benchmarks":{"1":{"raw":0.8743,"skill":0.8303,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.7705,"combined":0.6876},"2":{"raw":0.6377,"skill":0.5816,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7321,"combined":0.6881},"3":{"raw":0.5,"skill":0.4904,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.2873,"combined":0.2662},"4":{"raw":0.8847,"skill":0.8832,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.7643,"combined":0.7595},"5":{"raw":0.9001,"skill":0.8995,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.5004,"combined":0.4926},"6":{"raw":0.6909,"skill":0.2753,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0.0125,"combined":0.0021},"10":{"raw":0.6274,"skill":0.38,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5197,"skill":0.3055,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5142,"skill":0.2723,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.4971,"combined":0.239},"20":{"raw":0.66,"skill":0.3201,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6248,"combined":0.242},"21":{"raw":0.5776,"skill":0.1553,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5773,"combined":0.1459},"22":{"raw":0.0824,"skill":0.0752,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0815,"combined":0.0646},"23":{"raw":0.5142,"skill":0.0284,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.537,"skill":0.3827,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3724,"skill":0.1633,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2924,"combined":0.0467},"26":{"raw":0.8923,"skill":0.8564,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.7577,"skill":0.6768,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6646,"skill":0.3292,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5636,"combined":0.1182},"29":{"raw":0.7688,"skill":0.6917,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6087,"combined":0.4733},"30":{"raw":0.2544,"skill":0.0989,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3465,"random":0.25},{"track":"10-choice","raw":0.1622,"random":0.1}],"private_eq":0.4184,"combined":0.2879},"31":{"raw":0.0858,"skill":0.0043,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1109,"combined":0.0214},"32":{"raw":0.5399,"skill":0.2685,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.3986,"combined":0.0339},"33":{"raw":0.1715,"skill":0.1605,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.1829,"combined":0.1635},"36":{"raw":0.3645,"skill":0.281,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.3114,"combined":0.2131},"37":{"raw":0.2734,"skill":0.0887,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2872,"combined":0.0967},"38":{"raw":0.0365,"skill":0.0057,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.0483,"combined":0.0075},"39":{"raw":0.7074,"skill":0.5696,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.799,"combined":0.702},"40":{"raw":0.1724,"skill":0,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.0863,"headline":false},{"track":"A · English","score":0.1724,"headline":true},{"track":"C · Arabic pairs","score":0.395,"headline":false},{"track":"C · English pairs","score":0.46,"headline":false}],"private_eq":0.2227,"combined":0},"41":{"raw":0.4359,"skill":0.1539,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3647,"combined":0.0371},"42":{"raw":0.6352,"skill":0.2903,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.6212,"combined":0.2556},"43":{"raw":0.3807,"skill":0.0174,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.617,"combined":0.3864},"44":{"raw":0.5548,"skill":0.1096,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6189,"combined":0.23},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3604,"skill":0.0717,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4615,"combined":0.2104},"56":{"raw":0.5,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"57":{"raw":0.298,"skill":0.2104,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.336,"combined":0.2456},"58":{"raw":0.4807,"skill":0.2473,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3313,"combined":0.0206},"59":{"raw":0.5258,"skill":0.0168,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5312,"combined":0.0178},"61":{"raw":0.5687,"skill":0.1374,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6102,"combined":0.2124},"62":{"raw":0.9173,"skill":0.8897,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6702,"combined":0.5562},"64":{"raw":0.3447,"skill":0.1809,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.325,"combined":0.1475}},"coverage":1,"pending":0,"notes":["Training includes When2Call train preferences (author-declared)."],"scores":{"balanced_raw":48.83,"balanced_skill":20.6,"breadth_skill":27.95,"public_skill":30.68},"frozen_scores":{"balanced_raw":48.83,"balanced_skill":30.68,"breadth_skill":27.95},"categories":[{"id":"knowledge","raw":0.3591,"skill":0.15,"combined":0.1519,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4864,"skill":0.2658,"combined":0.2219,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5941,"skill":0.4095,"combined":0.3116,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5934,"skill":0.566,"combined":0.4428,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4232,"skill":0.1386,"combined":0.1621,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"exaone-4.0-1.2b-jev-v0.3","pub":30.68,"same_skills":26.92,"new_domains":7.77,"v":20.6,"lab":false,"jev":false,"rank_v":196,"rank_pub":187,"name":"EXAONE-4.0-1.2B-JEV v0.3","eq":{"S":26.18,"O":12.24},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":15.71,"new_domains":4.9}}},{"engine":"rsi-jev-v5.0-vl-3b","name":"RSI-Jev v5.0 VL 3B","long_name":"RSI-Jev v5.0 VL 3B","variant":null,"meta":{"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/shgao/rsi-jev-v5.0-vl-3b","code_url":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B-Base","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6404,"conf":0.6838,"ece":0.0478,"brier":0.4664,"over95":0.0251,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.059,"conf":0.0335,"acc":0.0676},{"lo":0.1,"w":0.0068,"conf":0.1574,"acc":0.1896},{"lo":0.2,"w":0.0213,"conf":0.2649,"acc":0.2088},{"lo":0.3,"w":0.0477,"conf":0.3548,"acc":0.2922},{"lo":0.4,"w":0.0682,"conf":0.4524,"acc":0.4322},{"lo":0.5,"w":0.1696,"conf":0.5487,"acc":0.5399},{"lo":0.6,"w":0.1294,"conf":0.6475,"acc":0.6322},{"lo":0.7,"w":0.097,"conf":0.7488,"acc":0.7175},{"lo":0.8,"w":0.1005,"conf":0.8536,"acc":0.7933},{"lo":0.9,"w":0.3006,"conf":0.9719,"acc":0.8805}],"areas":{"knowledge":{"ece":0.0602,"acc":0.5817,"conf":0.6391},"language":{"ece":0.0423,"acc":0.7462,"conf":0.742},"retrieval":{"ece":0.1078,"acc":0.6848,"conf":0.7859},"tools":{"ece":0.1286,"acc":0.7951,"conf":0.9108},"arts":{"ece":0.0385,"acc":0.4511,"conf":0.4595}}},"gaps":null,"latency":{"median":18.7,"p95":78.7,"mean":41.5},"results":{"1":{"score":0.8583,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":36},"2":{"score":0.6261,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":458.8},"3":{"score":0.5531,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":153.4},"4":{"score":0.662,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":28.6},"5":{"score":0.7603,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":42.1},"6":{"score":0.7489,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":65.1},"9":{"score":0.2273,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":113.7},"10":{"score":0.6414,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":28.9},"11":{"score":0.7195,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":117.6},"12":{"score":0.6525,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":18.6},"20":{"score":0.6014,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.3},"21":{"score":0.5666,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":18},"22":{"score":0.0725,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":130.8},"23":{"score":0.6356,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.8},"24":{"score":0.6933,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":18.3},"25":{"score":0.3622,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":18.6},"26":{"score":0.9668,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.2},"27":{"score":0.8899,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.3},"28":{"score":0.7001,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":18},"29":{"score":0.8679,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":18.3},"30":{"score":0.2995,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0996,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":27.3},"32":{"score":0.5931,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":29.2},"33":{"score":0.3073,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":40.5},"36":{"score":0.4035,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":796.3},"37":{"score":0.3987,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.7},"38":{"score":0.1076,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":204.7},"39":{"score":0.8993,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":19.2},"40":{"score":0.3171,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":18},"41":{"score":0.5075,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":18.3},"42":{"score":0.7724,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":19},"43":{"score":0.4228,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":18.2},"44":{"score":0.6684,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":18.2},"45":{"score":0.0978,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":18.6},"50":{"score":0.3866,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":27.3},"56":{"score":0.6305,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":44.3},"57":{"score":0.4324,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":18.4},"58":{"score":0.6247,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":18.3},"59":{"score":0.6158,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":26.6},"61":{"score":0.6198,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":18.9},"62":{"score":0.5742,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":24.8},"64":{"score":0.5417,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":18.3}},"benchmarks":{"1":{"raw":0.8583,"skill":0.8087,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.8553,"combined":0.8034},"2":{"raw":0.6261,"skill":0.5682,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7017,"combined":0.6526},"3":{"raw":0.5531,"skill":0.5445,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.4451,"combined":0.4289},"4":{"raw":0.662,"skill":0.6577,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.7254,"combined":0.7197},"5":{"raw":0.7603,"skill":0.7589,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.6527,"combined":0.6476},"6":{"raw":0.7489,"skill":0.4113,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2273,"skill":0.2273,"coverage":1,"random":0,"tracks":[],"private_eq":0.125,"combined":0.1159},"10":{"raw":0.6414,"skill":0.4033,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7195,"skill":0.5944,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6525,"skill":0.4795,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6529,"combined":0.4751},"20":{"raw":0.6011,"skill":0.2022,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6498,"combined":0.2925},"21":{"raw":0.5666,"skill":0.1332,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6072,"combined":0.2064},"22":{"raw":0.0702,"skill":0.063,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.1273,"combined":0.1113},"23":{"raw":0.6282,"skill":0.2564,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6933,"skill":0.5911,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3622,"skill":0.1497,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3215,"combined":0.0859},"26":{"raw":0.9668,"skill":0.9557,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8899,"skill":0.8532,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7001,"skill":0.4002,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.606,"combined":0.2039},"29":{"raw":0.8679,"skill":0.8239,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.8289,"combined":0.7703},"30":{"raw":0.2995,"skill":0.1558,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4071,"random":0.25},{"track":"10-choice","raw":0.1918,"random":0.1}],"private_eq":0.4972,"combined":0.3845},"31":{"raw":0.0996,"skill":0.0193,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0923,"combined":0.0009},"32":{"raw":0.5931,"skill":0.3531,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0.3073,"skill":0.2981,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.3333,"combined":0.3177},"36":{"raw":0.4035,"skill":0.3252,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.3955,"combined":0.3093},"37":{"raw":0.3987,"skill":0.2458,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.543,"combined":0.4212},"38":{"raw":0.1076,"skill":0.0791,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.196,"combined":0.1617},"39":{"raw":0.8993,"skill":0.8519,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6815,"combined":0.5272},"40":{"raw":0.3171,"skill":0.1215,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.379,"headline":false},{"track":"A · English","score":0.3171,"headline":true},{"track":"C · Arabic pairs","score":0.465,"headline":false},{"track":"C · English pairs","score":0.53,"headline":false}],"private_eq":0.3681,"combined":0.1787},"41":{"raw":0.5075,"skill":0.2612,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.4455,"combined":0.1597},"42":{"raw":0.7724,"skill":0.5572,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.9011,"combined":0.8064},"43":{"raw":0.4228,"skill":0.0842,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.5988,"combined":0.3572},"44":{"raw":0.6684,"skill":0.3368,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7115,"combined":0.4174},"45":{"raw":0.0978,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3866,"skill":0.1098,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4219,"combined":0.1523},"56":{"raw":0.6305,"skill":0.261,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6878,"combined":0.3694},"57":{"raw":0.4324,"skill":0.3616,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4024,"combined":0.3211},"58":{"raw":0.6247,"skill":0.456,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4047,"combined":0.1282},"59":{"raw":0.6158,"skill":0.2034,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6305,"combined":0.2261},"61":{"raw":0.6198,"skill":0.2396,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6971,"combined":0.3882},"62":{"raw":0.5742,"skill":0.4323,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5737,"combined":0.4261},"64":{"raw":0.5417,"skill":0.4271,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.4785,"combined":0.3416}},"coverage":1,"pending":0,"notes":["Training sources include GSM8K and ANLI (author-declared)."],"scores":{"balanced_raw":53.6,"balanced_skill":38.65,"breadth_skill":36.54,"public_skill":38.11},"frozen_scores":{"balanced_raw":53.6,"balanced_skill":38.11,"breadth_skill":36.54},"categories":[{"id":"knowledge","raw":0.4209,"skill":0.2407,"combined":0.218,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6132,"skill":0.4503,"combined":0.4231,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5818,"skill":0.4298,"combined":0.4834,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.578,"skill":0.5281,"combined":0.4951,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.4657,"skill":0.1986,"combined":0.2208,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"rsi-jev-v5.0-vl-3b","pub":38.11,"same_skills":38.69,"new_domains":29.85,"v":38.65,"lab":false,"jev":false,"rank_v":139,"rank_pub":161,"name":"RSI-Jev v5.0 VL 3B","eq":{"S":38.08,"O":39.51},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":22.85,"new_domains":15.8}}},{"engine":"ezjev-4b-s2","name":"ezjev 4B s2","long_name":"ezjev 4B s2","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4660000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/everettjf/ezjev-4b-s2","code_url":null,"base_model":"Qwen/Qwen3.5-4B","base_url":"https://huggingface.co/Qwen/Qwen3.5-4B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7026,"conf":0.7655,"ece":0.063,"brier":0.4031,"over95":0.0251,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0048,"conf":0.087,"acc":0.0439},{"lo":0.1,"w":0.0273,"conf":0.1494,"acc":0.0728},{"lo":0.2,"w":0.0245,"conf":0.2514,"acc":0.1603},{"lo":0.3,"w":0.0354,"conf":0.3564,"acc":0.2109},{"lo":0.4,"w":0.0488,"conf":0.4542,"acc":0.3524},{"lo":0.5,"w":0.1152,"conf":0.5516,"acc":0.5358},{"lo":0.6,"w":0.0938,"conf":0.6494,"acc":0.5965},{"lo":0.7,"w":0.0975,"conf":0.7491,"acc":0.7074},{"lo":0.8,"w":0.1254,"conf":0.8539,"acc":0.762},{"lo":0.9,"w":0.4273,"conf":0.9725,"acc":0.9118}],"areas":{"knowledge":{"ece":0.0962,"acc":0.6402,"conf":0.7365},"language":{"ece":0.022,"acc":0.8171,"conf":0.8247},"retrieval":{"ece":0.1484,"acc":0.6993,"conf":0.8437},"tools":{"ece":0.0185,"acc":0.9138,"conf":0.9045},"arts":{"ece":0.0726,"acc":0.5096,"conf":0.5772}}},"gaps":null,"latency":{"median":35.5,"p95":192.1,"mean":72.9},"results":{"1":{"score":0.9604,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":136.5},"2":{"score":0.6603,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":385.3},"3":{"score":0.8248,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":222},"4":{"score":0.8558,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":74.2},"5":{"score":0.928,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":110.7},"6":{"score":0.7989,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":184.6},"9":{"score":0.2045,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":353},"10":{"score":0.4105,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":73.9},"11":{"score":0.8248,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":368.4},"12":{"score":0.6401,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":34.4},"20":{"score":0.8352,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.1},"21":{"score":0.6176,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":33.9},"22":{"score":0.0575,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":256},"23":{"score":0.5764,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.1},"24":{"score":0.7608,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":33.9},"25":{"score":0.3878,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":34.1},"26":{"score":0.9815,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":33.8},"27":{"score":0.9317,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":33.7},"28":{"score":0.7877,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":33.8},"29":{"score":0.9243,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":34.7},"30":{"score":0.4177,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1354,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":72.6},"32":{"score":0.5545,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":73.4},"33":{"score":0.2745,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":152.7},"36":{"score":0.4436,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":581.8},"37":{"score":0.5662,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":35.6},"38":{"score":0.3439,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":400.6},"39":{"score":0.8221,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":96.9},"40":{"score":0.5441,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":34.3},"41":{"score":0.7759,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":34.5},"42":{"score":0.7819,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":36.1},"43":{"score":0.5316,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":34},"44":{"score":0.6766,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":34.4},"45":{"score":0.1118,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":35.6},"50":{"score":0.4266,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":73.8},"56":{"score":0.7195,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":133.5},"57":{"score":0.5426,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":34.8},"58":{"score":0.7116,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":34.5},"59":{"score":0.8038,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":72.2},"61":{"score":0.6198,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":36.2},"62":{"score":0.8814,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":38.9},"64":{"score":0.7159,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":34.4}},"benchmarks":{"1":{"raw":0.9604,"skill":0.9466,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9252,"combined":0.8989},"2":{"raw":0.6603,"skill":0.6077,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.742,"combined":0.6996},"3":{"raw":0.8248,"skill":0.8214,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.6691,"combined":0.6599},"4":{"raw":0.8558,"skill":0.8539,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8648,"combined":0.8625},"5":{"raw":0.928,"skill":0.9276,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.6595,"combined":0.6545},"6":{"raw":0.7989,"skill":0.5285,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2045,"skill":0.2045,"coverage":1,"random":0,"tracks":[],"private_eq":0.2125,"combined":0.2044},"10":{"raw":0.4105,"skill":0.0191,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8248,"skill":0.7467,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6401,"skill":0.4609,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.5948,"combined":0.3871},"20":{"raw":0.8356,"skill":0.6712,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8642,"combined":0.7263},"21":{"raw":0.6176,"skill":0.2352,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6619,"combined":0.317},"22":{"raw":0.0541,"skill":0.0467,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0755,"combined":0.0585},"23":{"raw":0.5518,"skill":0.1035,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7608,"skill":0.6811,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3485,"combined":0.1223},"26":{"raw":0.9815,"skill":0.9753,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9317,"skill":0.9089,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7877,"skill":0.5754,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6808,"combined":0.3553},"29":{"raw":0.9243,"skill":0.8991,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.8839,"combined":0.8444},"30":{"raw":0.4177,"skill":0.3036,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.555,"random":0.25},{"track":"10-choice","raw":0.2805,"random":0.1}],"private_eq":0.5746,"combined":0.4794},"31":{"raw":0.1354,"skill":0.0583,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1729,"combined":0.0897},"32":{"raw":0.5545,"skill":0.2917,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.4275,"combined":0.0803},"33":{"raw":0.2745,"skill":0.2649,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.3132,"combined":0.2971},"36":{"raw":0.4436,"skill":0.3706,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.4058,"combined":0.3211},"37":{"raw":0.5662,"skill":0.4559,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.6319,"combined":0.534},"38":{"raw":0.3439,"skill":0.3229,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.4941,"combined":0.4729},"39":{"raw":0.8221,"skill":0.7383,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6592,"combined":0.494},"40":{"raw":0.5441,"skill":0.4135,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2581,"headline":false},{"track":"A · English","score":0.5441,"headline":true},{"track":"C · Arabic pairs","score":0.635,"headline":false},{"track":"C · English pairs","score":0.91,"headline":false}],"private_eq":0.553,"combined":0.4193},"41":{"raw":0.7759,"skill":0.6638,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.411,"combined":0.1074},"42":{"raw":0.7819,"skill":0.5757,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.8289,"combined":0.6643},"43":{"raw":0.5316,"skill":0.2568,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6248,"combined":0.3989},"44":{"raw":0.6766,"skill":0.3532,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7036,"combined":0.4014},"45":{"raw":0.1118,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4266,"skill":0.1678,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4367,"combined":0.174},"56":{"raw":0.7195,"skill":0.439,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6236,"combined":0.2395},"57":{"raw":0.5426,"skill":0.4855,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5547,"combined":0.4944},"58":{"raw":0.7116,"skill":0.582,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4133,"combined":0.1408},"59":{"raw":0.8038,"skill":0.5932,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6739,"combined":0.3171},"61":{"raw":0.6198,"skill":0.2396,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.728,"combined":0.4507},"62":{"raw":0.8814,"skill":0.8419,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.7056,"combined":0.604},"64":{"raw":0.7159,"skill":0.6449,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.6932,"combined":0.6131}},"coverage":1,"pending":0,"notes":["Training includes train/dev splits of about 17 index benchmarks (author-declared)."],"scores":{"balanced_raw":63.16,"balanced_skill":43.9,"breadth_skill":48.74,"public_skill":50.82},"frozen_scores":{"balanced_raw":63.16,"balanced_skill":50.82,"breadth_skill":48.74},"categories":[{"id":"knowledge","raw":0.47,"skill":0.31,"combined":0.2854,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7206,"skill":0.6049,"combined":0.4706,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6937,"skill":0.5632,"combined":0.5197,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7201,"skill":0.6992,"combined":0.6256,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5336,"skill":0.3115,"combined":0.3778,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"ezjev-4b-s2","pub":50.82,"same_skills":46.22,"new_domains":31.21,"v":43.9,"lab":false,"jev":false,"rank_v":119,"rank_pub":115,"name":"ezjev 4B s2","eq":{"S":45.7,"O":41.19},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":27.42,"new_domains":16.48}}},{"engine":"bekko-v0-17m","name":"Bekko System One v0 17M","long_name":"Bekko System One v0 17M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":17000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/hotchpotch/bekko-system-one-v0-17m","code_url":null,"base_model":"jhu-clsp/ettin-encoder-17m","base_url":"https://huggingface.co/jhu-clsp/ettin-encoder-17m","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110189,"unsupported":12,"error":0},"lifted":false,"calibration":{"acc":0.336,"conf":0.4964,"ece":0.1605,"brier":0.7457,"over95":0.0055,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0746,"conf":0.0533,"acc":0.0216},{"lo":0.1,"w":0.0342,"conf":0.1418,"acc":0.0656},{"lo":0.2,"w":0.0522,"conf":0.2665,"acc":0.2169},{"lo":0.3,"w":0.1668,"conf":0.3565,"acc":0.2909},{"lo":0.4,"w":0.1484,"conf":0.4436,"acc":0.3291},{"lo":0.5,"w":0.216,"conf":0.5439,"acc":0.4659},{"lo":0.6,"w":0.1224,"conf":0.6469,"acc":0.438},{"lo":0.7,"w":0.0872,"conf":0.7451,"acc":0.3565},{"lo":0.8,"w":0.0619,"conf":0.8483,"acc":0.3429},{"lo":0.9,"w":0.0364,"conf":0.943,"acc":0.4648}],"areas":{"knowledge":{"ece":0.146,"acc":0.3019,"conf":0.4479},"language":{"ece":0.205,"acc":0.3553,"conf":0.5602},"retrieval":{"ece":0.1392,"acc":0.4382,"conf":0.5693},"tools":{"ece":0.3213,"acc":0.1667,"conf":0.4879},"arts":{"ece":0.0443,"acc":0.396,"conf":0.4353}}},"gaps":null,"latency":{"median":6.8,"p95":16.5,"mean":8.6},"results":{"1":{"score":0.3459,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":8.8},"2":{"score":0.2859,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":32},"3":{"score":0.0236,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":34.2},"4":{"score":0.5982,"jev":null,"metric":"macro-F1","requests":3080,"answered":3073,"unsupported":7,"errors":0,"median_ms":7.4},"5":{"score":0.461,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":8.5},"6":{"score":0.4931,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":13.7},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":34.3},"10":{"score":0.3775,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":9},"11":{"score":0.28,"jev":null,"metric":"macro-F1","requests":123,"answered":120,"unsupported":3,"errors":0,"median_ms":77.9},"12":{"score":0.2501,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":6},"20":{"score":0.5216,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":5.7},"21":{"score":0.5171,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":5.4},"22":{"score":0.005,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":18.8},"23":{"score":0.52,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.7},"24":{"score":0.2944,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":5.8},"25":{"score":0.3061,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":6.1},"26":{"score":0.4297,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":5.5},"27":{"score":0.2824,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":5.5},"28":{"score":0.5004,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":5.5},"29":{"score":0.3055,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":5.9},"30":{"score":0.1937,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0714,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":7.1},"32":{"score":0.3923,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":9.7},"33":{"score":0.0006,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":14.7},"36":{"score":0.1682,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":49.6},"37":{"score":0.1743,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6.9},"38":{"score":0.0286,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":31.6},"39":{"score":0.2961,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":6.9},"40":{"score":0.2761,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":7.3},"41":{"score":0.3978,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":6.9},"42":{"score":0.2738,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":7.1},"43":{"score":0.4088,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":5.8},"44":{"score":0.4962,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":6},"45":{"score":0.1737,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":6.3},"50":{"score":0.469,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":8.2},"56":{"score":0.5305,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":12},"57":{"score":0.1625,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":6},"58":{"score":0.3154,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":6},"59":{"score":0.5157,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":8.8},"61":{"score":0.5062,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":7.3},"62":{"score":0.1936,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":7.7},"64":{"score":0.2841,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":6}},"benchmarks":{"1":{"raw":0.3459,"skill":0.117,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.3415,"combined":0.1019},"2":{"raw":0.2838,"skill":0.1729,"coverage":0.9927,"random":0.1341,"tracks":[],"private_eq":0.3815,"combined":0.2785},"3":{"raw":0.0236,"skill":0.0048,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.0189,"combined":0},"4":{"raw":0.5968,"skill":0.5917,"coverage":0.9977,"random":0.0127,"tracks":[],"private_eq":0.4983,"combined":0.487},"5":{"raw":0.461,"skill":0.4577,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.2783,"combined":0.2666},"6":{"raw":0.4931,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0,"combined":0},"10":{"raw":0.3775,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.2722,"skill":0,"coverage":0.9756,"random":0.3085,"tracks":[]},"12":{"raw":0.2501,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.3324,"combined":0},"20":{"raw":0.5124,"skill":0.0248,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"21":{"raw":0.5171,"skill":0.0342,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5028,"combined":0},"22":{"raw":0.004,"skill":0,"coverage":0.9995,"random":0.0078,"tracks":[],"private_eq":0.0078,"combined":0},"23":{"raw":0.5276,"skill":0.0552,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.2944,"skill":0.0592,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3061,"skill":0.0748,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2581,"combined":0.0004},"26":{"raw":0.4297,"skill":0.2394,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.2824,"skill":0.0429,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5004,"skill":0.0008,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5113,"combined":0.0123},"29":{"raw":0.3055,"skill":0.074,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3371,"combined":0.107},"30":{"raw":0.1937,"skill":0.0241,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2798,"random":0.25},{"track":"10-choice","raw":0.1077,"random":0.1}],"private_eq":0.1979,"combined":0.0176},"31":{"raw":0.0714,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0854,"combined":0},"32":{"raw":0.3923,"skill":0.0339,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.3746,"combined":0},"33":{"raw":0.0006,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0131,"combined":0},"36":{"raw":0.1682,"skill":0.059,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.135,"combined":0.0112},"37":{"raw":0.1743,"skill":0,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2027,"combined":0},"38":{"raw":0.0286,"skill":0,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.031,"combined":0},"39":{"raw":0.2961,"skill":0,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.3201,"combined":0},"40":{"raw":0.2761,"skill":0.0687,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2525,"headline":false},{"track":"A · English","score":0.2761,"headline":true},{"track":"C · Arabic pairs","score":0.54,"headline":false},{"track":"C · English pairs","score":0.41,"headline":false}],"private_eq":0.2237,"combined":0},"41":{"raw":0.3978,"skill":0.0967,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3333,"combined":0},"42":{"raw":0.2738,"skill":0,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.486,"combined":0},"43":{"raw":0.4088,"skill":0.062,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.3697,"combined":0},"44":{"raw":0.4962,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"45":{"raw":0.1737,"skill":0.0115,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.469,"skill":0.2293,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4022,"combined":0.1234},"56":{"raw":0.5305,"skill":0.061,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5252,"combined":0.0405},"57":{"raw":0.1625,"skill":0.058,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3154,"skill":0.0077,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3157,"combined":0},"59":{"raw":0.5157,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.5062,"skill":0.0124,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5092,"combined":0.0081},"62":{"raw":0.1936,"skill":0,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.25,"combined":0},"64":{"raw":0.2841,"skill":0.1051,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.2867,"combined":0.0991}},"coverage":0.9992,"pending":0.0008,"notes":["Encoder; 8 self-declared training matches counted wrong."],"scores":{"balanced_raw":29.75,"balanced_skill":7.66,"breadth_skill":6.36,"public_skill":7.44},"frozen_scores":{"balanced_raw":29.75,"balanced_skill":7.44,"breadth_skill":6.36},"categories":[{"id":"knowledge","raw":0.2761,"skill":0.0253,"combined":0,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2901,"skill":0.027,"combined":0.0064,"coverage":0.9974,"pending":0,"n":9},{"id":"retrieval","raw":0.4064,"skill":0.2126,"combined":0.1448,"coverage":0.9996,"pending":0,"n":6},{"id":"tools","raw":0.1705,"skill":0.0591,"combined":0.068,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.3857,"skill":0.0748,"combined":0.0393,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"bekko-v0-17m","pub":7.44,"same_skills":5.89,"new_domains":7.4,"v":7.66,"lab":false,"jev":false,"rank_v":235,"rank_pub":234,"name":"Bekko System One v0 17M","eq":{"S":4.91,"O":11.78},"rank_tie":56,"tied":true,"parts":{"public":0,"same_skills":2.94,"new_domains":4.71}}},{"engine":"bekko-v0-68m","name":"Bekko System One v0 68M","long_name":"Bekko System One v0 68M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":68000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/hotchpotch/bekko-system-one-v0-68m","code_url":null,"base_model":"jhu-clsp/ettin-encoder-68m","base_url":"https://huggingface.co/jhu-clsp/ettin-encoder-68m","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110189,"unsupported":12,"error":0},"lifted":false,"calibration":{"acc":0.4126,"conf":0.526,"ece":0.1134,"brier":0.6848,"over95":0.0111,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0478,"conf":0.0534,"acc":0.0404},{"lo":0.1,"w":0.0414,"conf":0.1564,"acc":0.0999},{"lo":0.2,"w":0.0553,"conf":0.2585,"acc":0.1838},{"lo":0.3,"w":0.132,"conf":0.3495,"acc":0.3145},{"lo":0.4,"w":0.1359,"conf":0.4467,"acc":0.402},{"lo":0.5,"w":0.2285,"conf":0.5469,"acc":0.4867},{"lo":0.6,"w":0.1506,"conf":0.6458,"acc":0.4823},{"lo":0.7,"w":0.0958,"conf":0.7447,"acc":0.4996},{"lo":0.8,"w":0.0541,"conf":0.8473,"acc":0.5513},{"lo":0.9,"w":0.0586,"conf":0.9641,"acc":0.6612}],"areas":{"knowledge":{"ece":0.1385,"acc":0.3232,"conf":0.4617},"language":{"ece":0.1046,"acc":0.4888,"conf":0.5828},"retrieval":{"ece":0.1039,"acc":0.6123,"conf":0.6653},"tools":{"ece":0.3198,"acc":0.2132,"conf":0.533},"arts":{"ece":0.0374,"acc":0.4321,"conf":0.4498}}},"gaps":null,"latency":{"median":11.8,"p95":30.6,"mean":15},"results":{"1":{"score":0.3737,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":15.1},"2":{"score":0.4865,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":70.7},"3":{"score":0.0256,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":84.5},"4":{"score":0.7181,"jev":null,"metric":"macro-F1","requests":3080,"answered":3073,"unsupported":7,"errors":0,"median_ms":12.2},"5":{"score":0.7644,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":14.4},"6":{"score":0.5178,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":25.1},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":59.9},"10":{"score":0.4701,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":14.5},"11":{"score":0.4185,"jev":null,"metric":"macro-F1","requests":123,"answered":120,"unsupported":3,"errors":0,"median_ms":142.8},"12":{"score":0.317,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":10.9},"20":{"score":0.737,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.6},"21":{"score":0.4597,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":10.4},"22":{"score":0.088,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":48.3},"23":{"score":0.496,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.2},"24":{"score":0.3289,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":10.7},"25":{"score":0.2551,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":10.9},"26":{"score":0.5703,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":10.3},"27":{"score":0.3754,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":10.5},"28":{"score":0.5233,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":10.3},"29":{"score":0.3661,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":10.8},"30":{"score":0.2062,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.078,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.8},"32":{"score":0.5293,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":14.8},"33":{"score":0,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":21.4},"36":{"score":0.2151,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":113.6},"37":{"score":0.2714,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":11.9},"38":{"score":0.0422,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":39.2},"39":{"score":0.4592,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":11.2},"40":{"score":0.2649,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":10.9},"41":{"score":0.325,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":11.1},"42":{"score":0.4848,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":12.3},"43":{"score":0.3737,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":10.6},"44":{"score":0.5054,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":10.8},"45":{"score":0.1557,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":11.2},"50":{"score":0.3371,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":13.2},"56":{"score":0.477,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":17.4},"57":{"score":0.1653,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":10.9},"58":{"score":0.3296,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":10.9},"59":{"score":0.0144,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":14},"61":{"score":0.5665,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":12.2},"62":{"score":0.3738,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":12.9},"64":{"score":0.3295,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":10.6}},"benchmarks":{"1":{"raw":0.3737,"skill":0.1545,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.3265,"combined":0.0814},"2":{"raw":0.4829,"skill":0.4029,"coverage":0.9927,"random":0.1341,"tracks":[],"private_eq":0.5139,"combined":0.4332},"3":{"raw":0.0256,"skill":0.0068,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.0189,"combined":0},"4":{"raw":0.7165,"skill":0.7128,"coverage":0.9977,"random":0.0127,"tracks":[],"private_eq":0.7054,"combined":0.6992},"5":{"raw":0.7644,"skill":0.763,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.5849,"combined":0.5786},"6":{"raw":0.5178,"skill":0,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0,"combined":0},"10":{"raw":0.4701,"skill":0.1183,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.4067,"skill":0.142,"coverage":0.9756,"random":0.3085,"tracks":[]},"12":{"raw":0.317,"skill":0,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.3324,"combined":0},"20":{"raw":0.7381,"skill":0.4762,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7642,"combined":0.524},"21":{"raw":0.4597,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"22":{"raw":0.0844,"skill":0.0772,"coverage":0.9995,"random":0.0078,"tracks":[],"private_eq":0.0565,"combined":0.0391},"23":{"raw":0.5177,"skill":0.0354,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.3289,"skill":0.1052,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2551,"skill":0.0068,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2674,"combined":0.013},"26":{"raw":0.5703,"skill":0.4269,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.3754,"skill":0.167,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.5233,"skill":0.0466,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5387,"combined":0.0678},"29":{"raw":0.3661,"skill":0.1548,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3627,"combined":0.1415},"30":{"raw":0.2062,"skill":0.0383,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.2828,"random":0.25},{"track":"10-choice","raw":0.1296,"random":0.1}],"private_eq":0.2148,"combined":0.0383},"31":{"raw":0.078,"skill":0,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0873,"combined":0},"32":{"raw":0.5293,"skill":0.2517,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.371,"combined":0},"33":{"raw":0,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0131,"combined":0},"36":{"raw":0.2151,"skill":0.112,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.2518,"combined":0.1449},"37":{"raw":0.2714,"skill":0.0861,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2027,"combined":0},"38":{"raw":0.0422,"skill":0.0116,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.0419,"combined":0.0009},"39":{"raw":0.4592,"skill":0.2046,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.5298,"combined":0.3015},"40":{"raw":0.2649,"skill":0.0544,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2373,"headline":false},{"track":"A · English","score":0.2649,"headline":true},{"track":"C · Arabic pairs","score":0.505,"headline":false},{"track":"C · English pairs","score":0.27,"headline":false}],"private_eq":0.2757,"combined":0.0585},"41":{"raw":0.325,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3714,"combined":0.0473},"42":{"raw":0.4848,"skill":0,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.486,"combined":0},"43":{"raw":0.3737,"skill":0.0063,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.3697,"combined":0},"44":{"raw":0.5054,"skill":0.0108,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"45":{"raw":0.1557,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3371,"skill":0.0379,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4219,"combined":0.1523},"56":{"raw":0.477,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5209,"combined":0.0318},"57":{"raw":0.1653,"skill":0.0612,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.3296,"skill":0.0283,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3289,"combined":0.0171},"59":{"raw":0.0144,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.5665,"skill":0.133,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5821,"combined":0.1556},"62":{"raw":0.3738,"skill":0.1651,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3351,"combined":0.1043},"64":{"raw":0.3295,"skill":0.1619,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.3174,"combined":0.1379}},"coverage":0.9992,"pending":0.0008,"notes":["Encoder; 8 self-declared training matches counted wrong."],"scores":{"balanced_raw":33.88,"balanced_skill":12.4,"breadth_skill":11.19,"public_skill":13.01},"frozen_scores":{"balanced_raw":33.88,"balanced_skill":13.01,"breadth_skill":11.19},"categories":[{"id":"knowledge","raw":0.2839,"skill":0.0399,"combined":0.0089,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.2997,"skill":0.0637,"combined":0.0648,"coverage":0.9974,"pending":0,"n":9},{"id":"retrieval","raw":0.5076,"skill":0.3219,"combined":0.2855,"coverage":0.9996,"pending":0,"n":6},{"id":"tools","raw":0.2474,"skill":0.141,"combined":0.1133,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.4111,"skill":0.1314,"combined":0.1685,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"bekko-v0-68m","pub":13.01,"same_skills":12.62,"new_domains":8.73,"v":12.4,"lab":false,"jev":false,"rank_v":223,"rank_pub":228,"name":"Bekko System One v0 68M","eq":{"S":11.72,"O":13.43},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":7.03,"new_domains":5.37}}},{"engine":"bekko-v0-400m","name":"Bekko System One v0 400M","long_name":"Bekko System One v0 400M","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":400000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/hotchpotch/bekko-system-one-v0-400m","code_url":null,"base_model":"jhu-clsp/ettin-encoder-400m","base_url":"https://huggingface.co/jhu-clsp/ettin-encoder-400m","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110189,"unsupported":12,"error":0},"lifted":false,"calibration":{"acc":0.4889,"conf":0.5998,"ece":0.1108,"brier":0.6306,"over95":0.0189,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0333,"conf":0.0756,"acc":0.0581},{"lo":0.1,"w":0.0356,"conf":0.1442,"acc":0.0747},{"lo":0.2,"w":0.0434,"conf":0.2587,"acc":0.2205},{"lo":0.3,"w":0.0902,"conf":0.3527,"acc":0.2523},{"lo":0.4,"w":0.0968,"conf":0.4487,"acc":0.3744},{"lo":0.5,"w":0.2103,"conf":0.5476,"acc":0.5025},{"lo":0.6,"w":0.1438,"conf":0.6472,"acc":0.5408},{"lo":0.7,"w":0.1111,"conf":0.7478,"acc":0.6417},{"lo":0.8,"w":0.1033,"conf":0.8502,"acc":0.6559},{"lo":0.9,"w":0.1323,"conf":0.9567,"acc":0.7055}],"areas":{"knowledge":{"ece":0.1343,"acc":0.4024,"conf":0.5367},"language":{"ece":0.1186,"acc":0.5893,"conf":0.7034},"retrieval":{"ece":0.1284,"acc":0.7062,"conf":0.7259},"tools":{"ece":0.2752,"acc":0.3181,"conf":0.5932},"arts":{"ece":0.0386,"acc":0.4272,"conf":0.455}}},"gaps":null,"latency":{"median":17.1,"p95":72.2,"mean":27.8},"results":{"1":{"score":0.4274,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":25.7},"2":{"score":0.6391,"jev":null,"metric":"nDCG@10","requests":685,"answered":680,"unsupported":5,"errors":0,"median_ms":226.7},"3":{"score":0.0394,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":210.2},"4":{"score":0.8035,"jev":null,"metric":"macro-F1","requests":3080,"answered":3073,"unsupported":7,"errors":0,"median_ms":25.3},"5":{"score":0.8586,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":33.1},"6":{"score":0.6612,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":58},"9":{"score":0,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":167.9},"10":{"score":0.4152,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":25.7},"11":{"score":0.6508,"jev":null,"metric":"macro-F1","requests":123,"answered":120,"unsupported":3,"errors":0,"median_ms":393.9},"12":{"score":0.4114,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":15.4},"20":{"score":0.724,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.3},"21":{"score":0.5498,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":14.9},"22":{"score":0.028,"jev":null,"metric":"accuracy","requests":2000,"answered":1999,"unsupported":1,"errors":0,"median_ms":132.1},"23":{"score":0.5022,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":16.7},"24":{"score":0.4148,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":15.3},"25":{"score":0.2857,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":15.6},"26":{"score":0.7942,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":14.9},"27":{"score":0.6041,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":14.9},"28":{"score":0.6922,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":14.8},"29":{"score":0.6251,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":15.4},"30":{"score":0.2945,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1134,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":21.3},"32":{"score":0.5691,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":25.4},"33":{"score":0,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":45.2},"36":{"score":0.3476,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":356.5},"37":{"score":0.4076,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":17.6},"38":{"score":0.0969,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":88.1},"39":{"score":0.7247,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":16.2},"40":{"score":0.2604,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":15.5},"41":{"score":0.4251,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":15.9},"42":{"score":0.5968,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":18},"43":{"score":0.3842,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":15.4},"44":{"score":0.5588,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":15.4},"45":{"score":0.1257,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":16.8},"50":{"score":0.3156,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":22.5},"56":{"score":0.544,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":36.3},"57":{"score":0.1961,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":15.6},"58":{"score":0.3817,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":15.6},"59":{"score":0.0121,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":22.7},"61":{"score":0.5683,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":17.9},"62":{"score":0.3469,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":20.7},"64":{"score":0.411,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":15.3}},"benchmarks":{"1":{"raw":0.4274,"skill":0.227,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.4263,"combined":0.2177},"2":{"raw":0.6344,"skill":0.5778,"coverage":0.9927,"random":0.1341,"tracks":[],"private_eq":0.6927,"combined":0.642},"3":{"raw":0.0394,"skill":0.0209,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.0189,"combined":0},"4":{"raw":0.8017,"skill":0.7991,"coverage":0.9977,"random":0.0127,"tracks":[],"private_eq":0.698,"combined":0.6916},"5":{"raw":0.8586,"skill":0.8577,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.6366,"combined":0.6312},"6":{"raw":0.6612,"skill":0.2056,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0,"skill":0,"coverage":1,"random":0,"tracks":[],"private_eq":0,"combined":0},"10":{"raw":0.4152,"skill":0.027,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.6181,"skill":0.4477,"coverage":0.9756,"random":0.3085,"tracks":[]},"12":{"raw":0.4114,"skill":0.1183,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.4888,"combined":0.2265},"20":{"raw":0.7209,"skill":0.4419,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8571,"combined":0.7119},"21":{"raw":0.5498,"skill":0.0997,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5326,"combined":0.0554},"22":{"raw":0.0297,"skill":0.0221,"coverage":0.9995,"random":0.0078,"tracks":[],"private_eq":0.0079,"combined":0},"23":{"raw":0.4973,"skill":0,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.4148,"skill":0.2197,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2857,"skill":0.0476,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.2643,"combined":0.0088},"26":{"raw":0.7942,"skill":0.7255,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.6041,"skill":0.472,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6922,"skill":0.3844,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7581,"combined":0.5116},"29":{"raw":0.6251,"skill":0.5001,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5463,"combined":0.3891},"30":{"raw":0.2945,"skill":0.1493,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.398,"random":0.25},{"track":"10-choice","raw":0.1911,"random":0.1}],"private_eq":0.3226,"combined":0.1705},"31":{"raw":0.1134,"skill":0.0344,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0892,"combined":0},"32":{"raw":0.5691,"skill":0.3149,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.389,"combined":0.0184},"33":{"raw":0,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0131,"combined":0},"36":{"raw":0.3476,"skill":0.2619,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.355,"combined":0.263},"37":{"raw":0.4076,"skill":0.2569,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.317,"combined":0.1345},"38":{"raw":0.0969,"skill":0.068,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.1357,"combined":0.0988},"39":{"raw":0.7247,"skill":0.5951,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.5094,"combined":0.2711},"40":{"raw":0.2604,"skill":0.0485,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2456,"headline":false},{"track":"A · English","score":0.2604,"headline":true},{"track":"C · Arabic pairs","score":0.5,"headline":false},{"track":"C · English pairs","score":0.495,"headline":false}],"private_eq":0.2696,"combined":0.0505},"41":{"raw":0.4251,"skill":0.1376,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.5884,"combined":0.3765},"42":{"raw":0.5968,"skill":0.2156,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.5541,"combined":0.1235},"43":{"raw":0.3842,"skill":0.023,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.3697,"combined":0},"44":{"raw":0.5588,"skill":0.1176,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"45":{"raw":0.1257,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3156,"skill":0.0068,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4861,"combined":0.2466},"56":{"raw":0.544,"skill":0.088,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5851,"combined":0.1617},"57":{"raw":0.1961,"skill":0.0958,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1261,"combined":0.0068},"58":{"raw":0.3817,"skill":0.1038,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3523,"combined":0.0514},"59":{"raw":0.0121,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.5683,"skill":0.1366,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6494,"combined":0.2917},"62":{"raw":0.3469,"skill":0.1292,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3351,"combined":0.1043},"64":{"raw":0.411,"skill":0.2637,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.3864,"combined":0.2252}},"coverage":0.9992,"pending":0.0008,"notes":["Encoder; 8 self-declared training matches counted wrong."],"scores":{"balanced_raw":40.77,"balanced_skill":17.98,"breadth_skill":20.88,"public_skill":22.45},"frozen_scores":{"balanced_raw":40.77,"balanced_skill":22.45,"breadth_skill":20.88},"categories":[{"id":"knowledge","raw":0.3281,"skill":0.1164,"combined":0.0786,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.4232,"skill":0.2398,"combined":0.1962,"coverage":0.9974,"pending":0,"n":9},{"id":"retrieval","raw":0.5954,"skill":0.4218,"combined":0.3774,"coverage":0.9996,"pending":0,"n":6},{"id":"tools","raw":0.2855,"skill":0.186,"combined":0.1823,"coverage":0.9986,"pending":0,"n":5},{"id":"arts","raw":0.4207,"skill":0.139,"combined":0.2457,"coverage":0.9999,"pending":0,"n":6}],"v03":{"engine":"bekko-v0-400m","pub":22.45,"same_skills":21.55,"new_domains":9.05,"v":17.98,"lab":false,"jev":false,"rank_v":202,"rank_pub":206,"name":"Bekko System One v0 400M","eq":{"S":20.75,"O":13.83},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":12.45,"new_domains":5.53}}},{"engine":"torchcast-decision-27b","name":"Torchcast Decision 27B","long_name":"Torchcast Decision 27B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27800000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/torchcast-ai/torchcast-decision-27b","code_url":null,"base_model":"Qwen/Qwen3.8-27B","base_url":"https://huggingface.co/Qwen/Qwen3.8-27B","weights_repo":null,"served_checkpoint":null,"org":{"id":"torchcast-ai","name":"Torchcast AI","logo":"data/orgs/torchcast-ai.png"}},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7866,"conf":0.8101,"ece":0.0235,"brier":0.2841,"over95":0.0075,"n":72594,"benchmarks":32,"rel":[{"lo":0,"w":0.0005,"conf":0.0888,"acc":0.0769},{"lo":0.1,"w":0.0067,"conf":0.1557,"acc":0.1374},{"lo":0.2,"w":0.0206,"conf":0.2601,"acc":0.2257},{"lo":0.3,"w":0.0381,"conf":0.3491,"acc":0.2833},{"lo":0.4,"w":0.0454,"conf":0.45,"acc":0.4219},{"lo":0.5,"w":0.0993,"conf":0.5504,"acc":0.5174},{"lo":0.6,"w":0.0778,"conf":0.6496,"acc":0.5877},{"lo":0.7,"w":0.0837,"conf":0.7503,"acc":0.7056},{"lo":0.8,"w":0.1051,"conf":0.8537,"acc":0.8202},{"lo":0.9,"w":0.5228,"conf":0.9797,"acc":0.973}],"areas":{"knowledge":{"ece":0.0729,"acc":0.6945,"conf":0.7674},"language":{"ece":0.0075,"acc":0.8926,"conf":0.891},"retrieval":{"ece":0.0188,"acc":0.8106,"conf":0.813},"tools":{"ece":0.0398,"acc":0.957,"conf":0.9333},"arts":{"ece":0.0372,"acc":0.6768,"conf":0.682}}},"gaps":null,"latency":{"median":99.9,"p95":1240,"mean":259},"results":{"1":{"score":0.977,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":178.2},"2":{"score":0.6837,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1880.8},"3":{"score":0.8524,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3165},"4":{"score":0.9107,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":148.7},"5":{"score":0.9347,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":199.6},"6":{"score":0.7975,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":474.4},"9":{"score":0.8977,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":999.7},"10":{"score":0.734,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":257.8},"11":{"score":0.8646,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":2498.2},"12":{"score":0.7875,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31.5},"20":{"score":0.9544,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.5},"21":{"score":0.626,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":18.7},"22":{"score":0.5075,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2289.4},"23":{"score":0.7018,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":63.3},"24":{"score":0.8648,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":24.7},"25":{"score":0.5153,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":36.9},"26":{"score":0.9933,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":18.7},"27":{"score":0.971,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":18.8},"28":{"score":0.9384,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":12.6},"29":{"score":0.9762,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":71.8},"30":{"score":0.6467,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1806,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":138.5},"32":{"score":0.6729,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":312.4},"33":{"score":0.1503,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":249.5},"36":{"score":0.5073,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":2440.1},"37":{"score":0.6162,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":115},"38":{"score":0.573,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1239.6},"39":{"score":0.932,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":64.7},"40":{"score":0.5007,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":18.8},"41":{"score":0.8195,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":37.2},"42":{"score":0.862,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":125.2},"43":{"score":0.786,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":31.4},"44":{"score":0.7448,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":31.7},"45":{"score":0.1337,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":56.6},"50":{"score":0.6026,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":223.7},"56":{"score":0.7035,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":241},"57":{"score":0.7039,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":36.8},"58":{"score":0.8072,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":31.5},"59":{"score":0.86,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":182.2},"61":{"score":0.8942,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":126.6},"62":{"score":0.8896,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":169.3},"64":{"score":0.7898,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":37.1}},"benchmarks":{"1":{"raw":0.977,"skill":0.9689,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9451,"combined":0.9261},"2":{"raw":0.6837,"skill":0.6347,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.748,"combined":0.7066},"3":{"raw":0.8524,"skill":0.8496,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.9033,"combined":0.9013},"4":{"raw":0.9107,"skill":0.9096,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8609,"combined":0.8585},"5":{"raw":0.9347,"skill":0.9343,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.7891,"combined":0.7864},"6":{"raw":0.7975,"skill":0.5252,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.8977,"skill":0.8977,"coverage":1,"random":0,"tracks":[],"private_eq":0.75,"combined":0.7481},"10":{"raw":0.734,"skill":0.5574,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8646,"skill":0.8042,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7875,"skill":0.6817,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6328,"combined":0.4446},"20":{"raw":0.9555,"skill":0.911,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.9607,"combined":0.9215},"21":{"raw":0.626,"skill":0.2519,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.637,"combined":0.2666},"22":{"raw":0.5037,"skill":0.4999,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.4722,"combined":0.4629},"23":{"raw":0.7191,"skill":0.4382,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8648,"skill":0.8197,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5153,"skill":0.3537,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5148,"combined":0.3466},"26":{"raw":0.9933,"skill":0.9911,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.971,"skill":0.9613,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9384,"skill":0.8768,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.803,"combined":0.6025},"29":{"raw":0.9762,"skill":0.9683,"coverage":1,"random":0.25,"tracks":[],"private_eq":1,"combined":1.001},"30":{"raw":0.6467,"skill":0.5807,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7589,"random":0.25},{"track":"10-choice","raw":0.5345,"random":0.1}],"private_eq":0.72,"combined":0.6577},"31":{"raw":0.1806,"skill":0.1075,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1767,"combined":0.0939},"32":{"raw":0.6729,"skill":0.48,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.5815,"combined":0.328},"33":{"raw":0.1503,"skill":0.139,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0475,"combined":0.0247},"36":{"raw":0.5073,"skill":0.4426,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.472,"combined":0.3968},"37":{"raw":0.6162,"skill":0.5186,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.7026,"combined":0.6237},"38":{"raw":0.573,"skill":0.5593,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.7436,"combined":0.7334},"39":{"raw":0.932,"skill":0.9,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.7165,"combined":0.5792},"40":{"raw":0.5007,"skill":0.3577,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3974,"headline":false},{"track":"A · English","score":0.5007,"headline":true},{"track":"C · Arabic pairs","score":0.875,"headline":false},{"track":"C · English pairs","score":0.96,"headline":false}],"private_eq":0.5466,"combined":0.411},"41":{"raw":0.8195,"skill":0.7293,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.4435,"combined":0.1567},"42":{"raw":0.862,"skill":0.7315,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.951,"combined":0.9046},"43":{"raw":0.786,"skill":0.6604,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6822,"combined":0.491},"44":{"raw":0.7448,"skill":0.4896,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7565,"combined":0.5084},"45":{"raw":0.1337,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6026,"skill":0.4233,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4615,"combined":0.2104},"56":{"raw":0.7035,"skill":0.407,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.722,"combined":0.4386},"57":{"raw":0.7039,"skill":0.667,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.582,"combined":0.5255},"58":{"raw":0.8072,"skill":0.7205,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.505,"combined":0.2753},"59":{"raw":0.86,"skill":0.7097,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6611,"combined":0.2902},"61":{"raw":0.8942,"skill":0.7884,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8317,"combined":0.6605},"62":{"raw":0.8896,"skill":0.8528,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.7107,"combined":0.6108},"64":{"raw":0.7898,"skill":0.7372,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.8006,"combined":0.7489}},"coverage":1,"pending":0,"notes":["Index results guided checkpoint selection (author-declared)."],"scores":{"balanced_raw":73.64,"balanced_skill":61.49,"breadth_skill":63.86,"public_skill":65.1},"frozen_scores":{"balanced_raw":73.64,"balanced_skill":65.1,"breadth_skill":63.86},"categories":[{"id":"knowledge","raw":0.575,"skill":0.4666,"combined":0.3892,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.8009,"skill":0.7204,"combined":0.5726,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7632,"skill":0.6754,"combined":0.6322,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.8641,"skill":0.8458,"combined":0.7886,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6995,"skill":0.5436,"combined":0.5221,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"torchcast-decision-27b","pub":65.1,"same_skills":58.13,"new_domains":52.19,"v":61.49,"lab":false,"jev":false,"rank_v":42,"rank_pub":39,"name":"Torchcast Decision 27B","eq":{"S":57.75,"O":67.1},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":34.65,"new_domains":26.84}}},{"engine":"kas-4b","name":"Kas 4B","long_name":"Kas 4B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4020000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/kpiya/kas-4b","code_url":null,"base_model":"Qwen/Qwen3-4B","base_url":"https://huggingface.co/Qwen/Qwen3-4B","weights_repo":null,"served_checkpoint":null},"completed":100834,"counts":{"ok":100833,"unsupported":1,"error":0},"lifted":false,"calibration":{"acc":0.6644,"conf":0.7611,"ece":0.0984,"brier":0.4632,"over95":0.0308,"n":34661,"benchmarks":31,"rel":[{"lo":0,"w":0.0138,"conf":0.0915,"acc":0.0219},{"lo":0.1,"w":0.0448,"conf":0.133,"acc":0.1016},{"lo":0.2,"w":0.0213,"conf":0.2515,"acc":0.2831},{"lo":0.3,"w":0.0243,"conf":0.3587,"acc":0.3675},{"lo":0.4,"w":0.044,"conf":0.4575,"acc":0.4115},{"lo":0.5,"w":0.0974,"conf":0.5527,"acc":0.525},{"lo":0.6,"w":0.0909,"conf":0.6501,"acc":0.5311},{"lo":0.7,"w":0.1094,"conf":0.7514,"acc":0.5964},{"lo":0.8,"w":0.1267,"conf":0.8532,"acc":0.6749},{"lo":0.9,"w":0.4275,"conf":0.9741,"acc":0.8803}],"areas":{"knowledge":{"ece":0.1301,"acc":0.6379,"conf":0.768},"language":{"ece":0.1513,"acc":0.6853,"conf":0.8366},"retrieval":{"ece":0.0769,"acc":0.6795,"conf":0.6648},"tools":{"ece":0.0541,"acc":0.8039,"conf":0.7788},"arts":{"ece":0.0944,"acc":0.5843,"conf":0.6776}}},"gaps":null,"latency":{"median":69.2,"p95":1242.3,"mean":284.5},"results":{"1":{"score":0.9221,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":234.2},"2":{"score":0.5584,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":2136.3},"3":{"score":0.5925,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":3575.4},"4":{"score":0.594,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":139.9},"5":{"score":0.7335,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":222.1},"6":{"score":0.6924,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":884.8},"9":{"score":0.1023,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":2320.8},"10":{"score":0.6754,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":246.5},"11":{"score":0.5028,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":4628.1},"12":{"score":0.5454,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":45.3},"20":{"score":0.9698,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.7},"21":{"score":0.6039,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":36.2},"22":{"score":0.004,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2389.5},"23":{"score":0.6466,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":99.9},"24":{"score":0.6864,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":39.4},"25":{"score":0.3724,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":51.4},"26":{"score":0.9588,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":36.8},"27":{"score":0.8891,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":37.2},"28":{"score":0.6732,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":36.2},"29":{"score":0.7779,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":58.2},"30":{"score":0.2582,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1994,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":158.4},"32":{"score":0.5904,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":200.5},"33":{"score":0.2958,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":496.5},"36":{"score":0.3396,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":3171.5},"37":{"score":0.5515,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":75.7},"38":{"score":0.1046,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2519.9},"39":{"score":0.8235,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":83.2},"40":{"score":0.3082,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":36.5},"41":{"score":0.5281,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":46.1},"42":{"score":0.7165,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":96.6},"43":{"score":0.8789,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":45.2},"44":{"score":0.8528,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":45.4},"45":{"score":0.0898,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":58.4},"50":{"score":0.6999,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":167.8},"56":{"score":0.539,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":404.3},"57":{"score":0.4229,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":53.5},"58":{"score":0.57,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":46.8},"59":{"score":0.5762,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":153.1},"61":{"score":0.6312,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":88.1},"62":{"score":0.454,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":153.6},"64":{"score":0.5038,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":50.8}},"benchmarks":{"1":{"raw":0.9221,"skill":0.8948,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.8903,"combined":0.8512},"2":{"raw":0.5576,"skill":0.489,"coverage":0.9985,"random":0.1341,"tracks":[],"private_eq":0.7118,"combined":0.6644},"3":{"raw":0.5925,"skill":0.5846,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.5774,"combined":0.5653},"4":{"raw":0.594,"skill":0.5888,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.7313,"combined":0.7257},"5":{"raw":0.7335,"skill":0.7319,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.4053,"combined":0.3958},"6":{"raw":0.6924,"skill":0.2788,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1023,"skill":0.1023,"coverage":1,"random":0,"tracks":[],"private_eq":0.0625,"combined":0.0527},"10":{"raw":0.6754,"skill":0.4599,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.5028,"skill":0.2809,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5454,"skill":0.3191,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.585,"combined":0.3722},"20":{"raw":0.968,"skill":0.936,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.9321,"combined":0.8637},"21":{"raw":0.6039,"skill":0.2078,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5972,"combined":0.1861},"22":{"raw":0.0025,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0119,"combined":0},"23":{"raw":0.638,"skill":0.276,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6864,"skill":0.5819,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3724,"skill":0.1633,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3267,"combined":0.0929},"26":{"raw":0.9588,"skill":0.9451,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8891,"skill":0.8521,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.6732,"skill":0.3464,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6734,"combined":0.3403},"29":{"raw":0.7779,"skill":0.7039,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.7591,"combined":0.6761},"30":{"raw":0.2582,"skill":0.1044,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3578,"random":0.25},{"track":"10-choice","raw":0.1585,"random":0.1}],"private_eq":0.3684,"combined":0.2266},"31":{"raw":0.1994,"skill":0.128,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1364,"combined":0.0495},"32":{"raw":0.5904,"skill":0.3488,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.4804,"combined":0.1654},"33":{"raw":0.2958,"skill":0.2865,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.3534,"combined":0.3383},"36":{"raw":0.3396,"skill":0.253,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.3877,"combined":0.3004},"37":{"raw":0.5515,"skill":0.4374,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.5511,"combined":0.4315},"38":{"raw":0.1046,"skill":0.076,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.031,"combined":0},"39":{"raw":0.8235,"skill":0.7404,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.669,"combined":0.5086},"40":{"raw":0.3082,"skill":0.1101,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.3108,"headline":false},{"track":"A · English","score":0.3082,"headline":true},{"track":"C · Arabic pairs","score":0.605,"headline":false},{"track":"C · English pairs","score":0.72,"headline":false}],"private_eq":0.2906,"combined":0.0778},"41":{"raw":0.5281,"skill":0.2922,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.6204,"combined":0.4251},"42":{"raw":0.7165,"skill":0.4484,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.8919,"combined":0.7883},"43":{"raw":0.8789,"skill":0.8079,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.4634,"combined":0.1399},"44":{"raw":0.8528,"skill":0.7056,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8518,"combined":0.7012},"45":{"raw":0.0898,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.6999,"skill":0.5644,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4763,"combined":0.2322},"56":{"raw":0.539,"skill":0.078,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6707,"combined":0.3348},"57":{"raw":0.4229,"skill":0.3509,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5069,"combined":0.44},"58":{"raw":0.57,"skill":0.3767,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4094,"combined":0.1351},"59":{"raw":0.5762,"skill":0.1213,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5778,"combined":0.1155},"61":{"raw":0.6312,"skill":0.2624,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.613,"combined":0.2181},"62":{"raw":0.454,"skill":0.272,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6194,"combined":0.4877},"64":{"raw":0.5038,"skill":0.3797,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.4478,"combined":0.3028}},"coverage":0.9481,"pending":0.0519,"notes":["Public board run by the lab (results dataset unavailable)."],"scores":{"balanced_raw":53.19,"balanced_skill":34,"breadth_skill":38.03,"public_skill":38.35},"frozen_scores":{"balanced_raw":53.19,"balanced_skill":38.35,"breadth_skill":38.03},"categories":[{"id":"knowledge","raw":0.4691,"skill":0.3222,"combined":0.2621,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5476,"skill":0.3507,"combined":0.3765,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.564,"skill":0.404,"combined":0.4077,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5429,"skill":0.4886,"combined":0.5379,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.5694,"skill":0.394,"combined":0.3157,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"kas-4b","pub":38.35,"same_skills":38.69,"new_domains":20.43,"v":34,"lab":false,"jev":false,"rank_v":156,"rank_pub":160,"name":"Kas 4B","eq":{"S":38.09,"O":27.88},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":22.85,"new_domains":11.15}}},{"engine":"lev1-3.7b","name":"Lev1 3.7B","long_name":"Lev1 3.7B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":3700000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/theaviv/Lev1-3.7B","code_url":null,"base_model":"IFM/K2-Horizon-3.7B","base_url":"https://huggingface.co/IFM/K2-Horizon-3.7B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6576,"conf":0.7153,"ece":0.0578,"brier":0.4474,"over95":0.0136,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0026,"conf":0.0851,"acc":0},{"lo":0.1,"w":0.0262,"conf":0.1526,"acc":0.1224},{"lo":0.2,"w":0.0361,"conf":0.2514,"acc":0.1983},{"lo":0.3,"w":0.0536,"conf":0.3476,"acc":0.3028},{"lo":0.4,"w":0.0677,"conf":0.4514,"acc":0.3697},{"lo":0.5,"w":0.148,"conf":0.5485,"acc":0.5127},{"lo":0.6,"w":0.1193,"conf":0.6486,"acc":0.6003},{"lo":0.7,"w":0.1065,"conf":0.7507,"acc":0.6837},{"lo":0.8,"w":0.1104,"conf":0.8518,"acc":0.747},{"lo":0.9,"w":0.3296,"conf":0.9717,"acc":0.9198}],"areas":{"knowledge":{"ece":0.0865,"acc":0.557,"conf":0.6435},"language":{"ece":0.052,"acc":0.7256,"conf":0.7757},"retrieval":{"ece":0.065,"acc":0.7702,"conf":0.8303},"tools":{"ece":0.0474,"acc":0.8872,"conf":0.9322},"arts":{"ece":0.036,"acc":0.5287,"conf":0.5427}}},"gaps":null,"latency":{"median":28.8,"p95":169.9,"mean":100.9},"results":{"1":{"score":0.941,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":41.8},"2":{"score":0.6343,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1494.2},"3":{"score":0.8012,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":452.9},"4":{"score":0.809,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":89.7},"5":{"score":0.9234,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":169.5},"6":{"score":0.7928,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":100.6},"9":{"score":0.2614,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":205.3},"10":{"score":0.6714,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":56.4},"11":{"score":0.7826,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":293.7},"12":{"score":0.5535,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":26},"20":{"score":0.8148,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26},"21":{"score":0.6298,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":15.9},"22":{"score":0.1475,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":612},"23":{"score":0.566,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26},"24":{"score":0.6714,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":17.2},"25":{"score":0.3214,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":25.9},"26":{"score":0.9419,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":15.9},"27":{"score":0.8848,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":16},"28":{"score":0.7056,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":15.9},"29":{"score":0.7631,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":35.9},"30":{"score":0.2805,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1498,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":75.1},"32":{"score":0.6197,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":90.9},"33":{"score":0.24,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":70.4},"36":{"score":0.4428,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1952.3},"37":{"score":0.4243,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":26.1},"38":{"score":0.1506,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1503.3},"39":{"score":0.8323,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":26},"40":{"score":0.3818,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":16},"41":{"score":0.4655,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":26},"42":{"score":0.7582,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":36.1},"43":{"score":0.5193,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":21.9},"44":{"score":0.598,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":25.7},"45":{"score":0.0978,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":26.4},"50":{"score":0.4368,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":70.1},"56":{"score":0.505,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":70.6},"57":{"score":0.3792,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":25.9},"58":{"score":0.6174,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":19.9},"59":{"score":0.7367,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":41.6},"61":{"score":0.6512,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":26.3},"62":{"score":0.8628,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":41.5},"64":{"score":0.6553,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":25.7}},"benchmarks":{"1":{"raw":0.941,"skill":0.9203,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9102,"combined":0.8784},"2":{"raw":0.6343,"skill":0.5777,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7127,"combined":0.6654},"3":{"raw":0.8012,"skill":0.7974,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.7709,"combined":0.7648},"4":{"raw":0.809,"skill":0.8065,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8539,"combined":0.8513},"5":{"raw":0.9234,"skill":0.9229,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.6966,"combined":0.6923},"6":{"raw":0.7928,"skill":0.5142,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.2614,"skill":0.2614,"coverage":1,"random":0,"tracks":[],"private_eq":0.22,"combined":0.212},"10":{"raw":0.6714,"skill":0.4532,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7826,"skill":0.6856,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.5535,"skill":0.3312,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.5374,"combined":0.3001},"20":{"raw":0.8158,"skill":0.6317,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8606,"combined":0.719},"21":{"raw":0.6298,"skill":0.2595,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6022,"combined":0.1962},"22":{"raw":0.1375,"skill":0.1308,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.1858,"combined":0.171},"23":{"raw":0.5855,"skill":0.1711,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6714,"skill":0.5619,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3214,"skill":0.0952,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3163,"combined":0.0789},"26":{"raw":0.9419,"skill":0.9225,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8848,"skill":0.8464,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7056,"skill":0.4112,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6185,"combined":0.2292},"29":{"raw":0.7631,"skill":0.6841,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6711,"combined":0.5574},"30":{"raw":0.2805,"skill":0.131,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.3738,"random":0.25},{"track":"10-choice","raw":0.1873,"random":0.1}],"private_eq":0.4101,"combined":0.2777},"31":{"raw":0.1498,"skill":0.074,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1679,"combined":0.0842},"32":{"raw":0.6197,"skill":0.3954,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.4131,"combined":0.0572},"33":{"raw":0.24,"skill":0.2299,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.2881,"combined":0.2714},"36":{"raw":0.4428,"skill":0.3697,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.3976,"combined":0.3117},"37":{"raw":0.4243,"skill":0.2779,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.3278,"combined":0.1482},"38":{"raw":0.1506,"skill":0.1234,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.2823,"combined":0.2518},"39":{"raw":0.8323,"skill":0.7533,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.7244,"combined":0.591},"40":{"raw":0.3818,"skill":0.2047,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4267,"headline":false},{"track":"A · English","score":0.3818,"headline":true},{"track":"C · Arabic pairs","score":0.725,"headline":false},{"track":"C · English pairs","score":0.9,"headline":false}],"private_eq":0.4154,"combined":0.2403},"41":{"raw":0.4655,"skill":0.1983,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.5951,"combined":0.3867},"42":{"raw":0.7582,"skill":0.5296,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.8664,"combined":0.7381},"43":{"raw":0.5193,"skill":0.2373,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6248,"combined":0.3989},"44":{"raw":0.598,"skill":0.196,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6718,"combined":0.3371},"45":{"raw":0.0978,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4368,"skill":0.1825,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4713,"combined":0.2248},"56":{"raw":0.505,"skill":0.01,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5081,"combined":0.0059},"57":{"raw":0.3792,"skill":0.3018,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.2901,"combined":0.1934},"58":{"raw":0.6174,"skill":0.4454,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3981,"combined":0.1185},"59":{"raw":0.7367,"skill":0.4541,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6551,"combined":0.2777},"61":{"raw":0.6512,"skill":0.3024,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7644,"combined":0.5244},"62":{"raw":0.8628,"skill":0.8171,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6955,"combined":0.5903},"64":{"raw":0.6553,"skill":0.5691,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.7009,"combined":0.6228}},"coverage":1,"pending":0,"notes":["Base run by the lab (stock K2-Horizon-3.7B)."],"scores":{"balanced_raw":57.49,"balanced_skill":35.41,"breadth_skill":40.45,"public_skill":42.68},"frozen_scores":{"balanced_raw":57.49,"balanced_skill":42.68,"breadth_skill":40.45},"categories":[{"id":"knowledge","raw":0.4127,"skill":0.2307,"combined":0.201,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6051,"skill":0.4433,"combined":0.4184,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.635,"skill":0.4711,"combined":0.4401,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7128,"skill":0.6884,"combined":0.637,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5434,"skill":0.3241,"combined":0.3868,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"lev1-3.7b","pub":42.68,"same_skills":41.45,"new_domains":19.89,"v":35.41,"lab":false,"jev":false,"rank_v":147,"rank_pub":144,"name":"Lev1 3.7B","eq":{"S":40.88,"O":27.21},"rank_tie":34,"tied":true,"parts":{"public":0,"same_skills":24.53,"new_domains":10.88}}},{"engine":"decisio-0.4.0-qwen3.6-35b-a3b","name":"decisio 0.4.0 (Qwen3.6-35B-A3B)","long_name":"decisio 0.4.0 (Qwen3.6-35B-A3B)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":35950000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/aminry/decisio","base_model":"Qwen/Qwen3.6-35B-A3B","base_url":"https://huggingface.co/Qwen/Qwen3.6-35B-A3B","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6792,"conf":0.7321,"ece":0.0554,"brier":0.4192,"over95":0.0135,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0197,"conf":0.0632,"acc":0.0707},{"lo":0.1,"w":0.0179,"conf":0.1478,"acc":0.2073},{"lo":0.2,"w":0.0215,"conf":0.2503,"acc":0.1744},{"lo":0.3,"w":0.048,"conf":0.3522,"acc":0.2856},{"lo":0.4,"w":0.0619,"conf":0.4511,"acc":0.3771},{"lo":0.5,"w":0.1257,"conf":0.5499,"acc":0.4877},{"lo":0.6,"w":0.117,"conf":0.65,"acc":0.5983},{"lo":0.7,"w":0.1107,"conf":0.753,"acc":0.6617},{"lo":0.8,"w":0.117,"conf":0.8519,"acc":0.7918},{"lo":0.9,"w":0.3606,"conf":0.9701,"acc":0.932}],"areas":{"knowledge":{"ece":0.0881,"acc":0.61,"conf":0.6981},"language":{"ece":0.0569,"acc":0.7673,"conf":0.8217},"retrieval":{"ece":0.0531,"acc":0.6646,"conf":0.6923},"tools":{"ece":0.0561,"acc":0.922,"conf":0.8763},"arts":{"ece":0.0758,"acc":0.5387,"conf":0.591}}},"gaps":null,"latency":{"median":70.5,"p95":261.2,"mean":112.9},"results":{"1":{"score":0.9604,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":140.1},"2":{"score":0.6251,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1233.4},"3":{"score":0.8169,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":221.8},"4":{"score":0.7461,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":46.3},"5":{"score":0.8223,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":51.8},"6":{"score":0.799,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":149.6},"9":{"score":0.3977,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":691.2},"10":{"score":0.4661,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":82.3},"11":{"score":0.7326,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":689.8},"12":{"score":0.686,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":39.5},"20":{"score":0.9034,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.6},"21":{"score":0.6081,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":38.1},"22":{"score":0.1455,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":174.7},"23":{"score":0.615,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":41.1},"24":{"score":0.8368,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":39.2},"25":{"score":0.5102,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":41.6},"26":{"score":0.9907,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":39.2},"27":{"score":0.9608,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":38.5},"28":{"score":0.8082,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":38.4},"29":{"score":0.9397,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":41.4},"30":{"score":0.351,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.162,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":77.7},"32":{"score":0.5811,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":56.9},"33":{"score":0.2315,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":332.9},"36":{"score":0.4435,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1301.9},"37":{"score":0.4495,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":73.5},"38":{"score":0.1743,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2277.5},"39":{"score":0.803,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":124.1},"40":{"score":0.4743,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":69},"41":{"score":0.4897,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":39.8},"42":{"score":0.7718,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":43},"43":{"score":0.6246,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":70.9},"44":{"score":0.6218,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.8},"45":{"score":0.1138,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":41},"50":{"score":0.4469,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":81.3},"56":{"score":0.7525,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":372.4},"57":{"score":0.6126,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":72.4},"58":{"score":0.6942,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":72},"59":{"score":0.7023,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":76.7},"61":{"score":0.6995,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":74.1},"62":{"score":0.7139,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":77.4},"64":{"score":0.6875,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":39.7}},"benchmarks":{"1":{"raw":0.9604,"skill":0.9466,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9501,"combined":0.9329},"2":{"raw":0.6251,"skill":0.5671,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7166,"combined":0.67},"3":{"raw":0.8169,"skill":0.8134,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.7964,"combined":0.7911},"4":{"raw":0.7461,"skill":0.7428,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8342,"combined":0.8312},"5":{"raw":0.8223,"skill":0.8212,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.5726,"combined":0.5661},"6":{"raw":0.799,"skill":0.5287,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3977,"skill":0.3977,"coverage":1,"random":0,"tracks":[],"private_eq":0.2925,"combined":0.2854},"10":{"raw":0.4661,"skill":0.1116,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7326,"skill":0.6133,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.686,"skill":0.5297,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6617,"combined":0.4884},"20":{"raw":0.9044,"skill":0.8087,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.9178,"combined":0.8347},"21":{"raw":0.6081,"skill":0.2161,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6022,"combined":0.1962},"22":{"raw":0.1423,"skill":0.1356,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.1167,"combined":0.1005},"23":{"raw":0.6004,"skill":0.2009,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8368,"skill":0.7824,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5102,"skill":0.3469,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4659,"combined":0.2807},"26":{"raw":0.9907,"skill":0.9876,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9608,"skill":0.9477,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8082,"skill":0.6164,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7532,"combined":0.5017},"29":{"raw":0.9397,"skill":0.9196,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.9756,"combined":0.9681},"30":{"raw":0.351,"skill":0.2184,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.4556,"random":0.25},{"track":"10-choice","raw":0.2464,"random":0.1}],"private_eq":0.5527,"combined":0.4526},"31":{"raw":0.162,"skill":0.0873,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1738,"combined":0.0907},"32":{"raw":0.5811,"skill":0.334,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.5574,"combined":0.2892},"33":{"raw":0.2315,"skill":0.2213,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.2581,"combined":0.2406},"36":{"raw":0.4435,"skill":0.3704,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.4334,"combined":0.3527},"37":{"raw":0.4495,"skill":0.3095,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.5345,"combined":0.4104},"38":{"raw":0.1743,"skill":0.1479,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.331,"combined":0.3027},"39":{"raw":0.803,"skill":0.7103,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.7099,"combined":0.5694},"40":{"raw":0.4743,"skill":0.3237,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4255,"headline":false},{"track":"A · English","score":0.4743,"headline":true},{"track":"C · Arabic pairs","score":0.765,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"private_eq":0.4378,"combined":0.2694},"41":{"raw":0.4897,"skill":0.2346,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.6156,"combined":0.4178},"42":{"raw":0.7718,"skill":0.556,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.8428,"combined":0.6917},"43":{"raw":0.6246,"skill":0.4043,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6431,"combined":0.4283},"44":{"raw":0.6218,"skill":0.2436,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5924,"combined":0.1764},"45":{"raw":0.1138,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4469,"skill":0.1973,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4861,"combined":0.2466},"56":{"raw":0.7525,"skill":0.505,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7861,"combined":0.5683},"57":{"raw":0.6126,"skill":0.5643,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4874,"combined":0.4178},"58":{"raw":0.6942,"skill":0.5567,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4599,"combined":0.2091},"59":{"raw":0.7023,"skill":0.3827,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6284,"combined":0.2217},"61":{"raw":0.6995,"skill":0.399,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7448,"combined":0.4847},"62":{"raw":0.7139,"skill":0.6185,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6955,"combined":0.5903},"64":{"raw":0.6875,"skill":0.6094,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.7086,"combined":0.6326}},"coverage":1,"pending":0,"notes":["Inference technique on frozen weights; prompt partly chosen on index rows (author-declared)."],"scores":{"balanced_raw":61.23,"balanced_skill":50.82,"breadth_skill":47.15,"public_skill":48.43},"frozen_scores":{"balanced_raw":61.23,"balanced_skill":48.43,"breadth_skill":47.15},"categories":[{"id":"knowledge","raw":0.4882,"skill":0.3342,"combined":0.3119,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6488,"skill":0.5008,"combined":0.5025,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6539,"skill":0.5356,"combined":0.5399,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7166,"skill":0.6843,"combined":0.6694,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5649,"skill":0.3613,"combined":0.4021,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"decisio-0.4.0-qwen3.6-35b-a3b","pub":48.43,"same_skills":49.15,"new_domains":41.63,"v":50.82,"lab":false,"jev":false,"rank_v":92,"rank_pub":125,"name":"decisio 0.4.0 (Qwen3.6-35B-A3B)","eq":{"S":48.67,"O":54.06},"rank_tie":20,"tied":true,"parts":{"public":0,"same_skills":29.2,"new_domains":21.62}}},{"engine":"decisio-0.4.0-gemma-4-12b","name":"decisio 0.4.0 (Gemma 4 12B)","long_name":"decisio 0.4.0 (Gemma 4 12B)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":11960000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/aminry/decisio","base_model":"google/gemma-4-12B-it","base_url":"https://huggingface.co/google/gemma-4-12B-it","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6742,"conf":0.7773,"ece":0.1031,"brier":0.4554,"over95":0.0211,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0126,"conf":0.0727,"acc":0.0077},{"lo":0.1,"w":0.0195,"conf":0.1477,"acc":0.1124},{"lo":0.2,"w":0.0186,"conf":0.249,"acc":0.2101},{"lo":0.3,"w":0.0363,"conf":0.3521,"acc":0.3224},{"lo":0.4,"w":0.0471,"conf":0.4536,"acc":0.28},{"lo":0.5,"w":0.067,"conf":0.5525,"acc":0.4858},{"lo":0.6,"w":0.0901,"conf":0.6515,"acc":0.5638},{"lo":0.7,"w":0.1173,"conf":0.7537,"acc":0.6168},{"lo":0.8,"w":0.1618,"conf":0.8544,"acc":0.6786},{"lo":0.9,"w":0.4297,"conf":0.9596,"acc":0.8787}],"areas":{"knowledge":{"ece":0.1055,"acc":0.6047,"conf":0.7086},"language":{"ece":0.0844,"acc":0.7833,"conf":0.8675},"retrieval":{"ece":0.2455,"acc":0.5918,"conf":0.8168},"tools":{"ece":0.0565,"acc":0.9413,"conf":0.8867},"arts":{"ece":0.1323,"acc":0.5363,"conf":0.6686}}},"gaps":null,"latency":{"median":37.8,"p95":275.9,"mean":111.9},"results":{"1":{"score":0.9616,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":123.6},"2":{"score":0.6222,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":1129.4},"3":{"score":0.8701,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":681.3},"4":{"score":0.7294,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":65.3},"5":{"score":0.8716,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":88.6},"6":{"score":0.8005,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":251.1},"9":{"score":0.4432,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":527.7},"10":{"score":0.0468,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":110.2},"11":{"score":0.7432,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":670.9},"12":{"score":0.7018,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":31},"20":{"score":0.9072,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.6},"21":{"score":0.6172,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":26.6},"22":{"score":0.062,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":460},"23":{"score":0.63,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":32.6},"24":{"score":0.7828,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":27.5},"25":{"score":0.3878,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":32.6},"26":{"score":0.9781,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":26.9},"27":{"score":0.9497,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":27.1},"28":{"score":0.7301,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":26.5},"29":{"score":0.867,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":35.3},"30":{"score":0.3874,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1648,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":71.8},"32":{"score":0.5811,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":77.6},"33":{"score":0.2636,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":224.5},"36":{"score":0.4711,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1449.1},"37":{"score":0.5354,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":42.4},"38":{"score":0.1552,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":1676.6},"39":{"score":0.9048,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":74.7},"40":{"score":0.4273,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":27},"41":{"score":0.5913,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":31.3},"42":{"score":0.8128,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":32.8},"43":{"score":0.6544,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":31.1},"44":{"score":0.6344,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":28.3},"45":{"score":0.1058,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":35.9},"50":{"score":0.4648,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":88.9},"56":{"score":0.813,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":249.7},"57":{"score":0.5497,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":32.4},"58":{"score":0.6906,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":30.2},"59":{"score":0.696,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":47.3},"61":{"score":0.6933,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":50.4},"62":{"score":0.7612,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":48.3},"64":{"score":0.6326,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":31}},"benchmarks":{"1":{"raw":0.9616,"skill":0.9482,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9501,"combined":0.9329},"2":{"raw":0.6222,"skill":0.5637,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7281,"combined":0.6834},"3":{"raw":0.8701,"skill":0.8676,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.8422,"combined":0.8383},"4":{"raw":0.7294,"skill":0.7259,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8441,"combined":0.8413},"5":{"raw":0.8716,"skill":0.8708,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.6621,"combined":0.6572},"6":{"raw":0.8005,"skill":0.5322,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.4432,"skill":0.4432,"coverage":1,"random":0,"tracks":[],"private_eq":0.465,"combined":0.4599},"10":{"raw":0.0468,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7432,"skill":0.6287,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7018,"skill":0.5533,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6566,"combined":0.4807},"20":{"raw":0.91,"skill":0.8199,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8464,"combined":0.6903},"21":{"raw":0.6172,"skill":0.2344,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.642,"combined":0.2768},"22":{"raw":0.0599,"skill":0.0526,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.1273,"combined":0.1113},"23":{"raw":0.6235,"skill":0.247,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7828,"skill":0.7104,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3878,"skill":0.1837,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4067,"combined":0.2008},"26":{"raw":0.9781,"skill":0.9708,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9497,"skill":0.9329,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7301,"skill":0.4602,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6509,"combined":0.2948},"29":{"raw":0.867,"skill":0.8227,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.7738,"combined":0.6959},"30":{"raw":0.3874,"skill":0.2655,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.5155,"random":0.25},{"track":"10-choice","raw":0.2593,"random":0.1}],"private_eq":0.5509,"combined":0.4504},"31":{"raw":0.1648,"skill":0.0903,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1777,"combined":0.095},"32":{"raw":0.5811,"skill":0.334,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.5574,"combined":0.2892},"33":{"raw":0.2636,"skill":0.2538,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.3132,"combined":0.2971},"36":{"raw":0.4711,"skill":0.4017,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.4476,"combined":0.3689},"37":{"raw":0.5354,"skill":0.4173,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.5438,"combined":0.4222},"38":{"raw":0.1552,"skill":0.1282,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.249,"combined":0.2171},"39":{"raw":0.9048,"skill":0.86,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.7132,"combined":0.5743},"40":{"raw":0.4273,"skill":0.2633,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4985,"headline":false},{"track":"A · English","score":0.4273,"headline":true},{"track":"C · Arabic pairs","score":0.86,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"private_eq":0.4052,"combined":0.227},"41":{"raw":0.5913,"skill":0.387,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.555,"combined":0.3259},"42":{"raw":0.8128,"skill":0.6358,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.8402,"combined":0.6865},"43":{"raw":0.6544,"skill":0.4517,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6223,"combined":0.3949},"44":{"raw":0.6344,"skill":0.2688,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6983,"combined":0.3907},"45":{"raw":0.1058,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4648,"skill":0.2232,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4911,"combined":0.2539},"56":{"raw":0.813,"skill":0.626,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7947,"combined":0.5857},"57":{"raw":0.5497,"skill":0.4935,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4785,"combined":0.4077},"58":{"raw":0.6906,"skill":0.5515,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4506,"combined":0.1955},"59":{"raw":0.696,"skill":0.3697,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6424,"combined":0.251},"61":{"raw":0.6933,"skill":0.3866,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.742,"combined":0.4791},"62":{"raw":0.7612,"skill":0.6816,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6651,"combined":0.5493},"64":{"raw":0.6326,"skill":0.5408,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.6855,"combined":0.6034}},"coverage":1,"pending":0,"notes":["Inference technique on frozen weights; prompt partly chosen on index rows (author-declared)."],"scores":{"balanced_raw":62.11,"balanced_skill":48.75,"breadth_skill":47.87,"public_skill":49.72},"frozen_scores":{"balanced_raw":62.11,"balanced_skill":49.72,"breadth_skill":47.87},"categories":[{"id":"knowledge","raw":0.4702,"skill":0.3076,"combined":0.2996,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.661,"skill":0.5238,"combined":0.4397,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6861,"skill":0.58,"combined":0.5648,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7453,"skill":0.7162,"combined":0.7071,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5513,"skill":0.353,"combined":0.3871,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"decisio-0.4.0-gemma-4-12b","pub":49.72,"same_skills":48.44,"new_domains":38.31,"v":48.75,"lab":false,"jev":false,"rank_v":97,"rank_pub":122,"name":"decisio 0.4.0 (Gemma 4 12B)","eq":{"S":47.94,"O":49.95},"rank_tie":22,"tied":true,"parts":{"public":0,"same_skills":28.77,"new_domains":19.98}}},{"engine":"hopper-12b-trained","name":"Hopper 12B (trained)","long_name":"Hopper 12B (trained)","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":11960000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/HopitAI/hopper","code_url":null,"base_model":"google/gemma-4-12B-it","base_url":"https://huggingface.co/google/gemma-4-12B-it","weights_repo":null,"served_checkpoint":null,"org":{"id":"HopitAI","name":"HopitAI","logo":"data/orgs/HopitAI.png"}},"completed":100833,"counts":{"ok":100833,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6708,"conf":0.7469,"ece":0.0762,"brier":0.449,"over95":0.0094,"n":34661,"benchmarks":31,"rel":[{"lo":0,"w":0,"conf":0.0944,"acc":0},{"lo":0.1,"w":0.0036,"conf":0.1702,"acc":0.1679},{"lo":0.2,"w":0.0186,"conf":0.2601,"acc":0.0986},{"lo":0.3,"w":0.051,"conf":0.357,"acc":0.1949},{"lo":0.4,"w":0.0628,"conf":0.4536,"acc":0.3833},{"lo":0.5,"w":0.1132,"conf":0.5488,"acc":0.441},{"lo":0.6,"w":0.1255,"conf":0.6515,"acc":0.5736},{"lo":0.7,"w":0.145,"conf":0.7508,"acc":0.6835},{"lo":0.8,"w":0.1829,"conf":0.8514,"acc":0.7784},{"lo":0.9,"w":0.2973,"conf":0.963,"acc":0.9113}],"areas":{"knowledge":{"ece":0.0892,"acc":0.5977,"conf":0.6868},"language":{"ece":0.017,"acc":0.8008,"conf":0.8122},"retrieval":{"ece":0.1314,"acc":0.5795,"conf":0.7109},"tools":{"ece":0.0396,"acc":0.8648,"conf":0.8771},"arts":{"ece":0.1612,"acc":0.5394,"conf":0.7006}}},"gaps":null,"latency":{"median":289.7,"p95":3910,"mean":1205.5},"results":{"1":{"score":0.9693,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":994.8},"2":{"score":0.6557,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":9349.8},"3":{"score":0.7441,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":32445},"4":{"score":0.7254,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":1180},"5":{"score":0.8607,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":1990},"6":{"score":0.7929,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":2679.9},"9":{"score":0.3182,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":14504.9},"10":{"score":0.159,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":879.9},"11":{"score":0.7478,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":15430},"12":{"score":0.6596,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":219.7},"20":{"score":0.8292,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.9},"21":{"score":0.6126,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":209.7},"22":{"score":0.159,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":10814.9},"23":{"score":0.616,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":449.7},"24":{"score":0.7382,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":219.7},"25":{"score":0.3827,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":289.5},"26":{"score":0.9482,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":209.9},"27":{"score":0.913,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":209.9},"28":{"score":0.7411,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":200},"29":{"score":0.8073,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":249.9},"30":{"score":0.5421,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.1722,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":1029.8},"32":{"score":0.6104,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":499.9},"33":{"score":0.2448,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":2509.8},"36":{"score":0.4212,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":11599.9},"37":{"score":0.4825,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":299.8},"38":{"score":0.2103,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":13399.8},"39":{"score":0.9181,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":399.9},"40":{"score":0.4867,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":219.9},"41":{"score":0.6722,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":229.9},"42":{"score":0.7757,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":339.9},"43":{"score":0.5807,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":229.9},"44":{"score":0.6398,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":219.8},"45":{"score":0.1198,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":259.9},"50":{"score":0.4415,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":479.5},"56":{"score":0.7085,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2389.8},"57":{"score":0.5355,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":259.9},"58":{"score":0.6898,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":229.9},"59":{"score":0.7418,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":419.9},"61":{"score":0.7145,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":329.7},"62":{"score":0.715,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":499.9},"64":{"score":0.6496,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":259.9}},"benchmarks":{"1":{"raw":0.9693,"skill":0.9586,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9451,"combined":0.9261},"2":{"raw":0.6557,"skill":0.6024,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7409,"combined":0.6983},"3":{"raw":0.7441,"skill":0.7392,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.7455,"combined":0.7386},"4":{"raw":0.7254,"skill":0.7219,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8465,"combined":0.8438},"5":{"raw":0.8607,"skill":0.8599,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.6348,"combined":0.6294},"6":{"raw":0.7929,"skill":0.5144,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.3182,"skill":0.3182,"coverage":1,"random":0,"tracks":[],"private_eq":0.345,"combined":0.3385},"10":{"raw":0.159,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7478,"skill":0.6353,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.6596,"skill":0.4901,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6417,"combined":0.4581},"20":{"raw":0.8323,"skill":0.6645,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.782,"combined":0.56},"21":{"raw":0.6126,"skill":0.2253,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6569,"combined":0.3069},"22":{"raw":0.1513,"skill":0.1447,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.2562,"combined":0.2427},"23":{"raw":0.632,"skill":0.2639,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.7382,"skill":0.6509,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3827,"skill":0.1769,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3766,"combined":0.1602},"26":{"raw":0.9482,"skill":0.9309,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.913,"skill":0.884,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7411,"skill":0.4822,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6983,"combined":0.3907},"29":{"raw":0.8073,"skill":0.7431,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.7922,"combined":0.7208},"30":{"raw":0.5421,"skill":0.4509,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.6376,"random":0.25},{"track":"10-choice","raw":0.4466,"random":0.1}],"private_eq":0.6718,"combined":0.5986},"31":{"raw":0.1722,"skill":0.0984,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1729,"combined":0.0897},"32":{"raw":0.6104,"skill":0.3806,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.5093,"combined":0.2119},"33":{"raw":0.2448,"skill":0.2348,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0576,"combined":0.0351},"36":{"raw":0.4212,"skill":0.3452,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.4296,"combined":0.3483},"37":{"raw":0.4825,"skill":0.3509,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.5066,"combined":0.375},"38":{"raw":0.2103,"skill":0.185,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.3671,"combined":0.3403},"39":{"raw":0.9181,"skill":0.8795,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6267,"combined":0.4456},"40":{"raw":0.4867,"skill":0.3396,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4481,"headline":false},{"track":"A · English","score":0.4867,"headline":true},{"track":"C · Arabic pairs","score":0.81,"headline":false},{"track":"C · English pairs","score":0.945,"headline":false}],"private_eq":0.5374,"combined":0.399},"41":{"raw":0.6722,"skill":0.5083,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.6809,"combined":0.5169},"42":{"raw":0.7757,"skill":0.5636,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.7474,"combined":0.5039},"43":{"raw":0.5807,"skill":0.3347,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6457,"combined":0.4324},"44":{"raw":0.6398,"skill":0.2796,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7089,"combined":0.4121},"45":{"raw":0.1198,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4415,"skill":0.1895,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4417,"combined":0.1814},"56":{"raw":0.7085,"skill":0.417,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.692,"combined":0.3779},"57":{"raw":0.5355,"skill":0.4776,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.4111,"combined":0.331},"58":{"raw":0.6898,"skill":0.5504,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.4129,"combined":0.1402},"59":{"raw":0.7418,"skill":0.4646,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.6825,"combined":0.3351},"61":{"raw":0.7145,"skill":0.429,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7785,"combined":0.5529},"62":{"raw":0.715,"skill":0.62,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6702,"combined":0.5562},"64":{"raw":0.6496,"skill":0.562,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.6626,"combined":0.5744}},"coverage":0.9481,"pending":0.0519,"notes":["Public board run by the lab; submitted rows not adoptable."],"scores":{"balanced_raw":61.05,"balanced_skill":45.76,"breadth_skill":46.83,"public_skill":48.27},"frozen_scores":{"balanced_raw":61.05,"balanced_skill":48.27,"breadth_skill":46.83},"categories":[{"id":"knowledge","raw":0.4794,"skill":0.317,"combined":0.2783,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.6716,"skill":0.5378,"combined":0.4709,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.6537,"skill":0.5317,"combined":0.529,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.6935,"skill":0.6626,"combined":0.6649,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.5532,"skill":0.3417,"combined":0.3731,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"hopper-12b-trained","pub":48.27,"same_skills":46.97,"new_domains":34.05,"v":45.76,"lab":false,"jev":false,"rank_v":108,"rank_pub":126,"name":"Hopper 12B (trained)","eq":{"S":46.46,"O":44.7},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":27.88,"new_domains":17.88}}},{"engine":"dm-jepa-1.1","name":"DM-JEPA 1.1","long_name":"DM-JEPA 1.1","variant":null,"meta":{"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":149000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/DangerLabs/DM-JEPA","code_url":null,"base_model":"answerdotai/ModernBERT-base","base_url":"https://huggingface.co/answerdotai/ModernBERT-base","weights_repo":null,"served_checkpoint":null,"org":{"id":"DangerLabs","name":"Danger Labs","logo":"data/orgs/DangerLabs.png"}},"completed":100833,"counts":{"ok":100831,"unsupported":2,"error":0},"lifted":false,"calibration":{"acc":0.4658,"conf":0.9109,"ece":0.4451,"brier":0.9364,"over95":0.2799,"n":34661,"benchmarks":31,"rel":[{"lo":0,"w":0,"conf":null,"acc":null},{"lo":0.1,"w":0.0004,"conf":0.1582,"acc":0},{"lo":0.2,"w":0.0006,"conf":0.2624,"acc":0.2313},{"lo":0.3,"w":0.0062,"conf":0.3431,"acc":0.0657},{"lo":0.4,"w":0.0114,"conf":0.4638,"acc":0.1697},{"lo":0.5,"w":0.0437,"conf":0.5519,"acc":0.3092},{"lo":0.6,"w":0.0449,"conf":0.6517,"acc":0.3557},{"lo":0.7,"w":0.0687,"conf":0.7528,"acc":0.3353},{"lo":0.8,"w":0.0975,"conf":0.8568,"acc":0.2911},{"lo":0.9,"w":0.7265,"conf":0.9836,"acc":0.5263}],"areas":{"knowledge":{"ece":0.426,"acc":0.4644,"conf":0.8905},"language":{"ece":0.4948,"acc":0.4613,"conf":0.9556},"retrieval":{"ece":0.3479,"acc":0.5702,"conf":0.9181},"tools":{"ece":0.4572,"acc":0.4287,"conf":0.8839},"arts":{"ece":0.4665,"acc":0.4151,"conf":0.8816}}},"gaps":null,"latency":{"median":63.2,"p95":2068.1,"mean":520.4},"results":{"1":{"score":0.3813,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":186.1},"2":{"score":0.1304,"jev":null,"metric":"nDCG@10","requests":685,"answered":684,"unsupported":1,"errors":0,"median_ms":1445.3},"3":{"score":0.0276,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":597.1},"4":{"score":0.9353,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":151.5},"5":{"score":0.9294,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":250.2},"6":{"score":0.6602,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":150.8},"9":{"score":0.125,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":946.2},"10":{"score":0.2288,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":60.8},"11":{"score":0.2874,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":1113},"12":{"score":0.9862,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":45.9},"20":{"score":0.6402,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.1},"21":{"score":0.4935,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":44.6},"22":{"score":0,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":252.5},"23":{"score":0.4968,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":46},"24":{"score":0.6656,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":47.3},"25":{"score":0.2821,"jev":null,"metric":"accuracy","requests":196,"answered":195,"unsupported":1,"errors":0,"median_ms":48},"26":{"score":0.8582,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":46.9},"27":{"score":0.8524,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":47.5},"28":{"score":0.9755,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":44.4},"29":{"score":0.7673,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":48},"30":{"score":0.5713,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.106,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":90.6},"32":{"score":0.3949,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":50.1},"33":{"score":0.0006,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":656.1},"36":{"score":0.0883,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1500},"37":{"score":0.1206,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":50.2},"38":{"score":0.0312,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2634.8},"39":{"score":0.1867,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":125.2},"40":{"score":0.2508,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":230.8},"41":{"score":0.247,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":45.9},"42":{"score":0.2845,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":47.3},"43":{"score":0.3561,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":47.6},"44":{"score":0.4966,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":44.8},"45":{"score":0.152,"jev":null,"metric":"accuracy","requests":501,"answered":500,"unsupported":1,"errors":0,"median_ms":52.8},"50":{"score":0.4451,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":50.5},"56":{"score":0.4745,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":412.3},"57":{"score":0.234,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":56.4},"58":{"score":0.4028,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":49.2},"59":{"score":0.2025,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":47.9},"61":{"score":0.5038,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":46.4},"62":{"score":0.3196,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":53.9},"64":{"score":0.2159,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":49.5}},"benchmarks":{"1":{"raw":0.3813,"skill":0.1648,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.3914,"combined":0.17},"2":{"raw":0.1302,"skill":0,"coverage":0.9985,"random":0.1341,"tracks":[],"private_eq":0.1349,"combined":0},"3":{"raw":0.0276,"skill":0.0089,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.0378,"combined":0.009},"4":{"raw":0.9353,"skill":0.9345,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.1656,"combined":0.1461},"5":{"raw":0.9294,"skill":0.929,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.0971,"combined":0.0822},"6":{"raw":0.6602,"skill":0.2033,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.125,"skill":0.125,"coverage":1,"random":0,"tracks":[],"private_eq":0.0425,"combined":0.0325},"10":{"raw":0.2288,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.2874,"skill":0,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.9862,"skill":0.9793,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.3324,"combined":0},"20":{"raw":0.6366,"skill":0.2732,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5497,"combined":0.09},"21":{"raw":0.4935,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"22":{"raw":0,"skill":0,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0078,"combined":0},"23":{"raw":0.5152,"skill":0.0304,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6656,"skill":0.5541,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.2806,"skill":0.0408,"coverage":0.9949,"random":0.25,"tracks":[],"private_eq":0.254,"combined":0},"26":{"raw":0.8582,"skill":0.8109,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8524,"skill":0.8031,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.9755,"skill":0.951,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"29":{"raw":0.7673,"skill":0.6897,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4178,"combined":0.2158},"30":{"raw":0.5713,"skill":0.4909,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.7051,"random":0.25},{"track":"10-choice","raw":0.4375,"random":0.1}],"private_eq":0.175,"combined":0},"31":{"raw":0.106,"skill":0.0263,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0863,"combined":0},"32":{"raw":0.3949,"skill":0.038,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.3938,"combined":0.0262},"33":{"raw":0.0006,"skill":0,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.0131,"combined":0},"36":{"raw":0.0883,"skill":0,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.116,"combined":0},"37":{"raw":0.1206,"skill":0,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.2027,"combined":0},"38":{"raw":0.0312,"skill":0.0002,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.031,"combined":0},"39":{"raw":0.1867,"skill":0,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.3201,"combined":0},"40":{"raw":0.2508,"skill":0.0362,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2502,"headline":false},{"track":"A · English","score":0.2508,"headline":true},{"track":"C · Arabic pairs","score":0.51,"headline":false},{"track":"C · English pairs","score":0.54,"headline":false}],"private_eq":0.2511,"combined":0.0264},"41":{"raw":0.247,"skill":0,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.3333,"combined":0},"42":{"raw":0.2845,"skill":0,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.486,"combined":0},"43":{"raw":0.3561,"skill":0,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.393,"combined":0.0269},"44":{"raw":0.4966,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5289,"combined":0.048},"45":{"raw":0.1517,"skill":0,"coverage":0.998,"random":0.1641,"tracks":[]},"50":{"raw":0.4451,"skill":0.1947,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.3479,"combined":0.0437},"56":{"raw":0.4745,"skill":0,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"57":{"raw":0.234,"skill":0.1385,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.1109,"combined":0},"58":{"raw":0.4028,"skill":0.1344,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3169,"combined":0},"59":{"raw":0.2025,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.5038,"skill":0.0076,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.526,"combined":0.0421},"62":{"raw":0.3196,"skill":0.0928,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3199,"combined":0.0838},"64":{"raw":0.2159,"skill":0.0199,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.2637,"combined":0.07}},"coverage":0.9479,"pending":0.0521,"notes":["DM-JEPA 1.1 run by the lab; submitted rows not adoptable."],"scores":{"balanced_raw":37.21,"balanced_skill":4.96,"breadth_skill":17.86,"public_skill":19.19},"frozen_scores":{"balanced_raw":37.21,"balanced_skill":19.19,"breadth_skill":17.86},"categories":[{"id":"knowledge","raw":0.3649,"skill":0.1727,"combined":0.0039,"coverage":0.9993,"pending":0,"n":11},{"id":"language","raw":0.3824,"skill":0.2169,"combined":0.0262,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5216,"skill":0.34,"combined":0.0428,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.1973,"skill":0.0789,"combined":0.0595,"coverage":0.9997,"pending":0,"n":5},{"id":"arts","raw":0.3844,"skill":0.0864,"combined":0.0365,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"dm-jepa-1.1","pub":19.19,"same_skills":4.17,"new_domains":4.04,"v":4.96,"lab":false,"jev":false,"rank_v":240,"rank_pub":218,"name":"DM-JEPA 1.1","eq":{"S":3.17,"O":7.64},"rank_tie":58,"tied":true,"parts":{"public":0,"same_skills":1.9,"new_domains":3.05}}},{"engine":"jevstral-8b","name":"Jevstral 8B","long_name":"Jevstral 8B","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":8000000000,"params_estimated":true,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/AmirBraham/jevstral-8b","code_url":null,"base_model":"mistralai/Ministral-3-8B-Base-2512","base_url":"https://huggingface.co/mistralai/Ministral-3-8B-Base-2512","weights_repo":null,"served_checkpoint":null},"completed":110201,"counts":{"ok":110201,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.6158,"conf":0.6528,"ece":0.0472,"brier":0.4948,"over95":0.0076,"n":34838,"benchmarks":32,"rel":[{"lo":0,"w":0.0574,"conf":0.0544,"acc":0.057},{"lo":0.1,"w":0.01,"conf":0.1575,"acc":0.165},{"lo":0.2,"w":0.0274,"conf":0.2554,"acc":0.2995},{"lo":0.3,"w":0.0647,"conf":0.3539,"acc":0.4102},{"lo":0.4,"w":0.087,"conf":0.447,"acc":0.4198},{"lo":0.5,"w":0.1341,"conf":0.5479,"acc":0.5136},{"lo":0.6,"w":0.1261,"conf":0.6509,"acc":0.6157},{"lo":0.7,"w":0.1447,"conf":0.7512,"acc":0.6892},{"lo":0.8,"w":0.1604,"conf":0.8508,"acc":0.7787},{"lo":0.9,"w":0.1883,"conf":0.9489,"acc":0.8946}],"areas":{"knowledge":{"ece":0.0425,"acc":0.5619,"conf":0.6001},"language":{"ece":0.0552,"acc":0.6901,"conf":0.742},"retrieval":{"ece":0.0912,"acc":0.7051,"conf":0.6255},"tools":{"ece":0.0487,"acc":0.8133,"conf":0.8171},"arts":{"ece":0.1288,"acc":0.4105,"conf":0.5316}}},"gaps":null,"latency":{"median":28.3,"p95":185.1,"mean":79.3},"results":{"1":{"score":0.9138,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":63.9},"2":{"score":0.6067,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":994},"3":{"score":0.6083,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":385.5},"4":{"score":0.831,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":57.9},"5":{"score":0.7416,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":87.8},"6":{"score":0.7842,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":167.4},"9":{"score":0.1705,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":236.6},"10":{"score":0.5965,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":66},"11":{"score":0.593,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":267.3},"12":{"score":0.4998,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":22.3},"20":{"score":0.541,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.9},"21":{"score":0.5285,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":19.8},"22":{"score":0.0515,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":400},"23":{"score":0.553,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":39.9},"24":{"score":0.6779,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":20.5},"25":{"score":0.3776,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":23.4},"26":{"score":0.9655,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":20.1},"27":{"score":0.8993,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":20.1},"28":{"score":0.7111,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":19.8},"29":{"score":0.741,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":24.3},"30":{"score":0.4219,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.0854,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":52.6},"32":{"score":0.6144,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":60.2},"33":{"score":0.2388,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":71.9},"36":{"score":0.3833,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":1472.1},"37":{"score":0.3683,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":30.2},"38":{"score":0.0985,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":358},"39":{"score":0.7815,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":43.9},"40":{"score":0.4333,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":20.9},"41":{"score":0.4369,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":23.4},"42":{"score":0.7375,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":36.2},"43":{"score":0.4211,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":22.6},"44":{"score":0.6474,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":22.2},"45":{"score":0.1178,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":24.3},"50":{"score":0.3783,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":54.2},"56":{"score":0.5165,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":73.1},"57":{"score":0.4787,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":23.9},"58":{"score":0.5713,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":23.3},"59":{"score":0.1338,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":52.1},"61":{"score":0.552,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":34.4},"62":{"score":0.5203,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":47.7},"64":{"score":0.572,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":23.2}},"benchmarks":{"1":{"raw":0.9138,"skill":0.8837,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9102,"combined":0.8784},"2":{"raw":0.6067,"skill":0.5458,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.6891,"combined":0.6378},"3":{"raw":0.6083,"skill":0.6008,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.6182,"combined":0.6074},"4":{"raw":0.831,"skill":0.8288,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.7704,"combined":0.7658},"5":{"raw":0.7416,"skill":0.74,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.5541,"combined":0.5473},"6":{"raw":0.7842,"skill":0.494,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.1705,"skill":0.1705,"coverage":1,"random":0,"tracks":[],"private_eq":0.2375,"combined":0.2297},"10":{"raw":0.5965,"skill":0.3286,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.593,"skill":0.4114,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.4998,"skill":0.2507,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.5975,"combined":0.3912},"20":{"raw":0.5538,"skill":0.1077,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5533,"combined":0.0973},"21":{"raw":0.5285,"skill":0.0571,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5,"combined":0},"22":{"raw":0.0535,"skill":0.0461,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.0624,"combined":0.0452},"23":{"raw":0.5359,"skill":0.0717,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.6779,"skill":0.5705,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.3776,"skill":0.1701,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.3152,"combined":0.0774},"26":{"raw":0.9655,"skill":0.954,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.8993,"skill":0.8657,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.7111,"skill":0.4222,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6784,"combined":0.3504},"29":{"raw":0.741,"skill":0.6547,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.7114,"combined":0.6118},"30":{"raw":0.4219,"skill":0.303,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.508,"random":0.25},{"track":"10-choice","raw":0.3359,"random":0.1}],"private_eq":0.571,"combined":0.475},"31":{"raw":0.0854,"skill":0.0039,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.0844,"combined":0},"32":{"raw":0.6144,"skill":0.387,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.4371,"combined":0.0958},"33":{"raw":0.2388,"skill":0.2287,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.2932,"combined":0.2766},"36":{"raw":0.3833,"skill":0.3024,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.352,"combined":0.2595},"37":{"raw":0.3683,"skill":0.2077,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.5216,"combined":0.3941},"38":{"raw":0.0985,"skill":0.0697,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.1519,"combined":0.1157},"39":{"raw":0.7815,"skill":0.6786,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6998,"combined":0.5544},"40":{"raw":0.4333,"skill":0.2709,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.2655,"headline":false},{"track":"A · English","score":0.4333,"headline":true},{"track":"C · Arabic pairs","score":0.495,"headline":false},{"track":"C · English pairs","score":0.505,"headline":false}],"private_eq":0.4052,"combined":0.227},"41":{"raw":0.4369,"skill":0.1554,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.4929,"combined":0.2316},"42":{"raw":0.7375,"skill":0.4893,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.9019,"combined":0.808},"43":{"raw":0.4211,"skill":0.0814,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.6014,"combined":0.3613},"44":{"raw":0.6474,"skill":0.2948,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6559,"combined":0.3049},"45":{"raw":0.1178,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.3783,"skill":0.0977,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4911,"combined":0.2539},"56":{"raw":0.5165,"skill":0.033,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5508,"combined":0.0923},"57":{"raw":0.4787,"skill":0.4137,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.3702,"combined":0.2845},"58":{"raw":0.5713,"skill":0.3786,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.3592,"combined":0.0615},"59":{"raw":0.1338,"skill":0,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.5177,"combined":0},"61":{"raw":0.552,"skill":0.104,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.526,"combined":0.0421},"62":{"raw":0.5203,"skill":0.3604,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.5991,"combined":0.4603},"64":{"raw":0.572,"skill":0.465,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.6012,"combined":0.4968}},"coverage":1,"pending":0,"notes":["Base not runnable on the stock harness, so compared with Jev."],"scores":{"balanced_raw":50.42,"balanced_skill":32.23,"breadth_skill":32.48,"public_skill":34.01},"frozen_scores":{"balanced_raw":50.42,"balanced_skill":34.01,"breadth_skill":32.48},"categories":[{"id":"knowledge","raw":0.4281,"skill":0.2467,"combined":0.2254,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.5004,"skill":0.3364,"combined":0.3726,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.5733,"skill":0.3924,"combined":0.366,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.5785,"skill":0.5293,"combined":0.5761,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.437,"skill":0.1409,"combined":0.1765,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"jevstral-8b","pub":34.01,"same_skills":36.31,"new_domains":19.77,"v":32.23,"lab":false,"jev":false,"rank_v":160,"rank_pub":176,"name":"Jevstral 8B","eq":{"S":35.68,"O":27.07},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":21.41,"new_domains":10.83}}},{"engine":"deck31b","name":"deck31b (Gemma 4 31B)","long_name":"deck31b (Gemma 4 31B)","variant":null,"meta":{"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":32680000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":null,"code_url":"https://github.com/krishna-gogineni-765/deck31b","base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null},"completed":100833,"counts":{"ok":100833,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7176,"conf":0.9217,"ece":0.2043,"brier":0.4691,"over95":0.1268,"n":34661,"benchmarks":31,"rel":[{"lo":0,"w":0.0005,"conf":0.0851,"acc":0},{"lo":0.1,"w":0.0041,"conf":0.1548,"acc":0.1695},{"lo":0.2,"w":0.0067,"conf":0.2542,"acc":0.0473},{"lo":0.3,"w":0.0108,"conf":0.356,"acc":0.1488},{"lo":0.4,"w":0.0225,"conf":0.4601,"acc":0.1228},{"lo":0.5,"w":0.0273,"conf":0.5583,"acc":0.2384},{"lo":0.6,"w":0.0283,"conf":0.6537,"acc":0.3736},{"lo":0.7,"w":0.0338,"conf":0.7506,"acc":0.4082},{"lo":0.8,"w":0.0553,"conf":0.8556,"acc":0.4595},{"lo":0.9,"w":0.8106,"conf":0.9853,"acc":0.8091}],"areas":{"knowledge":{"ece":0.2163,"acc":0.6715,"conf":0.8877},"language":{"ece":0.1512,"acc":0.8209,"conf":0.9718},"retrieval":{"ece":0.3361,"acc":0.6049,"conf":0.941},"tools":{"ece":0.0365,"acc":0.9386,"conf":0.9736},"arts":{"ece":0.2749,"acc":0.5813,"conf":0.853}}},"gaps":null,"latency":{"median":218.5,"p95":2027.3,"mean":677.3},"results":{"1":{"score":0.9811,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":632.8},"2":{"score":0.6124,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":6909.2},"3":{"score":0.8484,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":4711.7},"4":{"score":0.7802,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":217.7},"5":{"score":0.8994,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":213.8},"6":{"score":0.7994,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1338.3},"9":{"score":0.6477,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":6037.9},"10":{"score":0.0384,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":428.8},"11":{"score":0.7256,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":7942},"12":{"score":0.7368,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":214.8},"20":{"score":0.8912,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":214.7},"21":{"score":0.6286,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":214.4},"22":{"score":0.254,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":2430.6},"23":{"score":0.6562,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":216.6},"24":{"score":0.865,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":215},"25":{"score":0.5102,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":214.4},"26":{"score":0.9899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":214.7},"27":{"score":0.9718,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":214.8},"28":{"score":0.8493,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":215},"29":{"score":0.921,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":215.4},"30":{"score":0.7282,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.2188,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":215.5},"32":{"score":0.641,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":253.7},"33":{"score":0.3085,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":1713},"36":{"score":0.4266,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":7232.5},"37":{"score":0.5307,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":216.8},"38":{"score":0.2013,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":13506.7},"39":{"score":0.9339,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":423.6},"40":{"score":0.5135,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":219.3},"41":{"score":0.6899,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":215.2},"42":{"score":0.8355,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":219},"43":{"score":0.807,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":214.3},"44":{"score":0.7622,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":214.9},"45":{"score":0.1337,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":216.7},"50":{"score":0.4869,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":224.2},"56":{"score":0.89,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1951},"57":{"score":0.6926,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":215.4},"58":{"score":0.763,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":215.1},"59":{"score":0.7928,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":223.1},"61":{"score":0.7588,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":216.3},"62":{"score":0.773,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":258.4},"64":{"score":0.7481,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":215}},"benchmarks":{"1":{"raw":0.9811,"skill":0.9745,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9501,"combined":0.9329},"2":{"raw":0.6124,"skill":0.5523,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7043,"combined":0.6556},"3":{"raw":0.8484,"skill":0.8455,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.9186,"combined":0.9171},"4":{"raw":0.7802,"skill":0.7774,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8672,"combined":0.865},"5":{"raw":0.8994,"skill":0.8988,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.7659,"combined":0.7628},"6":{"raw":0.7994,"skill":0.5297,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.6477,"skill":0.6477,"coverage":1,"random":0,"tracks":[],"private_eq":0.645,"combined":0.6419},"10":{"raw":0.0384,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.7256,"skill":0.6032,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7368,"skill":0.6058,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6533,"combined":0.4757},"20":{"raw":0.8947,"skill":0.7895,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.9214,"combined":0.842},"21":{"raw":0.6286,"skill":0.2572,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.5823,"combined":0.156},"22":{"raw":0.2506,"skill":0.2447,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.2658,"combined":0.2525},"23":{"raw":0.6526,"skill":0.3053,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.865,"skill":0.82,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.5102,"skill":0.3469,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.523,"combined":0.3577},"26":{"raw":0.9899,"skill":0.9865,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9718,"skill":0.9624,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.8493,"skill":0.6986,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.813,"combined":0.6227},"29":{"raw":0.921,"skill":0.8947,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.994,"combined":0.9929},"30":{"raw":0.7282,"skill":0.6763,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8044,"random":0.25},{"track":"10-choice","raw":0.652,"random":0.1}],"private_eq":0.7882,"combined":0.7413},"31":{"raw":0.2188,"skill":0.1492,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1709,"combined":0.0875},"32":{"raw":0.641,"skill":0.4293,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.6392,"combined":0.4208},"33":{"raw":0.3085,"skill":0.2993,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.4386,"combined":0.4256},"36":{"raw":0.4266,"skill":0.3514,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.4639,"combined":0.3876},"37":{"raw":0.5307,"skill":0.4114,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.6489,"combined":0.5556},"38":{"raw":0.2013,"skill":0.1757,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.3663,"combined":0.3395},"39":{"raw":0.9339,"skill":0.9028,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.7261,"combined":0.5935},"40":{"raw":0.5135,"skill":0.3741,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.5116,"headline":false},{"track":"A · English","score":0.5135,"headline":true},{"track":"C · Arabic pairs","score":0.905,"headline":false},{"track":"C · English pairs","score":0.97,"headline":false}],"private_eq":0.5727,"combined":0.445},"41":{"raw":0.6899,"skill":0.5349,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.6602,"combined":0.4855},"42":{"raw":0.8355,"skill":0.68,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.8999,"combined":0.804},"43":{"raw":0.807,"skill":0.6938,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.7395,"combined":0.583},"44":{"raw":0.7622,"skill":0.5244,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7671,"combined":0.5298},"45":{"raw":0.1337,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4869,"skill":0.2553,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4861,"combined":0.2466},"56":{"raw":0.89,"skill":0.78,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8546,"combined":0.7069},"57":{"raw":0.6926,"skill":0.6543,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.5196,"combined":0.4545},"58":{"raw":0.763,"skill":0.6565,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.502,"combined":0.2709},"59":{"raw":0.7928,"skill":0.5704,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.7103,"combined":0.3934},"61":{"raw":0.7588,"skill":0.5176,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.798,"combined":0.5924},"62":{"raw":0.773,"skill":0.6973,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6752,"combined":0.563},"64":{"raw":0.7481,"skill":0.6851,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.7776,"combined":0.7198}},"coverage":0.9481,"pending":0.0519,"notes":["Public board run by the lab; option cap lifted (two-letter labels)."],"scores":{"balanced_raw":68.16,"balanced_skill":63.05,"breadth_skill":57.32,"public_skill":58.19},"frozen_scores":{"balanced_raw":68.16,"balanced_skill":58.19,"breadth_skill":57.32},"categories":[{"id":"knowledge","raw":0.5837,"skill":0.4667,"combined":0.4477,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7108,"skill":0.6002,"combined":0.5742,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7132,"skill":0.6276,"combined":0.6475,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7831,"skill":0.7558,"combined":0.7556,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6102,"skill":0.4229,"combined":0.4434,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"deck31b","pub":58.19,"same_skills":58.6,"new_domains":54.77,"v":63.05,"lab":false,"jev":false,"rank_v":37,"rank_pub":75,"name":"deck31b (Gemma 4 31B)","eq":{"S":58.23,"O":70.29},"rank_tie":8,"tied":true,"parts":{"public":0,"same_skills":34.94,"new_domains":28.12}}},{"engine":"quyet-1.0-large","name":"Quyet-1.0-Large","long_name":"Quyet-1.0-Large","variant":null,"meta":{"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":32680000000,"params_estimated":false,"params_from_name":false,"trained_bytes":null,"model_url":"https://huggingface.co/chinhnc/Quyet-1.0-Large","code_url":null,"base_model":"google/gemma-4-31B-it","base_url":"https://huggingface.co/google/gemma-4-31B-it","weights_repo":null,"served_checkpoint":null},"completed":100833,"counts":{"ok":100833,"unsupported":0,"error":0},"lifted":false,"calibration":{"acc":0.7347,"conf":0.8038,"ece":0.0778,"brier":0.3748,"over95":0.0192,"n":34661,"benchmarks":31,"rel":[{"lo":0,"w":0.0155,"conf":0.065,"acc":0.1191},{"lo":0.1,"w":0.0191,"conf":0.1458,"acc":0.1669},{"lo":0.2,"w":0.0187,"conf":0.2531,"acc":0.3574},{"lo":0.3,"w":0.0281,"conf":0.3541,"acc":0.3866},{"lo":0.4,"w":0.035,"conf":0.4574,"acc":0.4649},{"lo":0.5,"w":0.0732,"conf":0.5488,"acc":0.4595},{"lo":0.6,"w":0.0789,"conf":0.6492,"acc":0.5461},{"lo":0.7,"w":0.081,"conf":0.7516,"acc":0.6343},{"lo":0.8,"w":0.1264,"conf":0.8545,"acc":0.6874},{"lo":0.9,"w":0.524,"conf":0.9714,"acc":0.9176}],"areas":{"knowledge":{"ece":0.1078,"acc":0.6793,"conf":0.7871},"language":{"ece":0.0209,"acc":0.8659,"conf":0.8829},"retrieval":{"ece":0.2072,"acc":0.6339,"conf":0.7411},"tools":{"ece":0.0349,"acc":0.9352,"conf":0.945},"arts":{"ece":0.1033,"acc":0.5699,"conf":0.6603}}},"gaps":null,"latency":{"median":105.4,"p95":1588.6,"mean":356.6},"results":{"1":{"score":0.9746,"jev":null,"metric":"case exact accuracy","requests":1694,"answered":1694,"unsupported":0,"errors":0,"median_ms":303.8},"2":{"score":0.6646,"jev":null,"metric":"nDCG@10","requests":685,"answered":685,"unsupported":0,"errors":0,"median_ms":3447.6},"3":{"score":0.8445,"jev":null,"metric":"accuracy","requests":508,"answered":508,"unsupported":0,"errors":0,"median_ms":2180.4},"4":{"score":0.8471,"jev":null,"metric":"macro-F1","requests":3080,"answered":3080,"unsupported":0,"errors":0,"median_ms":176.7},"5":{"score":0.9123,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":199},"6":{"score":0.7997,"jev":null,"metric":"selected quality (quality objective)","requests":10000,"answered":10000,"unsupported":0,"errors":0,"median_ms":1172.9},"9":{"score":0.5682,"jev":null,"metric":"case exact accuracy","requests":88,"answered":88,"unsupported":0,"errors":0,"median_ms":3717.5},"10":{"score":0.1126,"jev":null,"metric":"macro-F1","requests":2500,"answered":2500,"unsupported":0,"errors":0,"median_ms":281},"11":{"score":0.8484,"jev":null,"metric":"macro-F1","requests":123,"answered":123,"unsupported":0,"errors":0,"median_ms":9112.7},"12":{"score":0.7638,"jev":null,"metric":"macro-F1","requests":3200,"answered":3200,"unsupported":0,"errors":0,"median_ms":86.7},"20":{"score":0.9212,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":87},"21":{"score":0.6336,"jev":null,"metric":"accuracy","requests":2628,"answered":2628,"unsupported":0,"errors":0,"median_ms":78.2},"22":{"score":0.1845,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":1833.4},"23":{"score":0.6726,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":128},"24":{"score":0.8609,"jev":null,"metric":"accuracy","requests":14033,"answered":14033,"unsupported":0,"errors":0,"median_ms":80.3},"25":{"score":0.4898,"jev":null,"metric":"accuracy","requests":196,"answered":196,"unsupported":0,"errors":0,"median_ms":88.6},"26":{"score":0.9899,"jev":null,"metric":"accuracy","requests":2376,"answered":2376,"unsupported":0,"errors":0,"median_ms":78.6},"27":{"score":0.9787,"jev":null,"metric":"accuracy","requests":1172,"answered":1172,"unsupported":0,"errors":0,"median_ms":79.2},"28":{"score":0.91,"jev":null,"metric":"accuracy","requests":1267,"answered":1267,"unsupported":0,"errors":0,"median_ms":78},"29":{"score":0.9571,"jev":null,"metric":"accuracy","requests":10042,"answered":10042,"unsupported":0,"errors":0,"median_ms":91.9},"30":{"score":0.8355,"jev":0.7301,"metric":"accuracy (4- and 10-choice)","requests":2638,"answered":2638,"unsupported":0,"errors":0,"median_ms":null},"31":{"score":0.228,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":198.2},"32":{"score":0.6343,"jev":null,"metric":"accuracy","requests":752,"answered":752,"unsupported":0,"errors":0,"median_ms":227.7},"33":{"score":0.3006,"jev":null,"metric":"case exact accuracy","requests":1650,"answered":1650,"unsupported":0,"errors":0,"median_ms":789},"36":{"score":0.459,"jev":null,"metric":"nDCG@10","requests":220,"answered":220,"unsupported":0,"errors":0,"median_ms":5092.8},"37":{"score":0.4813,"jev":null,"metric":"macro-F1","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":112.9},"38":{"score":0.3256,"jev":null,"metric":"per-review F1","requests":1565,"answered":1565,"unsupported":0,"errors":0,"median_ms":2122.1},"39":{"score":0.9657,"jev":null,"metric":"macro-F1","requests":979,"answered":979,"unsupported":0,"errors":0,"median_ms":111.5},"40":{"score":0.4801,"jev":null,"metric":"Sarcasm F1 · track A, English","requests":4600,"answered":4600,"unsupported":0,"errors":0,"median_ms":78.2},"41":{"score":0.7756,"jev":null,"metric":"macro-F1","requests":3006,"answered":3006,"unsupported":0,"errors":0,"median_ms":87.4},"42":{"score":0.8429,"jev":null,"metric":"macro-F1","requests":5500,"answered":5500,"unsupported":0,"errors":0,"median_ms":129.1},"43":{"score":0.7772,"jev":null,"metric":"accuracy","requests":570,"answered":570,"unsupported":0,"errors":0,"median_ms":86.9},"44":{"score":0.7508,"jev":null,"metric":"accuracy","requests":5000,"answered":5000,"unsupported":0,"errors":0,"median_ms":86.7},"45":{"score":0.1317,"jev":null,"metric":"accuracy","requests":501,"answered":501,"unsupported":0,"errors":0,"median_ms":90},"50":{"score":0.4379,"jev":null,"metric":"accuracy","requests":1676,"answered":1676,"unsupported":0,"errors":0,"median_ms":202.5},"56":{"score":0.9,"jev":null,"metric":"accuracy","requests":2000,"answered":2000,"unsupported":0,"errors":0,"median_ms":522.7},"57":{"score":0.6705,"jev":null,"metric":"accuracy","requests":12032,"answered":12032,"unsupported":0,"errors":0,"median_ms":88.1},"58":{"score":0.8969,"jev":null,"metric":"accuracy","requests":5507,"answered":5507,"unsupported":0,"errors":0,"median_ms":87.3},"59":{"score":0.838,"jev":null,"metric":"F1 on hallucinated class","requests":2700,"answered":2700,"unsupported":0,"errors":0,"median_ms":199.8},"61":{"score":0.811,"jev":null,"metric":"accuracy","requests":4000,"answered":4000,"unsupported":0,"errors":0,"median_ms":126.9},"62":{"score":0.6763,"jev":null,"metric":"accuracy","requests":3652,"answered":3652,"unsupported":0,"errors":0,"median_ms":177},"64":{"score":0.767,"jev":null,"metric":"accuracy","requests":528,"answered":528,"unsupported":0,"errors":0,"median_ms":87.9}},"benchmarks":{"1":{"raw":0.9746,"skill":0.9657,"coverage":1,"random":0.2592,"tracks":[],"private_eq":0.9601,"combined":0.9465},"2":{"raw":0.6646,"skill":0.6126,"coverage":1,"random":0.1341,"tracks":[],"private_eq":0.7452,"combined":0.7034},"3":{"raw":0.8445,"skill":0.8415,"coverage":1,"random":0.0189,"tracks":[],"private_eq":0.8676,"combined":0.8645},"4":{"raw":0.8471,"skill":0.8451,"coverage":1,"random":0.0127,"tracks":[],"private_eq":0.8817,"combined":0.8798},"5":{"raw":0.9123,"skill":0.9118,"coverage":1,"random":0.006,"tracks":[],"private_eq":0.7113,"combined":0.7072},"6":{"raw":0.7997,"skill":0.5304,"coverage":1,"random":0.5735,"tracks":[]},"9":{"raw":0.5682,"skill":0.5682,"coverage":1,"random":0,"tracks":[],"private_eq":0.65,"combined":0.647},"10":{"raw":0.1126,"skill":0,"coverage":1,"random":0.399,"tracks":[]},"11":{"raw":0.8484,"skill":0.7808,"coverage":1,"random":0.3085,"tracks":[]},"12":{"raw":0.7638,"skill":0.6462,"coverage":1,"random":0.3324,"tracks":[],"private_eq":0.6779,"combined":0.513},"20":{"raw":0.9241,"skill":0.8481,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.9607,"combined":0.9215},"21":{"raw":0.6336,"skill":0.2671,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.6072,"combined":0.2064},"22":{"raw":0.1793,"skill":0.1729,"coverage":1,"random":0.0078,"tracks":[],"private_eq":0.171,"combined":0.1559},"23":{"raw":0.6655,"skill":0.3309,"coverage":1,"random":0.5,"tracks":[]},"24":{"raw":0.8609,"skill":0.8145,"coverage":1,"random":0.25,"tracks":[]},"25":{"raw":0.4898,"skill":0.3197,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.4815,"combined":0.3017},"26":{"raw":0.9899,"skill":0.9865,"coverage":1,"random":0.2502,"tracks":[]},"27":{"raw":0.9787,"skill":0.9716,"coverage":1,"random":0.2502,"tracks":[]},"28":{"raw":0.91,"skill":0.82,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8205,"combined":0.6379},"29":{"raw":0.9571,"skill":0.9428,"coverage":1,"random":0.25,"tracks":[],"private_eq":1,"combined":1.001},"30":{"raw":0.8355,"skill":0.8028,"coverage":1,"random":0.175,"tracks":[{"track":"4-choice","raw":0.8704,"random":0.25},{"track":"10-choice","raw":0.8006,"random":0.1}],"private_eq":0.8249,"combined":0.7863},"31":{"raw":0.228,"skill":0.1592,"coverage":1,"random":0.0819,"tracks":[],"private_eq":0.1935,"combined":0.1124},"32":{"raw":0.6343,"skill":0.4186,"coverage":1,"random":0.371,"tracks":[],"private_eq":0.6825,"combined":0.4904},"33":{"raw":0.3006,"skill":0.2913,"coverage":1,"random":0.0131,"tracks":[],"private_eq":0.4286,"combined":0.4154},"36":{"raw":0.459,"skill":0.388,"coverage":1,"random":0.116,"tracks":[],"private_eq":0.475,"combined":0.4003},"37":{"raw":0.4813,"skill":0.3493,"coverage":1,"random":0.2027,"tracks":[],"private_eq":0.6297,"combined":0.5312},"38":{"raw":0.3256,"skill":0.304,"coverage":1,"random":0.031,"tracks":[],"private_eq":0.5069,"combined":0.4863},"39":{"raw":0.9657,"skill":0.9496,"coverage":1,"random":0.3201,"tracks":[],"private_eq":0.6893,"combined":0.5388},"40":{"raw":0.4801,"skill":0.3312,"coverage":1,"random":0.2227,"tracks":[{"track":"A · Arabic","score":0.4857,"headline":false},{"track":"A · English","score":0.4801,"headline":true},{"track":"C · Arabic pairs","score":0.89,"headline":false},{"track":"C · English pairs","score":0.965,"headline":false}],"private_eq":0.4479,"combined":0.2826},"41":{"raw":0.7756,"skill":0.6635,"coverage":1,"random":0.3333,"tracks":[],"private_eq":0.6391,"combined":0.4535},"42":{"raw":0.8429,"skill":0.6944,"coverage":1,"random":0.486,"tracks":[],"private_eq":0.9582,"combined":0.9188},"43":{"raw":0.7772,"skill":0.6465,"coverage":1,"random":0.3697,"tracks":[],"private_eq":0.7682,"combined":0.629},"44":{"raw":0.7508,"skill":0.5016,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.7353,"combined":0.4655},"45":{"raw":0.1317,"skill":0,"coverage":1,"random":0.1641,"tracks":[]},"50":{"raw":0.4379,"skill":0.1843,"coverage":1,"random":0.311,"tracks":[],"private_eq":0.4615,"combined":0.2104},"56":{"raw":0.9,"skill":0.8,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8246,"combined":0.6462},"57":{"raw":0.6705,"skill":0.6294,"coverage":1,"random":0.1109,"tracks":[],"private_eq":0.6055,"combined":0.5522},"58":{"raw":0.8969,"skill":0.8506,"coverage":1,"random":0.3101,"tracks":[],"private_eq":0.5381,"combined":0.3238},"59":{"raw":0.838,"skill":0.6641,"coverage":1,"random":0.5177,"tracks":[],"private_eq":0.7056,"combined":0.3836},"61":{"raw":0.811,"skill":0.622,"coverage":1,"random":0.5,"tracks":[],"private_eq":0.8065,"combined":0.6096},"62":{"raw":0.6763,"skill":0.5684,"coverage":1,"random":0.25,"tracks":[],"private_eq":0.6245,"combined":0.4946},"64":{"raw":0.767,"skill":0.7087,"coverage":1,"random":0.2,"tracks":[],"private_eq":0.8082,"combined":0.7585}},"coverage":0.9481,"pending":0.0519,"notes":["Public board run by the lab; option cap lifted."],"scores":{"balanced_raw":69.84,"balanced_skill":60.77,"breadth_skill":59.98,"public_skill":60.79},"frozen_scores":{"balanced_raw":69.84,"balanced_skill":60.79,"breadth_skill":59.98},"categories":[{"id":"knowledge","raw":0.6038,"skill":0.497,"combined":0.4702,"coverage":1,"pending":0,"n":11},{"id":"language","raw":0.7597,"skill":0.6696,"combined":0.581,"coverage":1,"pending":0,"n":9},{"id":"retrieval","raw":0.7355,"skill":0.6584,"combined":0.6321,"coverage":1,"pending":0,"n":6},{"id":"tools","raw":0.7578,"skill":0.7255,"combined":0.7441,"coverage":1,"pending":0,"n":5},{"id":"arts","raw":0.6012,"skill":0.4187,"combined":0.4505,"coverage":1,"pending":0,"n":6}],"v03":{"engine":"quyet-1.0-large","pub":60.79,"same_skills":58.83,"new_domains":49.86,"v":60.77,"lab":false,"jev":false,"rank_v":45,"rank_pub":56,"name":"Quyet-1.0-Large","eq":{"S":58.46,"O":64.23},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.08,"new_domains":25.69}}}]} \ No newline at end of file diff --git a/data/methodology.json b/data/methodology.json deleted file mode 100644 index e065cfb05c6e1022d1135460b7ad90138720a7a7..0000000000000000000000000000000000000000 --- a/data/methodology.json +++ /dev/null @@ -1 +0,0 @@ -{"generated_utc":"2026-09-28T00:39:48+00:00","edition":{"id":"v0.4","label":"Decision Index 0.4.1","date":"2026-10-21","panel_id":"decision-index-0.3","jev_version":"jev-1.14.0","jev_edition":"Lab run, 2026-10-14"},"suite":{"requests":110201,"scoreable":109759,"excluded":442,"source_cases":110313,"fields":536776,"static_benchmarks":37,"jev_static_benchmarks":42,"reproduction_benchmarks":37,"seed":20260919,"selection":"Deterministic hash sample without replacement; ESCI proportional locale × relevance; request-budget datasets retain complete linked groups; no outcome-based selection.","corpus_sha256":"b2b56d6fb636837ca469e689087bdbf373dda8de7638aa2da6793e6eda0792d5","created_utc":"2026-09-22T20:05:14.197492+00:00","jev_version":"jev-1.14.0","jev_edition":"Lab run, 2026-10-14","jev_api_cost_usd":12.2567,"jev_billed_tokens":{"input_tokens":291826668,"output_tokens":44483219},"hardware":{"label":"1 x NVIDIA RTX PRO 6000","reproductions":"1 x NVIDIA RTX PRO 6000 (Blackwell Server Edition, 96 GB), one GPU per run, isolated Python environment per engine","jev":"TypeSafe hosted API (jev-1.14.0) over HTTPS; the frozen suite's request payloads still name jev-1.13.0, the model they were built with"},"release_notes":["Original 500-case pilot remains frozen separately.","Request caps count planned logical evaluations. Transport retries or context splits are logged separately and may add HTTP attempts.","Cached successful pilot responses reused only for identical full request SHA256; original latency and provenance retained.","Game protocols and budgets are pinned separately.","Release v2 keeps every v1 row except a stratified subset cut of ToolRet (1,000 of 7,961 queries incl. excluded ones) and BRIGHT (550 of 1,384 queries); all other benchmarks, question order, options and scoring are unchanged. Kept ToolRet and BRIGHT queries retain their complete candidate lists.","ToolRet and BRIGHT rows of questions listed in excluded-questions.json were carried over unchanged (they are never scored); the scored subsets are 1,000 and 550 query groups, see scored_cases."],"jev_summary_notes":["Metrics exclude failed/incomplete cases; coverage is explicit. No silent replacement of failures.","Different primary metrics are not averaged across benchmarks.","Latency includes HTTPS and concurrent service scheduling; not isolated model inference.","Full raw HTTP responses/payloads and all retry timings are in http-attempts.jsonl.","BPoMP/RouterBench/GSM8K and chunked tasks retain all linked requests for sampled cases."],"archive_limitations":["Context fit not certified under Jev tokenizer.","No inference, accuracy, or latency measured.","Source archive counts are not active evaluation counts: see dataset-evaluation-counts.csv. Only ESCI is capped at 2000.","Some downloaded environments still need executable adapters.","Previously used benchmarks are exposed development evidence.","Adapted scores are not automatically comparable with native leaderboards.","Source duplicates are retained and flagged by structural audit."],"source_pinning":"Git/HF commit where available plus local SHA256. Direct university/GitHub raw releases pinned by content hash.","origins":{"Original-task benchmarks":"published questions, answer spaces and scoring retained; extraction of a clearly labeled supported subset or random sampling does not by itself create a new task.","Jev-adapted benchmarks":"published source data or environment, with a decision interface, answer construction, candidate retrieval, partner protocol or evaluated objective adapted for Jev.","Purpose-built benchmarks":"new local scenarios and evaluation logic; currently the executable home-appliance benchmark."},"adaptation_classes":{"A_direct":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","B_mechanical":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","C_semantic_options":"The adaptation relies on authored semantic option spaces/scenarios or model-generated candidate answers absent from the original native answer format.","not_applicable":"Removed, skipped, or unspecified; no active adaptation classification."},"licence_caveat":"Licences are read from the archived source repository or dataset card and describe that artefact, not necessarily every text inside it; public availability is not a blanket grant to republish the underlying benchmark content.","edition":"release-v4","label":"Decision Index 0.4.1","base_edition":"release-v2.1","note":null},"excluded_questions":{"purpose":"Questions dropped from scoring for every engine so no model is graded on a case another model was structurally unable to attempt.","frozen_corpus":"The frozen corpus and its sha256 are unchanged; exclusion is applied at scoring time only.","criterion":"A row is excluded for reach only when nearly the whole field cannot attempt it. A limit that one model has and others do not is that model's result, not a defect in the suite, and is never grounds for removing the row.","provenance":"Defects verified against the upstream sources; cais/mmlu ships the duplicate choices verbatim and the builder applies no normalisation to MMLU.","rules":[{"reason":"duplicate_options","n":17,"rule":"Two or more options are byte-identical. A distinct-options contract (Solomon) refuses these outright, so scoring them rewards engines with looser input validation."},{"reason":"gold_duplicated","n":6,"rule":"The correct answer is itself one of the identical strings, so a textually correct choice is scored wrong at chance. Unanswerable."},{"reason":"whitespace_duplicate_options","n":3,"rule":"MuSR ships \"pantry\" and \"pantry \" as separate locations; the trailing space carries no task meaning."},{"reason":"gold_whitespace_duplicated","n":1,"rule":"As above, and the gold is the space-variant."},{"reason":"beyond_nearly_every_context_window","n":380,"rule":"Rows longer than almost every entrant can accept. Measured across the 27 completed sweeps, 14 answer none of them and only 4 answer all; the rest manage a fifth. They cannot separate models, and the group-completeness rule already drops them for most engines, so excluding them uniformly removes an incomparability rather than a result.","datasets":{"ToolRet":293,"BRIGHT":87},"engines_answering_all":4,"engines_answering_none":14,"engines_measured":27},{"reason":"sibling_chunk_of_an_excluded_group","n":35,"rule":"ToolRet partitions a query across chunks. Dropping only the over-long chunks would leave the query scored against a reduced candidate set, which is an easier question than the benchmark asks, so the surviving chunks of a partly excluded query are removed with it.","datasets":{"ToolRet":35}}],"counts":{"rows":442,"all_single_question":true,"by_dataset":{"HLE":12,"MMLU":9,"MuSR":4,"GPQA Diamond":2},"share_of_choice_questions":3.48e-05,"by_reason":{"defective":27,"beyond_reach":380,"sibling_chunks":35}},"deliberately_not_excluded":{"CRUXEval":"40 questions whose options differ only by case or whitespace are legitimate: they are Python values, and the case and spacing are exactly what the task tests.","MMLU/ARC/GPQA case collisions":"genotype notation (BB/Bb/bb), logic predicates (Cs/sC) and variable names (l/L) are semantic; only a case-blind comparison makes them look duplicated."},"by_dataset_and_reason":[{"dataset":"ToolRet","catalog_id":2,"reason":"beyond_nearly_every_context_window","n":293},{"dataset":"BRIGHT","catalog_id":36,"reason":"beyond_nearly_every_context_window","n":87},{"dataset":"ToolRet","catalog_id":2,"reason":"sibling_chunk_of_an_excluded_group","n":35},{"dataset":"HLE","catalog_id":45,"reason":"duplicate_options","n":10},{"dataset":"MMLU","catalog_id":24,"reason":"duplicate_options","n":5},{"dataset":"MMLU","catalog_id":24,"reason":"gold_duplicated","n":4},{"dataset":"MuSR","catalog_id":32,"reason":"whitespace_duplicate_options","n":3},{"dataset":"GPQA Diamond","catalog_id":25,"reason":"duplicate_options","n":2},{"dataset":"HLE","catalog_id":45,"reason":"gold_duplicated","n":2},{"dataset":"MuSR","catalog_id":32,"reason":"gold_whitespace_duplicated","n":1}]},"benchmarks":[{"id":1,"dataset":"BFCL","short":null,"kind":"static","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":"tools","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"case exact accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Says yes or no for each supplied function: call it for this request? All must be right.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2305.15334"},{"kind":"code","url":"https://github.com/gorilla-llm/gorilla"}],"cases":1694,"requests":1694,"fields":4768,"source_cases":1694,"unit":"case","scoring_metric":"tool-name exact-set accuracy","subset_rule":"All 1,694 prepared cases retained.","source_filtering":null,"note":"Published candidate schemas projected to tool-name decisions. Arguments/order/execution are not evaluated. Source-to-label gold audit required in addition to schema checks. Release v1: All prepared cases retained.","suite_category":"Tool use, routing and automation","subcategory":"Tool-name selection","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Projects structured calls to tool-name decisions; source argument/execution task is reduced.","candidate_origin":"Published tool schemas"},"random_baseline":{"value":0.2592,"label":"Random whole-case guesses","kind":"random_exact_case","method":"Exact expectation","explanation":"Mean probability that every field in a case is correct under independent uniform guesses; multiply field probabilities within each case, then average cases. This is not the per-field chance rate.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.964},"sources":[{"name":"gorilla-llm/gorilla","url":"https://github.com/gorilla-llm/gorilla.git","revision":"916260dfc116bf06793a1af79b4ec8195b0453b6"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"916260dfc116bf06793a1af79b4ec8195b0453b6"},{"id":2,"dataset":"ToolRet","short":"ToolRet","kind":"static","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":"tools","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"nDCG@10","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Rates how useful each of 32 search-hit tools is for a request; ranked by nDCG@10, on requests where at least one of the 32 is relevant.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2503.01763"},{"kind":"dataset","url":"https://huggingface.co/datasets/mangopy/ToolRet-Queries"}],"cases":685,"requests":1013,"fields":254650,"source_cases":7961,"unit":"retrieval query","scoring_metric":"nDCG@10; Recall@10; MRR; candidate recall","subset_rule":"1,000 of 7,704 scoreable v1 queries kept (1,328 requests): stratified by source domain, proportional allocation with at least one query per domain (seed 20260922); kept queries keep their complete candidate lists; no outcome-based selection.","source_filtering":"Candidate policy: query-only lexical overlap top32; no gold insertion; unjudged candidates are not treated as verified negatives.","note":"Query-only FTS5 BM25 candidate retrieval plus Jev relevance ranking. Candidate recall and nDCG/MRR required; unjudged candidates are not verified negatives. See retrieval audit for readiness. Release v1: All prepared cases retained.","suite_category":"Tool use, routing and automation","subcategory":"Tool retrieval","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"FTS5 BM25 candidate search, bounded pool, context packing and relevance ranking.","candidate_origin":"Query-only search over published tool corpus"},"random_baseline":{"value":0.1341,"label":"Random ranking, answerable queries","kind":"random_ranking","method":"Exact expectation, answerable queries","explanation":"Mean over the scored queries of the expected nDCG@10 under a uniform random permutation of that query's 32 candidates, using the full relevance judgments for the denominator; only queries with at least one relevant candidate are scored. This is the value the index uses.","monte_carlo_standard_error":null,"secondary_reference":{"label":"BM25 ranking","value":0.3717}},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.6505},"sources":[{"name":"mangopy/tool-retrieval-benchmark","url":"https://github.com/mangopy/tool-retrieval-benchmark.git","revision":"c4181d914a227134705ecb6bab13fbd92ccd2938"},{"name":"mangopy/ToolRet-Queries","url":"https://huggingface.co/datasets/mangopy/ToolRet-Queries","revision":null},{"name":"mangopy/ToolRet-Tools","url":"https://huggingface.co/datasets/mangopy/ToolRet-Tools","revision":null}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"c4181d914a227134705ecb6bab13fbd92ccd2938"},{"id":3,"dataset":"API-Bank","short":null,"kind":"static","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Reads an assistant conversation and picks the next API to call from a catalog of 53.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2304.08244"},{"kind":"code","url":"https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank"}],"cases":508,"requests":508,"fields":508,"source_cases":508,"unit":"API decision point","scoring_metric":"tool-name selection accuracy","subset_rule":"All 508 prepared cases retained.","source_filtering":"508 API decision points from 264 source files over the full 53-tool catalog; 3 excluded; every gold answer verified to be a candidate.","note":"508 source API decision points, full53-tool published catalog; prior observed results allowed, current/future results hidden. Tool-name selection only, not argument/execution success. Release v1: All prepared cases retained.","suite_category":"Tool use, routing and automation","subcategory":"Sequential tool-name selection","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Extracts each source API decision and preceding dialogue; arguments/execution excluded.","candidate_origin":"Published API catalog"},"random_baseline":{"value":0.0189,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.8661},"sources":[{"name":"AlibabaResearch/DAMO-ConvAI","url":"https://github.com/AlibabaResearch/DAMO-ConvAI.git","revision":"188835d4f9948563a6b9c8ac50cd0f3ae4021ed6"}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"},{"licence":"Apache-2.0","source":"archived source file","file":"api-bank/LICENSE"}],"source_revision":"188835d4f9948563a6b9c8ac50cd0f3ae4021ed6"},{"id":4,"dataset":"BANKING77","short":null,"kind":"static","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Picks the intent of a bank customer's message, e.g. \"How do I locate my card?\", from 77.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2003.04807"},{"kind":"dataset","url":"https://github.com/PolyAI-LDN/task-specific-datasets"}],"cases":3080,"requests":3080,"fields":3080,"source_cases":3080,"unit":"case","scoring_metric":"intent accuracy; macro-F1","subset_rule":"All 3,080 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Language understanding and classification","subcategory":"Banking intent","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native intent classification with the released label vocabulary.","candidate_origin":"Original77 intent labels"},"random_baseline":{"value":0.0127,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":6.55e-05,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7994},"sources":[{"name":"PolyAI/banking77","url":"https://huggingface.co/datasets/PolyAI/banking77","revision":null}],"licences":[{"licence":"CC-BY-4.0","source":"archived dataset card","file":"README.md"}],"source_revision":null},{"id":5,"dataset":"CLINC150+OOS","short":"CLINC150","kind":"static","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Reads a short assistant request and picks its intent from 150, or \"out of scope\" when none fits.","comparability_note":null,"references":[{"kind":"dataset","url":"https://github.com/clinc/oos-eval"}],"cases":5500,"requests":5500,"fields":5500,"source_cases":5500,"unit":"case","scoring_metric":"intent accuracy; OOS F1","subset_rule":"All 5,500 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Language understanding and classification","subcategory":"Intent and out-of-scope detection","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native intent/OOS classification; same fixed label set for every example.","candidate_origin":"Original150 intents plus OOS"},"random_baseline":{"value":0.006,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":3.34e-05,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.9028},"sources":[{"name":"clinc/oos-eval","url":"https://github.com/clinc/oos-eval","revision":null}],"licences":[],"source_revision":null},{"id":6,"dataset":"RouterBench","short":"RouterBench","kind":"static","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":"tools","in_panel":false,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"selected quality (quality objective)","lower_is_better":false,"index_rule":null,"left_out_of_index":true,"index_note":"Not in the index: the prompt reveals that GPT-4 is the best route, so the top models all pick GPT-4 and score within a fraction of a point of each other. Scores stay visible.","explainer":"Picks which of 11 AI models should answer a prompt; scored by its real answer.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2403.12031"},{"kind":"dataset","url":"https://huggingface.co/datasets/withmartian/routerbench"}],"cases":5001,"requests":10000,"fields":20000,"source_cases":29163,"unit":"underlying routed prompt","scoring_metric":"realized quality; cost; utility regret","subset_rule":"10,000 requests within a 10,000-request cap: 5,001 complete linked case groups of 29,163 prepared; groups kept intact.","source_filtering":"Offline released outcomes; calibration-derived profiles only. No live candidate model calls. Both shot regimes retained; report separately and cluster by source group. 28 exclusions recorded; cost penalty 10.0 per USD.","note":"Both0-shot/5-shot historical outcome tables; shared hash-grouped20% calibration split, remaining cases evaluated. Profile quality/cost derived only from calibration. Custom quality/cost utility scorer accepts ties; no live candidate calls. Release v1: 10000 requests within 10000 cap; complete linked cases retained.","suite_category":"Tool use, routing and automation","subcategory":"Model routing","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Calibration-only profiles, explicit quality/cost objectives, held-out outcome scoring.","candidate_origin":"Original11 candidate models"},"random_baseline":{"value":0.5735,"label":"Random model selection","kind":"random_routing","method":"Exact expectation","explanation":"Expected realized quality from selecting uniformly among the available models on each request. This is outcome quality, not the probability of matching one correct answer.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[{"track":"RouterBench-0shot","label":"Zero-shot outcome table","metric":"Realized selected-model quality","description":"Choose a model using profiles built from a separate calibration split; evaluate against recorded zero-shot outcomes. Each request tests quality-only and cost-aware objectives.","random":0.5245,"headline":false,"in_index":true},{"track":"RouterBench-5shot","label":"Five-shot outcome table","metric":"Realized selected-model quality","description":"The same routing decision, evaluated against recorded outcomes from models given five examples. Jev is not making live calls to the candidate models.","random":0.6225,"headline":false,"in_index":true}],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7988},"sources":[{"name":"withmartian/routerbench","url":"https://github.com/withmartian/routerbench.git","revision":"cc67d1008bd8f3cf1e8040cc3ba4034d31b93c0c"},{"name":"withmartian/routerbench","url":"https://huggingface.co/datasets/withmartian/routerbench","revision":null}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"cc67d1008bd8f3cf1e8040cc3ba4034d31b93c0c"},{"id":7,"dataset":"MiniWoB++","short":"MiniWoB++","kind":"interactive","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":"tools","in_panel":false,"in_frozen_panel":true,"dropped_interactive":true,"in_reproduction_suite":false,"metric":"episode success","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":9,"requests":null,"fields":null,"source_cases":null,"unit":"episode","scoring_metric":"native episode success","subset_rule":"9 configured episodes; fixed seeds; no performance-based replacement.","source_filtering":null,"note":"Dynamic request count depends on trajectory length. Configured count remains user-selectable. implemented; native headless browser smoke passed","suite_category":"Computer use and embodied decisions","subcategory":"Browser interaction","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Click-only task filtering and native browser action enumeration.","candidate_origin":"Visible DOM actions"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"interval_95":null,"interval_method":null,"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"Farama-Foundation/miniwob-plusplus","url":"https://github.com/Farama-Foundation/miniwob-plusplus.git","revision":"33c3b4ddef8c6eb67c57a29663d844b1eda7e614"}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"33c3b4ddef8c6eb67c57a29663d844b1eda7e614","protocol":null},{"id":9,"dataset":"Home appliance simulator","short":"Home appliances","kind":"static","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"case exact accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given a simulated home and a command, answers ~18 questions on handling it; all must be right. Duplicate and dev-set rows removed.","comparability_note":null,"references":[{"kind":"code","url":"https://github.com/apolinario/decision-index/blob/main/decision_index/suite/build/home_appliance.py"}],"cases":88,"requests":88,"fields":2895,"source_cases":160,"unit":"household request","scoring_metric":"whole-request exact match; executable state correctness; per-field accuracy","subset_rule":"All 160 prepared cases retained.","source_filtering":null,"note":"160 test households/requests, 2895 fields; separate 80-case dev file. Executable intent oracle and semantic validation; structural novelty does not establish pretraining or semantic independence. Release v1: All prepared cases retained.","suite_category":"Computer use and embodied decisions","subcategory":"Home appliance control","origin":"Purpose-built benchmarks","adaptation":{"class":"C_semantic_options","class_description":"The adaptation relies on authored semantic option spaces/scenarios or model-generated candidate answers absent from the original native answer format.","reason":"Custom household scenarios and decision fields authored for this suite, with executable oracle. Not a native public benchmark.","candidate_origin":"Locally authored device/command scenarios"},"random_baseline":{"value":0,"label":"Random whole-case guesses","kind":"random_exact_case","method":"Exact expectation, kept rows","explanation":"Mean probability that every field in a case is correct under independent uniform guesses, over the kept rows.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.4545},"sources":[{"name":"purpose-built locally (evaluation/home_benchmark)","url":null,"revision":null}],"licences":[],"source_revision":null},{"id":10,"dataset":"SGD/SGD-X","short":"SGD","kind":"static","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":true,"index_note":"Not in the index: build bug, pending re-run. The builder gave every SGD row the whole dialogue, including the turns after the one being classified. Scores stay visible; SGD returns to the index once re-run with the fixed builder.","explainer":"Reads a service dialogue so far and picks the intent the user is pursuing now, or NONE.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/1909.05855"},{"kind":"dataset","url":"https://github.com/google-research-datasets/dstc8-schema-guided-dialogue"}],"cases":2500,"requests":2500,"fields":2500,"source_cases":46116,"unit":"user-turn/service intent","scoring_metric":"service-given intent accuracy","subset_rule":"2,500 of 46,116 prepared cases, deterministic hash sample without replacement (seed 20260919); no outcome-based selection.","source_filtering":null,"note":"Service-given current-user-turn intent classification on official SGD test dialogues; SGD-X and freeform slot extraction are not yet evaluated. Release v1: 2500 seeded cases.","suite_category":"Language understanding and classification","subcategory":"Dialogue intent","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Selects current user turns, preserves preceding history, and supplies the service; not full dialogue-state generation.","candidate_origin":"Original service schemas and intent labels"},"random_baseline":{"value":0.399,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.00036,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.0903},"sources":[{"name":"google-research-datasets/dstc8-schema-guided-dialogue","url":"https://github.com/google-research-datasets/dstc8-schema-guided-dialogue.git","revision":"e852981ae34990f4358979625854259302feaa78"}],"licences":[{"licence":"CC-BY-SA-4.0","source":"archived source file","file":"LICENSE.txt"}],"source_revision":"e852981ae34990f4358979625854259302feaa78"},{"id":11,"dataset":"ContractNLI","short":null,"kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":"language","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Says whether a full NDA supports, contradicts or omits each of 17 fixed statements.","comparability_note":null,"references":[{"kind":"paper","url":"https://aclanthology.org/2021.findings-emnlp.164"},{"kind":"dataset","url":"https://stanfordnlp.github.io/contract-nli/"}],"cases":123,"requests":123,"fields":2091,"source_cases":123,"unit":"case","scoring_metric":"macro-F1; accuracy","subset_rule":"All 123 prepared cases retained.","source_filtering":"123 source cases, 2,091 decisions.","note":"123 test contracts, 17 original hypotheses each. Native classification track; evidence extraction not yet adapted. Release v1: All prepared cases retained.","suite_category":"Document reasoning and retrieval","subcategory":"Contract entailment","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Released document/hypothesis classification projected directly; evidence extraction not claimed.","candidate_origin":"Original hypotheses and NLI labels"},"random_baseline":{"value":0.3085,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.000301,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7076},"sources":[{"name":"stanfordnlp/contract-nli","url":"https://github.com/stanfordnlp/contract-nli.git","revision":"eced6528dd3c1d14d73f9a87df8f7bdbc03126f9"}],"licences":[{"licence":"CC-BY-4.0","source":"archived source file","file":"LICENSE"}],"source_revision":"eced6528dd3c1d14d73f9a87df8f7bdbc03126f9"},{"id":12,"dataset":"ANLI","short":null,"kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Decides whether a claim follows from, contradicts, or is neutral to a premise written to fool AI models.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/1910.14599"},{"kind":"dataset","url":"https://huggingface.co/datasets/facebook/anli"}],"cases":3200,"requests":3200,"fields":3200,"source_cases":3200,"unit":"case","scoring_metric":"accuracy by ANLI round","subset_rule":"All 3,200 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Language understanding and classification","subcategory":"Adversarial entailment","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native three-way entailment classification.","candidate_origin":"Original3 NLI labels"},"random_baseline":{"value":0.3324,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.000254,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7427},"sources":[{"name":"facebookresearch/anli","url":"https://github.com/facebookresearch/anli","revision":null}],"licences":[{"licence":"CC-BY-NC-4.0","source":"archived dataset card","file":"README.md"}],"source_revision":null},{"id":13,"dataset":"Boxoban","short":"Boxoban","kind":"interactive","area":"games","area_label":"Games & Strategy","area_color":"#ad6f48","frozen_panel_area":"games","in_panel":false,"in_frozen_panel":true,"dropped_interactive":true,"in_reproduction_suite":false,"metric":"levels solved within 255 moves","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":100,"requests":null,"fields":null,"source_cases":null,"unit":"level","scoring_metric":"fraction of levels solved; moves and pushes","subset_rule":"100 hard Boxoban levels reused unchanged from the existing pilot; each scored within a 255-move budget.","source_filtering":null,"note":"Dynamic request count depends on trajectory length. Configured count remains user-selectable. implemented","suite_category":"Games and strategic decisions","subcategory":"Sokoban planning","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Native levels and deterministic transitions; score level completion, not a fabricated next-move label.","candidate_origin":"Simulator movement actions"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"mean_best_goal_fraction":null,"score":null},"sources":[{"name":"google-deepmind/boxoban-levels","url":"https://github.com/google-deepmind/boxoban-levels.git","revision":"78011d8c86ad423bfb2e1005012393b4c3c24409"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"78011d8c86ad423bfb2e1005012393b4c3c24409"},{"id":14,"dataset":"BabyAI/MiniGrid","short":null,"kind":"interactive","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"episode success","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":100,"requests":null,"fields":null,"source_cases":null,"unit":"episode","scoring_metric":"native success rate; reward","subset_rule":"12 configured episodes; fixed seeds; no performance-based replacement.","source_filtering":null,"note":"Dynamic request count depends on trajectory length. Configured count remains user-selectable. implemented; requires benchmark venv","suite_category":"Computer use and embodied decisions","subcategory":"Instruction-following navigation","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Simulator wrapper with partial observation and native success.","candidate_origin":"Native MiniGrid actions"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"interval_95":null,"interval_method":null,"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"Farama-Foundation/Minigrid","url":"https://github.com/Farama-Foundation/Minigrid.git","revision":"8ea099e114b7d6465afabc76957e3d689663534c"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"8ea099e114b7d6465afabc76957e3d689663534c","protocol":null},{"id":15,"dataset":"RTFM","short":"RTFM","kind":"interactive","area":"games","area_label":"Games & Strategy","area_color":"#ad6f48","frozen_panel_area":"games","in_panel":false,"in_frozen_panel":true,"dropped_interactive":true,"in_reproduction_suite":false,"metric":"Native win rate (100-turn pilot)","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":100,"requests":null,"fields":null,"source_cases":null,"unit":"episode","scoring_metric":"episode win rate","subset_rule":"100 configured episodes; Fixed seeds 0..N-1; no performance-based replacement. Codenames uses each of66 eligible boards exactly once.","source_filtering":null,"note":"Dynamic request count depends on trajectory length. Configured count remains user-selectable. implemented; native source dynamics smoke passed","suite_category":"Computer use and embodied decisions","subcategory":"Manual-guided interaction","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Source simulator, held-out generation configuration and player-visible observation.","candidate_origin":"Native RTFM actions"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"interval_95":null,"interval_method":null,"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"facebookresearch/RTFM","url":"https://github.com/facebookresearch/RTFM.git","revision":"58f17955595b5a127c96d045d896fcbcc7d4b570"}],"licences":[{"licence":"CC-BY-NC-4.0","source":"archived source file","file":"LICENSE"}],"source_revision":"58f17955595b5a127c96d045d896fcbcc7d4b570","protocol":{"observation":"Native public observations plus explicit game rules and last12 public observations; hidden cards/roles omitted.","actions":"Full native option strings; one Jev decision followed by environment transition. One available action is automatic and logged.","sample":"Fixed seeds 0..N-1; no performance-based replacement. Codenames uses each of66 eligible boards exactly once.","partner":"Same seeded random teammate/opponent as pilot for Hanabi/Leduc; fixed lexical partner for Codenames.","scoring":"Native terminal team points for Hanabi; player0 net chips per hand for Leduc; native wins for RTFM; adapter wins for Codenames.","budget":{"room_shape":[10,10],"shuffle_wiki":true,"max_actions":100,"source_class":"GroupsNLDev","max_turns":100},"limitations":"Fixed-partner performance, not optimal partnership or exploitability. RTFM is a100-turn partial-observation track; Codenames is a lexical-partner adaptation. Intervals describe this protocol and sample, not all possible partners/boards."}},{"id":16,"dataset":"ScienceWorld","short":"ScienceWorld","kind":"interactive","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":"tools","in_panel":false,"in_frozen_panel":true,"dropped_interactive":true,"in_reproduction_suite":false,"metric":"episode success","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":10,"requests":null,"fields":null,"source_cases":null,"unit":"episode","scoring_metric":"native task completion reward","subset_rule":"10 configured episodes; fixed seeds; no performance-based replacement.","source_filtering":null,"note":"Dynamic request count depends on trajectory length. Configured count remains user-selectable. implemented; native JVM semantic probe passed","suite_category":"Computer use and embodied decisions","subcategory":"Scientific experimentation","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Simulator action enumeration and lossless hierarchy for more than255 commands.","candidate_origin":"Native legal commands"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"interval_95":null,"interval_method":null,"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"allenai/ScienceWorld","url":"https://github.com/allenai/ScienceWorld.git","revision":"e8216d6044e8e39be9fcb185e3b2dfb602584b52"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"e8216d6044e8e39be9fcb185e3b2dfb602584b52","protocol":null},{"id":17,"dataset":"Hanabi","short":"Hanabi","kind":"interactive","area":"games","area_label":"Games & Strategy","area_color":"#ad6f48","frozen_panel_area":"games","in_panel":false,"in_frozen_panel":true,"dropped_interactive":true,"in_reproduction_suite":false,"metric":"Mean team score / 25 with random teammate","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":100,"requests":null,"fields":null,"source_cases":null,"unit":"episode","scoring_metric":"team score","subset_rule":"100 configured episodes; Fixed seeds 0..N-1; no performance-based replacement. Codenames uses each of66 eligible boards exactly once.","source_filtering":null,"note":"Dynamic request count depends on trajectory length. Configured count remains user-selectable. implemented; requires benchmark venv","suite_category":"Games and strategic decisions","subcategory":"Cooperative hidden-information cards","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Player-visible observation and fixed partner; native team return.","candidate_origin":"Native OpenSpiel legal actions"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"interval_95":null,"interval_method":null,"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"google-deepmind/open_spiel","url":"https://github.com/google-deepmind/open_spiel.git","revision":"48401890ee9857e611678302371378175a8e4c6b"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"48401890ee9857e611678302371378175a8e4c6b","protocol":{"observation":"Native public observations plus explicit game rules and last12 public observations; hidden cards/roles omitted.","actions":"Full native option strings; one Jev decision followed by environment transition. One available action is automatic and logged.","sample":"Fixed seeds 0..N-1; no performance-based replacement. Codenames uses each of66 eligible boards exactly once.","partner":"Same seeded random teammate/opponent as pilot for Hanabi/Leduc; fixed lexical partner for Codenames.","scoring":"Native terminal team points for Hanabi; player0 net chips per hand for Leduc; native wins for RTFM; adapter wins for Codenames.","budget":{"game":"hanabi","players":2,"opponent":"seeded random","rules":"standard OpenSpiel Hanabi; player 0 information state only","max_actions":255,"max_turns":100},"limitations":"Fixed-partner performance, not optimal partnership or exploitability. RTFM is a100-turn partial-observation track; Codenames is a lexical-partner adaptation. Intervals describe this protocol and sample, not all possible partners/boards."}},{"id":18,"dataset":"Leduc poker","short":null,"kind":"interactive","area":"games","area_label":"Games & Strategy","area_color":"#ad6f48","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"Mean net chips per hand versus random opponent","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":300,"requests":null,"fields":null,"source_cases":null,"unit":"episode","scoring_metric":"player-0 mean return","subset_rule":"300 configured episodes; Fixed seeds 0..N-1; no performance-based replacement. Codenames uses each of66 eligible boards exactly once.","source_filtering":null,"note":"Dynamic request count depends on trajectory length. Configured count remains user-selectable. implemented; requires benchmark venv","suite_category":"Games and strategic decisions","subcategory":"Imperfect-information poker","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Player information state and fixed opponent; native game return.","candidate_origin":"Native OpenSpiel legal actions"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"interval_95":null,"interval_method":null,"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"google-deepmind/open_spiel","url":"https://github.com/google-deepmind/open_spiel.git","revision":"48401890ee9857e611678302371378175a8e4c6b"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"48401890ee9857e611678302371378175a8e4c6b","protocol":{"observation":"Native public observations plus explicit game rules and last12 public observations; hidden cards/roles omitted.","actions":"Full native option strings; one Jev decision followed by environment transition. One available action is automatic and logged.","sample":"Fixed seeds 0..N-1; no performance-based replacement. Codenames uses each of66 eligible boards exactly once.","partner":"Same seeded random teammate/opponent as pilot for Hanabi/Leduc; fixed lexical partner for Codenames.","scoring":"Native terminal team points for Hanabi; player0 net chips per hand for Leduc; native wins for RTFM; adapter wins for Codenames.","budget":{"game":"leduc_poker","opponent":"seeded random","rules":"standard OpenSpiel Leduc; chance outcomes seeded independently in metadata","max_actions":255,"max_turns":100},"limitations":"Fixed-partner performance, not optimal partnership or exploitability. RTFM is a100-turn partial-observation track; Codenames is a lexical-partner adaptation. Intervals describe this protocol and sample, not all possible partners/boards."}},{"id":19,"dataset":"Codenames","short":"Codenames","kind":"interactive","area":"games","area_label":"Games & Strategy","area_color":"#ad6f48","frozen_panel_area":"games","in_panel":false,"in_frozen_panel":true,"dropped_interactive":true,"in_reproduction_suite":false,"metric":"Win rate with fixed lexical clue partner","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":null,"comparability_note":null,"references":[],"cases":66,"requests":null,"fields":null,"source_cases":130,"unit":"episode","scoring_metric":"lexical-partner win rate; turns; safe guesses","subset_rule":"All 66 eligible source boards, one episode each, with a fixed WordNet lexical clue partner.","source_filtering":"130 source boards → 66 eligible with full own-word safe-clue coverage under WordNet 3.0; 64 excluded.","note":"66 unique eligible source boards; one episode per board in the lexical-partner benchmark.","suite_category":"Games and strategic decisions","subcategory":"Word association","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"WordNet rule-based clue partner and eligibility filtering; no model-authored clues or invented answer words. Named adaptation.","candidate_origin":"Original board words plus end-turn"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"metric":null,"episodes":null,"interval_95":null,"interval_method":null,"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"clembench/clembench","url":"https://github.com/clembench/clembench.git","revision":"adff278de1cf1cbc30a93f3000347b102cdfc0c5"},{"name":"aclanthology.org/2025.gem-1.63.pdf","url":"https://aclanthology.org/2025.gem-1.63.pdf","revision":null}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"adff278de1cf1cbc30a93f3000347b102cdfc0c5","protocol":{"observation":"Native public observations plus explicit game rules and last12 public observations; hidden cards/roles omitted.","actions":"Full native option strings; one Jev decision followed by environment transition. One available action is automatic and logged.","sample":"Fixed seeds 0..N-1; no performance-based replacement. Codenames uses each of66 eligible boards exactly once.","partner":"Same seeded random teammate/opponent as pilot for Hanabi/Leduc; fixed lexical partner for Codenames.","scoring":"Native terminal team points for Hanabi; player0 net chips per hand for Leduc; native wins for RTFM; adapter wins for Codenames.","budget":{"max_actions":100,"max_turns":100},"limitations":"Fixed-partner performance, not optimal partnership or exploitability. RTFM is a100-turn partial-observation track; Codenames is a lexical-partner adaptation. Intervals describe this protocol and sample, not all possible partners/boards."}},{"id":20,"dataset":"BPoMP","short":null,"kind":"static","area":"arts","area_label":"Arts & Human Taste","area_color":"#b6698a","frozen_panel_area":"arts","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given two versions of a limerick, one with its rhyme, meter or sense broken, picks the original.","comparability_note":null,"references":[{"kind":"dataset","url":"https://zenodo.org/records/7299879"}],"cases":811,"requests":5000,"fields":5000,"source_cases":22419,"unit":"underlying poem (paired requests)","scoring_metric":"original-limerick discrimination; macro by perturbation","subset_rule":"5,000 requests within a 5,000-request cap: 811 complete linked case groups of 22,419 prepared; groups kept intact.","source_filtering":"134,005 original/perturbed limerick pairs across 14 perturbation variants; 9 indistinguishable pairs excluded; 5,000 requests sampled as complete poem groups.","note":"Original-versus-perturbed limerick discrimination across14 source variants. Exact ties excluded, labels shuffled. Macro by perturbation and poem-clustered uncertainty; not human aesthetic preference. Release v1: 5000 requests within 5000 cap; complete linked cases retained.","suite_category":"Arts and human preferences","subcategory":"Poetry perturbations","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Filters indistinguishable pairs and randomizes order; no new poem options or aesthetic gold.","candidate_origin":"Released original/perturbed poem pairs"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.9006},"sources":[{"name":"zenodo.org/records/7299879","url":"https://zenodo.org/records/7299879","revision":null}],"licences":[],"source_revision":null},{"id":21,"dataset":"Humicroedit","short":null,"kind":"static","area":"arts","area_label":"Arts & Human Taste","area_color":"#b6698a","frozen_panel_area":"arts","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given two one-word edits of a news headline, picks the one people rated funnier.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/1906.00274"},{"kind":"dataset","url":"https://cs.rochester.edu/u/nhossain/humicroedit.html"}],"cases":2628,"requests":2628,"fields":2628,"source_cases":2628,"unit":"headline-edit pair","scoring_metric":"human-preferred edit accuracy excluding source ties","subset_rule":"All 2,628 prepared cases retained.","source_filtering":"2,960 official subtask-2 test pairs → 2,628; 332 equal-rating ties excluded (IDs recorded).","note":"Official full subtask2 test source has2960 pairs; scorer excludes332 equal-rating ties, leaving2628 scored requests. Tie IDs recorded. Release v1: All prepared cases retained.","suite_category":"Arts and human preferences","subcategory":"Humor preference","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Uses human-rated source pairs; equal-rating ties excluded from forced-choice score.","candidate_origin":"Released paired headline edits"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.6176},"sources":[{"name":"n-hossain/semeval-2020-task-7-humicroedit","url":"https://github.com/n-hossain/semeval-2020-task-7-humicroedit.git","revision":"7c0921e0aaf9fc671b17b68013a9ceb9edd42944"}],"licences":[],"source_revision":"7c0921e0aaf9fc671b17b68013a9ceb9edd42944"},{"id":22,"dataset":"POP909-CL","short":"POP909","kind":"static","area":"arts","area_label":"Arts & Human Taste","area_color":"#b6698a","frozen_panel_area":"arts","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given the notes around a beat in a pop song (not audio), picks the chord playing from 129.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2510.06528"},{"kind":"dataset","url":"https://github.com/AndyWeasley2004/POP909-CL-Dataset"}],"cases":2000,"requests":2000,"fields":2000,"source_cases":75582,"unit":"target chord window","scoring_metric":"chord pitch-class recognition; song macro","subset_rule":"2,000 of 75,582 prepared cases, deterministic hash sample without replacement (seed 20260919); no outcome-based selection.","source_filtering":"907 of 909 songs eligible (excluded: 518: algorithmic labels per source README; 620: known score/annotation misalignment); 75,582 chord windows, 129 fixed pitch-class labels; 2,000 sampled.","note":"907 source songs, symbolic chord recognition at fixed beat positions. Score channel0 visible; expert chord channel1 hidden.129 fixed pitch-class labels; song-clustered metrics, NoChord/Other reported separately. Release v1: 2000 seeded cases.","suite_category":"Arts and human preferences","subcategory":"Symbolic music harmony","origin":"Jev-adapted benchmarks","adaptation":{"class":"C_semantic_options","class_description":"The adaptation relies on authored semantic option spaces/scenarios or model-generated candidate answers absent from the original native answer format.","reason":"Musical judgment defines129 chord pitch-class categories and windows; gold comes from released expert annotation, not new model judgments.","candidate_origin":"Root-designed fixed chord vocabulary"},"random_baseline":{"value":0.0078,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":{"label":"Always predict GM (sample majority)","value":0.051,"uses_evaluation_labels":true,"explanation":"Descriptive class-imbalance reference selected using this evaluation sample; not a independently chosen predictive baseline."}},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.1645},"sources":[{"name":"AndyWeasley2004/POP909-CL-Dataset","url":"https://github.com/AndyWeasley2004/POP909-CL-Dataset.git","revision":"be9094392903c471a930519e1c0bacf8b6be5d62"}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"be9094392903c471a930519e1c0bacf8b6be5d62"},{"id":23,"dataset":"cfcolor","short":null,"kind":"static","area":"arts","area_label":"Arts & Human Taste","area_color":"#b6698a","frozen_panel_area":"arts","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given a person's earlier ratings of color palettes, predicts which of two new palettes they rated higher.","comparability_note":null,"references":[{"kind":"dataset","url":"https://www.dgp.toronto.edu/~donovan/cfcolor/"}],"cases":5000,"requests":5000,"fields":5000,"source_cases":11821,"unit":"held-out palette pair","scoring_metric":"human preference agreement; user macro","subset_rule":"5,000 of 11,821 prepared cases, deterministic hash sample without replacement (seed 20260919); no outcome-based selection.","source_filtering":"11,821 held-out pairs from 869 users with rating gap ≥ 2 (gap 2: 8,733, 3: 2,546, 4: 542); 5,000 sampled.","note":"Held-out human palette-preference pairs with original train-rating history. Gap>=2, nonoverlapping pairs, visible RGB duplicates excluded; user-macro accuracy. Human preference, not objective beauty. Release v1: 5000 seeded cases.","suite_category":"Arts and human preferences","subcategory":"Personalized color preference","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Deterministic held-out pairing and train-history selection; real human rating gap defines gold.","candidate_origin":"Released rated palettes"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.6634},"sources":[{"name":"www.dgp.toronto.edu/~donovan/cfcolor/","url":"https://www.dgp.toronto.edu/~donovan/cfcolor/","revision":null}],"licences":[],"source_revision":null},{"id":24,"dataset":"MMLU","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":"knowledge","in_panel":false,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":true,"explainer":"Four-option questions across 57 school and professional subjects, from law to algebra.","comparability_note":null,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/cais/mmlu"}],"cases":14042,"requests":14042,"fields":14042,"source_cases":14042,"unit":"case","scoring_metric":"accuracy by subject; micro/macro","subset_rule":"All 14,042 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Knowledge and science","subcategory":"Multidomain knowledge","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native MMLU test questions.","candidate_origin":"Original multiple-choice options"},"random_baseline":{"value":0.25,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.9167},"sources":[{"name":"cais/mmlu","url":"https://huggingface.co/datasets/cais/mmlu","revision":null}],"licences":[],"source_revision":null},{"id":25,"dataset":"GPQA Diamond","short":"GPQA Diamond","kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":"knowledge","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"198 hard four-option science questions written by PhD experts in physics, chemistry and biology.","comparability_note":null,"references":[{"kind":"dataset","url":"https://github.com/idavidrein/gpqa"}],"cases":198,"requests":198,"fields":198,"source_cases":198,"unit":"case","scoring_metric":"accuracy","subset_rule":"All 198 prepared cases retained.","source_filtering":null,"note":"All198 Diamond questions converted locally. Encrypted original retained; do not upload benchmark content publicly. Release v1: All prepared cases retained.","suite_category":"Knowledge and science","subcategory":"Graduate science","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Official Diamond subset with native options.","candidate_origin":"Original multiple-choice options"},"random_baseline":{"value":0.25,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7551},"sources":[{"name":"Idavidrein/gpqa","url":"https://huggingface.co/datasets/Idavidrein/gpqa","revision":null}],"licences":[],"source_revision":null},{"id":26,"dataset":"ARC-Easy","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":true,"explainer":"Grade-school science multiple choice, the easier set.","comparability_note":null,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/ai2_arc"}],"cases":2376,"requests":2376,"fields":2376,"source_cases":2376,"unit":"case","scoring_metric":"accuracy","subset_rule":"All 2,376 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Knowledge and science","subcategory":"Elementary science","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native ARC-Easy choices.","candidate_origin":"Original multiple-choice options"},"random_baseline":{"value":0.2502,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.9933},"sources":[{"name":"allenai/ai2_arc","url":"https://huggingface.co/datasets/allenai/ai2_arc","revision":null}],"licences":[],"source_revision":null},{"id":27,"dataset":"ARC-Challenge","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":true,"explainer":"Grade-school science multiple choice, the harder set that simple retrieval gets wrong.","comparability_note":null,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/ai2_arc"}],"cases":1172,"requests":1172,"fields":1172,"source_cases":1172,"unit":"case","scoring_metric":"accuracy","subset_rule":"All 1,172 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Knowledge and science","subcategory":"Challenging science","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native ARC-Challenge choices.","candidate_origin":"Original multiple-choice options"},"random_baseline":{"value":0.2502,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.9761},"sources":[{"name":"allenai/ai2_arc","url":"https://huggingface.co/datasets/allenai/ai2_arc","revision":null}],"licences":[],"source_revision":null},{"id":28,"dataset":"WinoGrande","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#407e9e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Fills a blank with one of two options using common sense, e.g. who \"got the easier cases\".","comparability_note":null,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/allenai/winogrande"}],"cases":1267,"requests":1267,"fields":1267,"source_cases":1267,"unit":"case","scoring_metric":"accuracy","subset_rule":"All 1,267 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Reasoning, mathematics and code","subcategory":"Commonsense coreference","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native WinoGrande alternatives.","candidate_origin":"Original2 completion options"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.9108},"sources":[{"name":"allenai/winogrande","url":"https://huggingface.co/datasets/allenai/winogrande","revision":null}],"licences":[],"source_revision":null},{"id":29,"dataset":"HellaSwag","short":null,"kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Picks the most sensible of four continuations of an everyday scene.","comparability_note":null,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/Rowan/hellaswag"}],"cases":10042,"requests":10042,"fields":10042,"source_cases":10042,"unit":"case","scoring_metric":"accuracy","subset_rule":"All 10,042 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Reasoning, mathematics and code","subcategory":"Commonsense continuation","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native HellaSwag alternatives.","candidate_origin":"Original continuation options"},"random_baseline":{"value":0.25,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.9434},"sources":[{"name":"Rowan/hellaswag","url":"https://huggingface.co/datasets/Rowan/hellaswag","revision":null}],"licences":[],"source_revision":null},{"id":30,"dataset":"GSM8K","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":"knowledge","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Picks a math word problem's answer from 4, and separately 10, numbers, with no working.","comparability_note":null,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/openai/gsm8k"}],"cases":1319,"requests":2638,"fields":2638,"source_cases":1319,"unit":"math problem (two MC variants)","scoring_metric":"MC accuracy separately for 4-choice and 10-choice variants","subset_rule":"All 1,319 prepared cases retained (2,638 requests).","source_filtering":"1,319 test problems, each rendered as a 4-choice and a 10-choice request (2,638 requests); synthetic numeric distractors.","note":"1319 source test problems in both 4- and 10-choice variants (2638 requests). Decimal-valued synthetic distractors, shared source grouping. Adapted MC accuracy only, not native free-response accuracy. Release v1: All prepared cases retained.","suite_category":"Reasoning, mathematics and code","subcategory":"Arithmetic word problems","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Fixed numeric recipe creates4/10 choices from free-response answers. Requires no per-item reasoning, but is adapted MC accuracy, not native GSM8K.","candidate_origin":"Algorithmic numeric distractors"},"random_baseline":{"value":0.175,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[{"track":"GSM8K-4choice","label":"Four answer choices","metric":"Multiple-choice accuracy","description":"Each original word problem is paired with its correct numeric answer and three synthetic distractors. Random guessing scores 25%.","random":0.25,"headline":false,"in_index":true},{"track":"GSM8K-10choice","label":"Ten answer choices","metric":"Multiple-choice accuracy","description":"The same source problems with nine synthetic distractors. Random guessing scores 10%. Neither track is native free-response GSM8K.","random":0.1,"headline":false,"in_index":true}],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7301},"sources":[{"name":"openai/gsm8k","url":"https://huggingface.co/datasets/openai/gsm8k","revision":null}],"licences":[],"source_revision":null},{"id":31,"dataset":"ChessBench","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":"games","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given a chess position and every legal move, picks the strongest one; tied best moves count.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2402.04494"},{"kind":"code","url":"https://github.com/google-deepmind/searchless_chess"}],"cases":5000,"requests":5000,"fields":5000,"source_cases":62561,"unit":"chess position","scoring_metric":"best-move accuracy accepting reference ties; mean value regret","subset_rule":"5,000 of 61,833 prepared cases, deterministic hash sample without replacement (seed 20260919); no outcome-based selection.","source_filtering":"62,561 test positions → 61,833 with complete legal-move coverage (728 excluded); 3,654 positions accept tied best moves; 5,000 sampled.","note":"Official DeepMind ChessBench action-value test data. All legal moves, exact ties accepted by custom scorer, reference value regret. Incomplete move coverage and non-decision positions explicitly excluded; no source sampling. Release v1: 5000 seeded cases.","suite_category":"Games and strategic decisions","subcategory":"Chess move quality","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Group published move values by position, verify complete legal coverage, accept reference ties.","candidate_origin":"All legal moves from source positions"},"random_baseline":{"value":0.0819,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.1688},"sources":[{"name":"google-deepmind/searchless_chess","url":"https://github.com/google-deepmind/searchless_chess.git","revision":"90ae0e6b121673fc3079aaeffa047580bb600c0a"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"90ae0e6b121673fc3079aaeffa047580bb600c0a"},{"id":32,"dataset":"MuSR","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Reads a long story and answers one question needing clues from across it, e.g. who the murderer is.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2310.16049"},{"kind":"dataset","url":"https://github.com/Zayne-sprague/MuSR"}],"cases":752,"requests":756,"fields":756,"source_cases":756,"unit":"case","scoring_metric":"accuracy by task","subset_rule":"All 756 prepared cases retained.","source_filtering":null,"note":"Release v1: All prepared cases retained.","suite_category":"Reasoning, mathematics and code","subcategory":"Multistep reasoning","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native MuSR alternatives.","candidate_origin":"Original multiple-choice options"},"random_baseline":{"value":0.371,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7261},"sources":[{"name":"Zayne-sprague/MuSR","url":"https://github.com/Zayne-sprague/MuSR.git","revision":"b1f4d4168a9cfc6760e8b74d728e4516023dfaa5"},{"name":"TAUR-Lab/MuSR","url":"https://huggingface.co/datasets/TAUR-Lab/MuSR","revision":null}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"b1f4d4168a9cfc6760e8b74d728e4516023dfaa5"},{"id":33,"dataset":"SATA-Bench","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"case exact accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given a passage, a question and candidate answers, says yes or no to each; all must be right.","comparability_note":null,"references":[{"kind":"dataset","url":"https://github.com/sata-bench/sata-bench"}],"cases":1650,"requests":1650,"fields":15517,"source_cases":1650,"unit":"select-all source question","scoring_metric":"exact answer-set accuracy; option F1","subset_rule":"All 1,650 prepared cases retained.","source_filtering":"All 1,650 source questions projected to 15,517 binary option decisions; scored as exact answer sets.","note":"All1650 source questions projected to15517 binary fields. Primary scoring unit remains exact answer set per source question. Release v1: All prepared cases retained.","suite_category":"Reasoning, mathematics and code","subcategory":"Select-all knowledge/reasoning","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"One binary decision per supplied option preserves the original answer-set task; exact-set score.","candidate_origin":"Original select-all alternatives"},"random_baseline":{"value":0.0131,"label":"Random whole-case guesses","kind":"random_exact_case","method":"Exact expectation","explanation":"Mean probability that every field in a case is correct under independent uniform guesses; multiply field probabilities within each case, then average cases. This is not the per-field chance rate.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.2788},"sources":[{"name":"sata-bench/sata-bench","url":"https://github.com/sata-bench/sata-bench.git","revision":"371dd0c18fe75a96fbbcf2d1507ceeaf0d5263c5"}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"371dd0c18fe75a96fbbcf2d1507ceeaf0d5263c5"},{"id":36,"dataset":"BRIGHT","short":null,"kind":"static","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":"language","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"nDCG@10","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Rates how helpful each of 32 search-hit documents is for a hard question; ranked by nDCG@10, on questions where at least one of the 32 is relevant.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2407.12883"},{"kind":"dataset","url":"https://huggingface.co/datasets/xlangai/BRIGHT"}],"cases":220,"requests":307,"fields":44286,"source_cases":1384,"unit":"retrieval query","scoring_metric":"nDCG@k; candidate recall","subset_rule":"550 of 1,297 scoreable v1 queries kept (637 requests): stratified by source domain, proportional allocation with at least one query per domain (seed 20260922); kept queries keep their complete candidate lists; no outcome-based selection.","source_filtering":"Candidate policy: query-only lexical overlap top32; no gold insertion; unjudged candidates are not treated as verified negatives.","note":"Official corpus, queries, judgments and ancillary release files downloaded. Unjudged documents must not be treated as verified negatives. Release v1: All prepared cases retained.","suite_category":"Document reasoning and retrieval","subcategory":"Reasoning-intensive retrieval","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"BM25 retrieval, source exclusion lists, full-document context checks and probability ranking.","candidate_origin":"Query-only search over published document corpus"},"random_baseline":{"value":0.116,"label":"Random ranking, answerable queries","kind":"random_ranking","method":"Exact expectation, answerable queries","explanation":"Mean over the scored queries of the expected nDCG@10 under a uniform random permutation of that query's 32 candidates, using the full relevance judgments for the denominator; only queries with at least one relevant candidate are scored. This is the value the index uses.","monte_carlo_standard_error":null,"secondary_reference":{"label":"BM25 ranking","value":0.1113}},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.4705},"sources":[{"name":"brightbenchmark.github.io","url":"https://brightbenchmark.github.io","revision":null}],"licences":[{"licence":"CC-BY-4.0","source":"archived dataset card","file":"README.md"}],"source_revision":null},{"id":37,"dataset":"Amazon ESCI","short":"Amazon ESCI","kind":"static","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":"language","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Labels a product as an exact match, substitute, complement or irrelevant to a shopping query.","comparability_note":null,"references":[{"kind":"dataset","url":"https://github.com/amazon-science/esci-data"}],"cases":5000,"requests":5000,"fields":5000,"source_cases":638016,"unit":"query-product pair","scoring_metric":"4-class accuracy; macro-F1","subset_rule":"5,000 of 638,016 source test pairs, proportional locale × relevance subset (seed 20260919).","source_filtering":"638,016 source test pairs (us 425,762, es 93,347, jp 118,907; labels I 70,440, E 393,363, S 153,357, C 20,856); 5,000 sampled proportionally by locale × relevance.","note":"Release v1: proportional locale × relevance subset of 5,000 source test pairs; replaces the former 2,000-pair active limit.","suite_category":"Document reasoning and retrieval","subcategory":"Product relevance","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native query-product classification. The separately recorded2,000-case sampling policy does not change this decision format.","candidate_origin":"Original4 relevance labels"},"random_baseline":{"value":0.2027,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.00017,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.5718},"sources":[{"name":"amazon-science/esci-data","url":"https://github.com/amazon-science/esci-data.git","revision":"7916cdf6ab75a462e77f20ab40428a10923998d5"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"7916cdf6ab75a462e77f20ab40428a10923998d5"},{"id":38,"dataset":"ACOS","short":null,"kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"per-review F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Reads a restaurant or laptop review and says yes/no for each aspect-and-sentiment pair; scored by F1 per review, averaged over reviews.","comparability_note":null,"references":[{"kind":"paper","url":"https://aclanthology.org/2021.acl-long.29"},{"kind":"dataset","url":"https://github.com/NUSTM/ACOS"}],"cases":400,"requests":1565,"fields":318945,"source_cases":1399,"unit":"source review","scoring_metric":"category/sentiment exact-set accuracy; pair micro-F1","subset_rule":"All 1,399 prepared cases retained (5,479 requests).","source_filtering":null,"note":"Official test reviews projected to fixed category-by-sentiment binary questions; chunks group back to source review. This is not full tuple extraction. Release v1: All prepared cases retained.","suite_category":"Language understanding and classification","subcategory":"Aspect/category sentiment","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Projects extraction tuples to fixed category-sentiment presence decisions; not full span/tuple extraction.","candidate_origin":"Source categories crossed with3 source sentiment labels"},"random_baseline":{"value":0.031,"label":"Always yes · per-review F1","kind":"random_exact_case","method":"Exact, always yes","explanation":"Per-review F1 of answering yes to every aspect-and-sentiment pair, averaged over reviews. A fair coin on every pair scores slightly lower, so always-yes is the chance level.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.3314},"sources":[{"name":"NUSTM/ACOS","url":"https://github.com/NUSTM/ACOS.git","revision":"45d179a3dcc6a3dedd848d81b16f2552454805fe"}],"licences":[],"source_revision":"45d179a3dcc6a3dedd848d81b16f2552454805fe"},{"id":39,"dataset":"FinEntity","short":"FinEntity","kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Says whether a financial news snippet is positive, negative or neutral about a named firm.","comparability_note":null,"references":[{"kind":"paper","url":"https://aclanthology.org/2023.emnlp-main.956"},{"kind":"dataset","url":"https://github.com/yixuantt/FinEntity"}],"cases":979,"requests":979,"fields":2129,"source_cases":979,"unit":"source document","scoring_metric":"entity-given sentiment macro-F1; accuracy","subset_rule":"All 979 prepared cases retained.","source_filtering":null,"note":"979 original documents projected to entity-given sentiment; one conflicting exact span quarantined. Evaluation-only unsplit source, not an invented official test split. Release v1: All prepared cases retained.","suite_category":"Language understanding and classification","subcategory":"Financial entity sentiment","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Entity-given projection of extraction; conflicting source annotation quarantined.","candidate_origin":"Original sentiment labels, provided source entity spans"},"random_baseline":{"value":0.3201,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.000314,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.8768},"sources":[{"name":"yixuantt/FinEntity","url":"https://github.com/yixuantt/FinEntity.git","revision":"3b6cedc5485b669c2ed168f1d949f517636eb7b8"}],"licences":[],"source_revision":"3b6cedc5485b669c2ed168f1d949f517636eb7b8"},{"id":40,"dataset":"iSarcasmEval","short":"iSarcasmEval","kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":"language","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"Sarcasm F1 · track A, English","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Says yes or no: is this tweet sarcastic? Scored with F1 on sarcastic English tweets (track A); other tracks listed under each result.","comparability_note":null,"references":[{"kind":"paper","url":"https://aclanthology.org/2022.semeval-1.111"},{"kind":"dataset","url":"https://github.com/iabufarha/iSarcasmEval"}],"cases":4600,"requests":4600,"fields":11600,"source_cases":4600,"unit":"case","scoring_metric":"A sarcasm F1; B category macro-F1; C pairwise accuracy (report separately)","subset_rule":"All 4,600 prepared cases retained.","source_filtering":"All five official test tasks: iSarcasmEval-A-Ar 1,400, iSarcasmEval-A-En 1,400, iSarcasmEval-B-En 1,400, iSarcasmEval-C-Ar 200, iSarcasmEval-C-En 200. Track B is excluded from the index; track A English is the headline.","note":"All five official test tasks converted; primary metrics differ: sarcasm F1, label macro-F1, pairwise accuracy. Release v1: All prepared cases retained.","suite_category":"Language understanding and classification","subcategory":"Sarcasm and irony","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native SemEval task labels and supplied paired utterances, scored separately by subtask.","candidate_origin":"Original binary/multilabel/pair alternatives"},"random_baseline":{"value":0.2227,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.00047,"secondary_reference":null},"tracks":[{"track":"iSarcasmEval-A-En","label":"A · English","metric":"Sarcasm F1","description":"Decide whether a single English text is sarcastic. The headline uses F1 for the sarcastic class, balancing missed sarcasm against false alarms; it is not overall accuracy.","random":0.2227,"headline":true,"in_index":true},{"track":"iSarcasmEval-A-Ar","label":"A · Arabic","metric":"Sarcasm F1","description":"The same single-text sarcasm judgment in Arabic, scored with F1 for the sarcastic class.","random":0.2225,"headline":false,"in_index":true},{"track":"iSarcasmEval-B-En","label":"B · English figurative","metric":"Six-label macro-F1","description":"Identify sarcasm, irony, satire, understatement, overstatement, and rhetorical questions. Multiple labels may apply; the metric averages positive-class F1 across the six labels.","random":0.0547,"headline":false,"in_index":false},{"track":"iSarcasmEval-C-En","label":"C · English pairs","metric":"Pairwise accuracy","description":"Given a sarcastic text and its non-sarcastic rephrasing, choose the sarcastic one. This provides a contrast that task A does not; random guessing scores 50%.","random":0.5,"headline":false,"in_index":true},{"track":"iSarcasmEval-C-Ar","label":"C · Arabic pairs","metric":"Pairwise accuracy","description":"Choose the sarcastic text from an Arabic sarcastic/literal pair. Random guessing scores 50%.","random":0.5,"headline":false,"in_index":true}],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.5327},"sources":[{"name":"iabufarha/iSarcasmEval","url":"https://github.com/iabufarha/iSarcasmEval.git","revision":"dfc708b53bde1bb571abfb5692f63231c2232195"}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"dfc708b53bde1bb571abfb5692f63231c2232195"},{"id":41,"dataset":"VAST","short":null,"kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":"language","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Says whether a comment is for, against, or neutral on a given topic.","comparability_note":null,"references":[{"kind":"dataset","url":"https://github.com/emilyallaway/zero-shot-stance"}],"cases":3006,"requests":3006,"fields":3006,"source_cases":3006,"unit":"case","scoring_metric":"stance macro-F1","subset_rule":"All 3,006 prepared cases retained.","source_filtering":null,"note":"Official VAST test split, all three labels, topic included. Release v1: All prepared cases retained.","suite_category":"Language understanding and classification","subcategory":"Topic-conditioned stance","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native stance classification with the source topic visible.","candidate_origin":"Original3 stance labels"},"random_baseline":{"value":0.3333,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.000273,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7365},"sources":[{"name":"emilyallaway/zero-shot-stance","url":"https://github.com/emilyallaway/zero-shot-stance.git","revision":"e7c4775182b184730f350995f8260579c9e066fe"}],"licences":[],"source_revision":"e7c4775182b184730f350995f8260579c9e066fe"},{"id":42,"dataset":"NLI4CT","short":"NLI4CT","kind":"static","area":"language","area_label":"Language Understanding","area_color":"#407e9e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"macro-F1","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Says whether clinical-trial report sections support or contradict a statement, some deliberately altered.","comparability_note":null,"references":[{"kind":"dataset","url":"https://github.com/ai-systems/Task-2-SemEval-2024"}],"cases":5500,"requests":5500,"fields":5500,"source_cases":5500,"unit":"case","scoring_metric":"NLI macro-F1; accuracy","subset_rule":"All 5,500 prepared cases retained.","source_filtering":"5,500 source cases, 5,500 decisions.","note":"All5500 official2024 test statements joined by ID to gold and complete specified trial sections; no gold evidence-span selection. Release v1: All prepared cases retained.","suite_category":"Document reasoning and retrieval","subcategory":"Clinical-trial entailment","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Source-designated trial sections and native binary judgment; gold evidence spans hidden.","candidate_origin":"Original entailment labels"},"random_baseline":{"value":0.486,"label":"Random choices · F1","kind":"random_f1","method":"Monte Carlo","explanation":"Uniform independent choice among each field's actual options, scored with the same F1 definition and observed/predicted label union as the model. F1 is nonlinear and class-frequency dependent; 1/K is not its expected value. This simulation error is not uncertainty in model quality.","monte_carlo_standard_error":0.000209,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.8512},"sources":[{"name":"ai-systems/Task-2-SemEval-2024","url":"https://github.com/ai-systems/Task-2-SemEval-2024.git","revision":"7f32fa6c7db43e577f22e9fc6c28eef1cc6d223e"}],"licences":[],"source_revision":"7f32fa6c7db43e577f22e9fc6c28eef1cc6d223e"},{"id":43,"dataset":"CRUXEval","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":"knowledge","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Reads a short Python function and an input and picks its output from 2 to 10 candidates, without running it.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2401.03065"},{"kind":"dataset","url":"https://github.com/facebookresearch/cruxeval"}],"cases":570,"requests":570,"fields":570,"source_cases":800,"unit":"code execution case","scoring_metric":"frozen output-choice accuracy","subset_rule":"All 570 prepared cases retained.","source_filtering":"800 source cases → 570 output-choice cases; 230 excluded for too few distinct archived candidates.","note":"Output-selection projection from all800 source cases, using archived incorrect model generations as distractors. Insufficient distinct candidates explicitly excluded. Parsed literal gold and Python-equivalence checks; not native pass@1. Release v1: All prepared cases retained.","suite_category":"Reasoning, mathematics and code","subcategory":"Code execution prediction","origin":"Jev-adapted benchmarks","adaptation":{"class":"C_semantic_options","class_description":"The adaptation relies on authored semantic option spaces/scenarios or model-generated candidate answers absent from the original native answer format.","reason":"Options rely on prior model generations of candidate answers, then mechanical equivalence checks. No new generation in this adaptation run.","candidate_origin":"Archived model-generated wrong outputs plus source gold"},"random_baseline":{"value":0.3697,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.7474},"sources":[{"name":"facebookresearch/cruxeval","url":"https://github.com/facebookresearch/cruxeval.git","revision":"190faf16d175b5847b0af05d937872b1fb395942"}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"190faf16d175b5847b0af05d937872b1fb395942"},{"id":44,"dataset":"CLadder","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":"knowledge","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Answers a yes/no question about cause and effect in a small made-up world with stated probabilities.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2312.04350"},{"kind":"dataset","url":"https://github.com/causalNLP/cladder"}],"cases":5000,"requests":5000,"fields":5000,"source_cases":10112,"unit":"case","scoring_metric":"causal yes/no accuracy","subset_rule":"5,000 of 10,112 prepared cases, deterministic hash sample without replacement (seed 20260919); no outcome-based selection.","source_filtering":null,"note":"Full10112-question balanced CLadder release; background, given information and question only. Not a newly held-out sample. Release v1: 5000 seeded cases.","suite_category":"Reasoning, mathematics and code","subcategory":"Causal inference","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Native causal binary judgment from supplied background and question.","candidate_origin":"Original yes/no outcomes"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.8858},"sources":[{"name":"causalNLP/cladder","url":"https://github.com/causalNLP/cladder.git","revision":"3d2d1169b4b939a09048a6a75956c8972a93cc38"}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":"3d2d1169b4b939a09048a6a75956c8972a93cc38"},{"id":45,"dataset":"HLE","short":null,"kind":"static","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"gold":1.2,"explainer":"Expert-level multiple choice from Humanity's Last Exam, text-only questions, 5 to 26 options.","comparability_note":"Not used in the frontier comparison on Jev's page: published HLE numbers cover the full benchmark and are not comparable to this suite's text-only multiple-choice subset.","references":[{"kind":"dataset","url":"https://huggingface.co/datasets/cais/hle"}],"cases":501,"requests":513,"fields":513,"source_cases":513,"unit":"case","scoring_metric":"text-only original-MC accuracy","subset_rule":"All 513 prepared cases retained.","source_filtering":"2,500 HLE rows → 513 text-only original multiple-choice items; excluded 342 image-dependent and 1645 open-answer items (IDs recorded).","note":"Only original text-only MC items; image-dependent/open-answer items excluded with recorded IDs. Not full HLE accuracy. Release v1: All prepared cases retained.","suite_category":"Knowledge and science","subcategory":"Frontier expert knowledge","origin":"Original-task benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Filters to text-only original-MC HLE items; does not invent choices for free-response items.","candidate_origin":"Original multiple-choice options only"},"random_baseline":{"value":0.1641,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.1936},"sources":[{"name":"cais/hle","url":"https://huggingface.co/datasets/cais/hle","revision":null}],"licences":[{"licence":"MIT","source":"archived dataset card","file":"README.md"}],"source_revision":null},{"id":48,"dataset":"ForecastBench","short":null,"kind":"static","area":"arts","area_label":"Arts & Human Taste","area_color":"#b6698a","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"Brier (lower is better)","lower_is_better":false,"index_rule":"clip((0.25 - Brier) / (0.25 - 0)) x coverage","left_out_of_index":false,"gold":null,"explainer":"Gives the probability that an event resolving July–September 2026 happens, from earlier information only.","comparability_note":null,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2409.19839"},{"kind":"dataset","url":"https://github.com/forecastingresearch/forecastbench-datasets"}],"cases":10139,"requests":10139,"fields":10139,"source_cases":10139,"unit":"dated event forecast","scoring_metric":"Brier score; calibration; log loss","subset_rule":"All 10,139 prepared cases retained.","source_filtering":"123,475 resolution records → 10,139 eligible forecasts resolving 2026-07-01 to 2026-09-18; excluded 104,563 outside the date policy and 8,773 unresolved or non-binary.","note":"10139 resolved binary/joint-event forecasts, resolution dates July1–September18 2026. Frozen pre-forecast context; market prior probabilities omitted, historical series baselines retained. Brier/log loss; unpublished Jev cutoff means contamination not certified. Release v1: All prepared cases retained.","suite_category":"Forecasting and collective decisions","subcategory":"Resolved-event forecasting","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Date/resolution filtering, frozen evidence extraction and joint-event rendering; source probabilities define the task, not invented outcomes.","candidate_origin":"Binary events and source joint-event directions"},"random_baseline":{"value":0.25,"label":"Always predict p = 0.5","kind":"uninformative_forecast","method":"Exact","explanation":"Binary Brier loss: 0 is perfect and 1 is maximally wrong. Reporting probability0.5 for every event always scores0.25. This is an uninformative probability forecast, not a random hard yes/no guess (whose expected Brier loss is0.5).","monte_carlo_standard_error":null,"secondary_reference":{"label":"Always p=0.5 log loss","value":0.6931}},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"forecastingresearch/forecastbench-datasets","url":"https://github.com/forecastingresearch/forecastbench-datasets.git","revision":"da48cfb37db3e674fd57030266c92009873be4b6"}],"licences":[{"licence":"CC-BY-SA-4.0","source":"archived source file","file":"LICENSE"}],"source_revision":"da48cfb37db3e674fd57030266c92009873be4b6","retired":true},{"id":50,"dataset":"Habermas Machine","short":"Habermas","kind":"static","area":"arts","area_label":"Arts & Human Taste","area_color":"#b6698a","frozen_panel_area":"arts","in_panel":true,"in_frozen_panel":true,"dropped_interactive":false,"in_reproduction_suite":true,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Given a group's opinions on a policy question, predicts which candidate statement the group ranked highest.","comparability_note":null,"references":[{"kind":"dataset","url":"https://github.com/google-deepmind/habermas_machine"}],"cases":1676,"requests":1676,"fields":1676,"source_cases":2779,"unit":"initial discussion panel","scoring_metric":"human preference agreement; preference regret","subset_rule":"All 1,676 prepared cases retained.","source_filtering":null,"note":"Complete initial human panels only; official evaluation cohorts. All tied minimum-summed-rank candidates accepted by the custom scorer. No count cap. Release v1: All prepared cases retained.","suite_category":"Forecasting and collective decisions","subcategory":"Democratic deliberation","origin":"Jev-adapted benchmarks","adaptation":{"class":"B_mechanical","class_description":"Extra deterministic processing changes the evaluated view: filtering, extraction, search, enumeration, pairing, simulation or numeric encoding. No per-item semantic option authoring.","reason":"Complete-panel filtering and predeclared rank aggregation; candidate prose/human preferences are source-provided.","candidate_origin":"Released consensus statements"},"random_baseline":{"value":0.311,"label":"Uniform random choice","kind":"random_accuracy","method":"Exact expectation","explanation":"Average accepted-answer count / option count across the actual sampled fields. Includes all tied correct moves/statements where the scorer accepts ties; variable option counts are averaged case by case.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.4457},"sources":[{"name":"google-deepmind/habermas_machine","url":"https://github.com/google-deepmind/habermas_machine.git","revision":"7923b71966c14136077c78d7841bc9e1a182dfe0"}],"licences":[{"licence":"Apache-2.0","source":"archived source file","file":"LICENSE"}],"source_revision":"7923b71966c14136077c78d7841bc9e1a182dfe0"},{"id":51,"dataset":"OpenBookQA","short":null,"kind":"calibration extension","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Answer elementary-science questions from four original choices, without the supporting fact.","comparability_note":null,"references":[],"cases":500,"requests":500,"fields":500,"source_cases":500,"unit":"case","scoring_metric":"accuracy; Brier; NLL; ECE; per-type/tier diagnostics","subset_rule":"OpenBookQA validation split, all 500 questions; main split without auxiliary gold facts.","source_filtering":null,"note":"See calibration-extension-v1/README.md; source protocol preserved.","suite_category":"Knowledge & Reasoning","subcategory":"Calibration and decision accuracy","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Wire-format conversion; original task retained.","candidate_origin":"Published options and labels"},"random_baseline":{"value":0.25,"label":"Uniform random choice","kind":"random_accuracy","method":null,"explanation":"Uniform guesses among the supplied answer classes; this is not a majority-class baseline.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"allenai/openbookqa","url":"https://huggingface.co/datasets/allenai/openbookqa","revision":null},{"name":"scienthoon/jev-ood-calibration","url":"https://github.com/scienthoon/jev-ood-calibration","revision":null}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":null},{"id":52,"dataset":"CommonsenseQA","short":null,"kind":"calibration extension","area":"knowledge","area_label":"Knowledge & Reasoning","area_color":"#5976bd","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Answer everyday commonsense questions from five original choices.","comparability_note":null,"references":[],"cases":1221,"requests":1221,"fields":1221,"source_cases":1221,"unit":"case","scoring_metric":"accuracy; Brier; NLL; ECE; per-type/tier diagnostics","subset_rule":"CommonsenseQA validation split, all 1,221 questions.","source_filtering":null,"note":"See calibration-extension-v1/README.md; source protocol preserved.","suite_category":"Knowledge & Reasoning","subcategory":"Calibration and decision accuracy","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Wire-format conversion; original task retained.","candidate_origin":"Published options and labels"},"random_baseline":{"value":0.2,"label":"Uniform random choice","kind":"random_accuracy","method":null,"explanation":"Uniform guesses among the supplied answer classes; this is not a majority-class baseline.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"tau/commonsense_qa","url":"https://huggingface.co/datasets/tau/commonsense_qa","revision":null},{"name":"scienthoon/jev-ood-calibration","url":"https://github.com/scienthoon/jev-ood-calibration","revision":null}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":null},{"id":53,"dataset":"Support-ticket calibration","short":"Support tickets","kind":"calibration extension","area":"tools","area_label":"Tools & Automation","area_color":"#b77a38","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Judge a support ticket for queue, anger and priority, against deliberately noisy labels.","comparability_note":null,"references":[],"cases":300,"requests":900,"fields":900,"source_cases":300,"unit":"case","scoring_metric":"accuracy; Brier; NLL; ECE; per-type/tier diagnostics","subset_rule":"Exact published 900 records grouped as 300 generated tickets; three tracks (queue, anger, priority); noisy labels preserved.","source_filtering":null,"note":"See calibration-extension-v1/README.md; source protocol preserved.","suite_category":"Tools & Automation","subcategory":"Calibration and decision accuracy","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Wire-format conversion; original task retained.","candidate_origin":"Published options and labels"},"random_baseline":{"value":null,"label":null,"kind":null,"method":null,"explanation":null,"monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"scienthoon/jev-ood-calibration","url":"https://github.com/scienthoon/jev-ood-calibration","revision":null}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":null},{"id":54,"dataset":"Phishing difficulty gradient","short":"Phishing gradient","kind":"calibration extension","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Judge generated emails as phishing across four difficulty tiers, from obvious to misleading.","comparability_note":null,"references":[],"cases":800,"requests":20,"fields":800,"source_cases":800,"unit":"message","scoring_metric":"accuracy; Brier; NLL; ECE; per-type/tier diagnostics","subset_rule":"Original four tiers, 200 messages each, batched in 40s with the published prompt (20 requests, 800 decisions).","source_filtering":null,"note":"See calibration-extension-v1/README.md; source protocol preserved.","suite_category":"Language & Retrieval","subcategory":"Calibration and decision accuracy","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Wire-format conversion; original task retained.","candidate_origin":"Published options and labels"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":null,"explanation":"Uniform guesses among the supplied answer classes; this is not a majority-class baseline.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"SamuelSacco/jev-exploration","url":"https://github.com/SamuelSacco/jev-exploration","revision":null}],"licences":[],"source_revision":null},{"id":55,"dataset":"Email spam classification","short":"Email spam","kind":"calibration extension","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":null,"in_panel":false,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Detect spam in real emails, scored on the structured-focus question wording.","comparability_note":null,"references":[],"cases":3000,"requests":3000,"fields":12000,"source_cases":19528,"unit":"case","scoring_metric":"accuracy; Brier; NLL; ECE; per-type/tier diagnostics","subset_rule":"3,000 uniformly sampled emails without replacement from 19,528; Python Random seed 20260920. See calibration-extension-v1/README.md; source protocol preserved.","source_filtering":"3,000 emails sampled uniformly without replacement from 19,528 (Python random.Random(20260920)); selected IDs pinned in the extension manifest.","note":"3,000 uniformly sampled emails without replacement from 19,528; Python Random seed 20260920. See calibration-extension-v1/README.md; source protocol preserved.","suite_category":"Language & Retrieval","subcategory":"Calibration and decision accuracy","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Wire-format conversion; original task retained.","candidate_origin":"Published options and labels"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":null,"explanation":"Uniform guesses among the supplied answer classes; this is not a majority-class baseline.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":null},"sources":[{"name":"bitnovus/jev-spam-eval","url":"https://github.com/bitnovus/jev-spam-eval","revision":null}],"licences":[{"licence":"MIT","source":"archived source file","file":"LICENSE"}],"source_revision":null},{"id":56,"dataset":"PhishNChips phishing decisions","short":"PhishNChips","kind":"calibration extension","area":"retrieval","area_label":"Retrieval & Classification","area_color":"#3e8e5e","frozen_panel_area":null,"in_panel":true,"in_frozen_panel":false,"dropped_interactive":false,"in_reproduction_suite":false,"metric":"accuracy","lower_is_better":false,"index_rule":null,"left_out_of_index":false,"explainer":"Says whether an email and its link are phishing or legitimate.","comparability_note":null,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/AreLit/PhishNChips"},{"kind":"code","url":"https://github.com/anisselbd/jev-phishing-bench"}],"cases":2000,"requests":2000,"fields":18000,"source_cases":2000,"unit":"case","scoring_metric":"accuracy; Brier; NLL; ECE; per-type/tier diagnostics","subset_rule":"Exact PhishNChips v5.2 source checksum, all 2,000 emails with the original nine-question payload.","source_filtering":null,"note":"See calibration-extension-v1/README.md; source protocol preserved.","suite_category":"Language & Retrieval","subcategory":"Calibration and decision accuracy","origin":"Original-task benchmarks","adaptation":{"class":"A_direct","class_description":"Original finite choices, class labels, or probabilities map directly; only parsing, serialization and label mapping are needed.","reason":"Wire-format conversion; original task retained.","candidate_origin":"Published options and labels"},"random_baseline":{"value":0.5,"label":"Uniform random choice","kind":"random_accuracy","method":null,"explanation":"Uniform guesses among the supplied answer classes; this is not a majority-class baseline.","monte_carlo_standard_error":null,"secondary_reference":null},"tracks":[],"jev":{"http_median_ms":null,"http_p95_ms":null,"score":0.8375},"sources":[{"name":"anisselbd/jev-phishing-bench","url":"https://github.com/anisselbd/jev-phishing-bench","revision":null},{"name":"AreLit/PhishNChips","url":"https://github.com/AreLit/PhishNChips","revision":null}],"licences":[{"licence":"per-source (MIT project code; CC-BY-4.0 Nazario corpus; OpenPhish/PhishTank/Tranco terms)","source":"archived SOURCE_LICENSES.md","file":"SOURCE_LICENSES.md"}],"source_revision":null}],"index":{"panel_id":"decision-index-0.3","proposal_panel_id":"proposed-core25-v1","proposal_status":"proposal_not_implemented","proposal_date":"2026-09-19","panel_full":37,"panel":37,"areas_count":5,"dropped":[{"id":7,"name":"MiniWoB++","frozen_area":"tools"},{"id":13,"name":"Boxoban","frozen_area":"games"},{"id":15,"name":"RTFM","frozen_area":"games"},{"id":16,"name":"ScienceWorld","frozen_area":"tools"},{"id":17,"name":"Hanabi","frozen_area":"games"},{"id":19,"name":"Codenames","frozen_area":"games"}],"folded":[{"from":"games","into":"knowledge","benchmarks":[{"id":31,"name":"ChessBench"}]}],"split":[{"from":"language","into":[{"id":"language","label":"Language Understanding","panel":[11,40,41]},{"id":"retrieval","label":"Retrieval & Classification","panel":[36,37]}]}],"headline_tracks":[{"id":40,"track":"iSarcasmEval-A-En","label":"A · English"}],"areas":[{"id":"knowledge","label":"Knowledge & Reasoning","color":"#5976bd","description":"Academic knowledge, commonsense, mathematics, and code.","jev_only":false,"folded_into":null,"weight":0.2585,"panel":[{"id":25,"name":"GPQA Diamond","short":"GPQA Diamond","gold":1.2},{"id":30,"name":"GSM8K","short":"GSM8K"},{"id":31,"name":"ChessBench","short":"ChessBench"},{"id":32,"name":"MuSR","short":"MuSR"},{"id":33,"name":"SATA-Bench","short":"SATA-Bench"},{"id":43,"name":"CRUXEval","short":"CRUXEval"},{"id":44,"name":"CLadder","short":"CLadder"},{"id":45,"name":"HLE","short":"HLE","gold":1.2},{"id":57,"name":"MMLU-Pro","short":"MMLU-Pro","gold":1.2},{"id":58,"name":"BBH","short":"BBH","gold":1.2},{"id":28,"name":"WinoGrande","short":"WinoGrande","gold":1.2}],"frozen_panel":[{"id":25,"name":"GPQA Diamond","dropped":false},{"id":30,"name":"GSM8K","dropped":false},{"id":31,"name":"ChessBench","dropped":false},{"id":32,"name":"MuSR","dropped":false},{"id":33,"name":"SATA-Bench","dropped":false},{"id":43,"name":"CRUXEval","dropped":false},{"id":44,"name":"CLadder","dropped":false},{"id":45,"name":"HLE","dropped":false},{"id":57,"name":"57","dropped":false},{"id":58,"name":"58","dropped":false}],"shown_only":[{"id":26,"name":"ARC-Easy"},{"id":27,"name":"ARC-Challenge"}]},{"id":"language","label":"Language Understanding","color":"#407e9e","description":"Read a text and judge what it means: entailment, sentiment, sarcasm and stance.","jev_only":false,"folded_into":null,"weight":0.2585,"panel":[{"id":11,"name":"ContractNLI","short":"ContractNLI"},{"id":12,"name":"ANLI","short":"ANLI","gold":1.2},{"id":29,"name":"HellaSwag","short":"HellaSwag","gold":1.2},{"id":38,"name":"ACOS","short":"ACOS"},{"id":39,"name":"FinEntity","short":"FinEntity"},{"id":40,"name":"iSarcasmEval","short":"iSarcasmEval"},{"id":41,"name":"VAST","short":"VAST"},{"id":42,"name":"NLI4CT","short":"NLI4CT"},{"id":59,"name":"RAGTruth","short":"RAGTruth"}],"frozen_panel":[{"id":11,"name":"ContractNLI","dropped":false},{"id":12,"name":"ANLI","dropped":false},{"id":28,"name":"WinoGrande","dropped":false},{"id":29,"name":"HellaSwag","dropped":false},{"id":38,"name":"ACOS","dropped":false},{"id":39,"name":"FinEntity","dropped":false},{"id":40,"name":"iSarcasmEval","dropped":false},{"id":41,"name":"VAST","dropped":false},{"id":42,"name":"NLI4CT","dropped":false},{"id":59,"name":"59","dropped":false}],"shown_only":[]},{"id":"retrieval","label":"Retrieval & Classification","color":"#3e8e5e","description":"Pick the right item from a large set: rank documents and products by relevance, route a message to one of many intents, sort out spam and phishing.","jev_only":false,"folded_into":null,"weight":0.2002,"panel":[{"id":4,"name":"BANKING77","short":"BANKING77","gold":1.2},{"id":5,"name":"CLINC150+OOS","short":"CLINC150","gold":1.2},{"id":36,"name":"BRIGHT","short":"BRIGHT","gold":1.2},{"id":37,"name":"Amazon ESCI","short":"Amazon ESCI"},{"id":56,"name":"PhishNChips phishing decisions","short":"PhishNChips"},{"id":61,"name":"HoVer","short":"HoVer"}],"frozen_panel":[{"id":4,"name":"BANKING77","dropped":false},{"id":5,"name":"CLINC150+OOS","dropped":false},{"id":36,"name":"BRIGHT","dropped":false},{"id":37,"name":"Amazon ESCI","dropped":false},{"id":56,"name":"PhishNChips phishing decisions","dropped":false},{"id":61,"name":"61","dropped":false}],"shown_only":[{"id":10,"name":"SGD/SGD-X"}]},{"id":"tools","label":"Tools & Automation","color":"#b77a38","description":"Tool choice, model routing, household control, and browser or simulator actions.","jev_only":false,"folded_into":null,"weight":0.1828,"panel":[{"id":1,"name":"BFCL","short":"BFCL","gold":1.2},{"id":2,"name":"ToolRet","short":"ToolRet"},{"id":3,"name":"API-Bank","short":"API-Bank","gold":1.2},{"id":9,"name":"Home appliance simulator","short":"Home appliances"},{"id":62,"name":"When2Call","short":"When2Call"}],"frozen_panel":[{"id":1,"name":"BFCL","dropped":false},{"id":2,"name":"ToolRet","dropped":false},{"id":3,"name":"API-Bank","dropped":false},{"id":9,"name":"Home appliance simulator","dropped":false},{"id":62,"name":"62","dropped":false}],"shown_only":[{"id":6,"name":"RouterBench"}]},{"id":"arts","label":"Arts & Human Taste","color":"#b6698a","description":"Poetry, music, humor, human preferences, and forecasting.","jev_only":false,"folded_into":null,"weight":0.1,"panel":[{"id":20,"name":"BPoMP","short":"BPoMP"},{"id":21,"name":"Humicroedit","short":"Humicroedit"},{"id":22,"name":"POP909-CL","short":"POP909"},{"id":23,"name":"cfcolor","short":"cfcolor"},{"id":50,"name":"Habermas Machine","short":"Habermas"},{"id":64,"name":"New Yorker","short":"New Yorker"}],"frozen_panel":[{"id":20,"name":"BPoMP","dropped":false},{"id":21,"name":"Humicroedit","dropped":false},{"id":22,"name":"POP909-CL","dropped":false},{"id":23,"name":"cfcolor","dropped":false},{"id":48,"name":"ForecastBench","dropped":false},{"id":50,"name":"Habermas Machine","dropped":false},{"id":64,"name":"64","dropped":false}],"shown_only":[]},{"id":"games","label":"Games & Strategy","color":"#ad6f48","description":"Move selection, spatial planning, rules, and decisions against other players.","jev_only":true,"folded_into":"knowledge","weight":0,"panel":[],"frozen_panel":[{"id":31,"name":"ChessBench","dropped":false}],"shown_only":[]}],"frozen_panel":[{"id":"knowledge","label":"Knowledge & Reasoning","benchmarks":[{"id":24,"name":null,"dropped":false},{"id":25,"name":null,"dropped":false},{"id":30,"name":null,"dropped":false},{"id":43,"name":null,"dropped":false},{"id":44,"name":null,"dropped":false}]},{"id":"language","label":"Language & Retrieval","benchmarks":[{"id":11,"name":null,"dropped":false},{"id":36,"name":null,"dropped":false},{"id":37,"name":null,"dropped":false},{"id":40,"name":null,"dropped":false},{"id":41,"name":null,"dropped":false}]},{"id":"tools","label":"Tools & Automation","benchmarks":[{"id":1,"name":null,"dropped":false},{"id":2,"name":null,"dropped":false},{"id":6,"name":null,"dropped":false},{"id":7,"name":null,"dropped":true},{"id":16,"name":null,"dropped":true}]},{"id":"games","label":"Games & strategy","benchmarks":[{"id":31,"name":null,"dropped":false},{"id":13,"name":null,"dropped":true},{"id":15,"name":null,"dropped":true},{"id":17,"name":null,"dropped":true},{"id":19,"name":null,"dropped":true}]},{"id":"arts","label":"Arts & Human Taste","benchmarks":[{"id":20,"name":null,"dropped":false},{"id":21,"name":null,"dropped":false},{"id":22,"name":null,"dropped":false},{"id":23,"name":null,"dropped":false},{"id":50,"name":null,"dropped":false}]}],"formulas":[{"id":"balanced_raw","normalization":"raw","formula":"100*(0.2585*category[knowledge] + 0.2585*category[language] + 0.2002*category[retrieval] + 0.1828*category[tools] + 0.1000*category[arts])"},{"id":"balanced_skill","normalization":"skill","formula":"Public index: 100*(0.2585*category[knowledge] + 0.2585*category[language] + 0.2002*category[retrieval] + 0.1828*category[tools] + 0.1000*category[arts]). Full score: 0.60*private_same_skills + 0.40*private_new_domains, both mapped onto the public scale with untuned stock models; the public index is shown but not counted"},{"id":"breadth_skill","normalization":"skill","shift":0.1,"formula":"100*(product_c((0.1+0.9*category[c])**w[c])-0.1)/0.9"}],"shown":"balanced_skill","hidden":["breadth_skill"],"functions":{"clip":"min(1,max(0,z))","raw_fraction":"clip(x)","raw_brier":"clip(1-x)","raw_bounded_score":"clip((x-L)/(U-L))","skill":"clip((raw-reference)/(1-reference))","benchmark":"sum_t(track_weight[t]*normalized_track[t])","category":"weighted mean of the area's benchmark scores (gold ★ benchmarks weigh 1.2, the others 1.0)"},"baseline_policy":{"versioned":true,"empirical_baselines_status":"must_freeze_before_ranking","uniform_f1_monte_carlo_se_max":0.001,"uniform_f1_fixed_label_universe":true,"raw_maximum":1,"selected_interactive_reference":0,"selected_interactive_reference_label":"task floor, not measured random policy"},"coverage":{"denominator":"all frozen source cases or episodes, preserving fixed subtrack and category weights","formula":"sum_c(0.2*mean_b(sum_t(w_bt*valid_bt/planned_bt)))","multiply_score_by_coverage":false,"official_requires":"all scheduled cases terminal valid or certified model failure; no pending evaluator defects or unrun results"},"failures":{"accuracy":0,"ndcg_incomplete_query":0,"quality":0,"game":"minimum terminal utility; zero for selected games","brier":1,"f1_per_class":"2*TP/(2*TP+FP+FN+M); denominator zero maps to zero","transient_retry_count":2,"valid_wrong_answer_retries":0,"unsupported":"full-denominator failure; never omit","infrastructure_bug":"repair; pending until fixed","unrun":"zero only in labeled provisional lower bound; final rank withheld"},"uncertainty":{"method":"paired hierarchical cluster bootstrap through complete index formula","replicates":2000,"confidence_level":0.95,"weights":"fixed","difference_intervals":true,"boundary_binary_diagnostics":"exact binomial or Wilson interval"},"supplementary":{"forecastbench":{"catalog_id":48,"reason":"retrospective resolved events and uncertified model cutoffs","raw":"clip(1-Brier)","skill":"clip(1-Brier/0.25)","index_weight":0},"leduc":{"catalog_id":18,"reason":"freeze terminal utility anchors/opponent panel before inclusion","index_weight":0}},"efficiency_in_score":false,"clarifications":["CRUXEval uses the sole frozen output-choice track; the proposal incorrectly mentioned an input track that does not exist in this suite.","Interactive scores use existing frozen pilots and first complete benchmark runs, not the proposed expanded three-repeat protocol.","Reproductions have no interactive runs. Those six benchmarks receive zero only as labeled provisional lower bounds, never an official final ranking.","Unsupported source cases receive zero before chance normalization. F1 uses fixed label universes and conservative missing-label penalties.","All linked requests must succeed for source-case support; no successful-intersection score shortcut.","Point estimates only; paired cluster-bootstrap uncertainty is not yet calculated.","The index panel is the Decision Index 0.3 panel (decision-index-0.3), not the frozen core25 proposal; core25 scores are still computed and kept as frozen_scores.","Benchmarks outside the frozen core25 panel enter with their board score times the answered share (unanswered or unsupported counts as wrong); the 19 core25 static benchmarks keep their case-group coverage adjustment."],"steps":["Score every benchmark with its own primary metric and adjust it for coverage: an unanswered, refused, errored or unsupported request counts as wrong (for the 19 core25 static benchmarks a case whose linked requests did not all succeed scores zero; for the others the score on answered requests is multiplied by the answered share).","Benchmarks with several tracks (RouterBench 0-shot/5-shot, GSM8K 4-/10-choice) average their tracks equally; iSarcasmEval uses track A English as its headline and track B never enters the index.","Lower-is-better metrics are converted against their baseline: ForecastBench value = clip((0.25 - Brier) / (0.25 - 0)) x coverage, so always predict p = 0.5 (Brier 0.25) or worse scores 0 and a perfect forecast scores 1.","Each of the five areas is the gold-weighted mean of its benchmarks (★ = gold, weight 1.2): Knowledge & Reasoning (10: GPQA Diamond ★, GSM8K, ChessBench, MuSR, SATA-Bench, CRUXEval, CLadder, HLE ★, MMLU-Pro ★, BBH ★), Language Understanding (10: ContractNLI, ANLI ★, WinoGrande ★, HellaSwag ★, ACOS, FinEntity, iSarcasmEval, VAST, NLI4CT, RAGTruth), Retrieval & Classification (6: BANKING77 ★, CLINC150 ★, BRIGHT ★, Amazon ESCI, PhishNChips, HoVer), Tools & Automation (5: BFCL ★, ToolRet, API-Bank ★, Home appliances, When2Call), Arts & Human Taste (7: BPoMP, Humicroedit, POP909, cfcolor, ForecastBench ★, Habermas, New Yorker).","Chance-correct every benchmark: k = clip((s − chance) / (1 − chance)) with the chance level listed below, so 0 is random guessing and 1 is perfect. Multi-track and retrieval benchmarks correct each track or query against its own chance level; ForecastBench is already 0 at its baseline.","Decision Index = 100 × weighted mean of the five area scores, each the gold-weighted mean of its chance-corrected benchmarks. Arts & Human Taste is fixed at 10%; the other four areas keep the 0.2.1 weights (Knowledge & Reasoning 25.8%, Language Understanding 25.8%, Retrieval & Classification 20.0%, Tools & Automation 18.3%). Gold ★ benchmarks weigh 1.2 inside their area and every other benchmark 1.0, normalized within the area. GSM8K is left out of the gold tier because its distractor options leak the answer. 37 benchmarks in all.","Not in the index, still shown on the board: MMLU, ARC-Easy, ARC-Challenge, RouterBench, SGD. RouterBench and SGD left the index in Decision Index 0.2.1; the reasons are listed below.","Raw accuracy (not chance-corrected) repeats the same averaging on the uncorrected scores and is shown on model pages; the breadth variant takes a shifted geometric mean of the five chance-corrected area scores and is computed but hidden.","The six interactive environments of the frozen core25 panel stay out for every model, Jev included, because no reproduction has run them."],"worked_example":{"model":"Jev","areas":[{"id":"knowledge","raw":0.6391,"skill":0.5387,"n":11},{"id":"language","raw":0.7097,"skill":0.5923,"n":9},{"id":"retrieval","raw":0.6825,"skill":0.5542,"n":6},{"id":"tools","raw":0.7764,"skill":0.7509,"n":5},{"id":"arts","raw":0.583,"skill":0.3908,"n":6}],"scores":{"balanced_raw":68.55,"balanced_skill":57.96,"breadth_skill":57.27},"frozen_scores":{"balanced_raw":54.47,"balanced_skill":43.61,"breadth_skill":36.98}},"frontier_comparison":{"note":"Jev's own page compares its suite scores with published frontier numbers on a few benchmarks; a benchmark enters that comparison only when the published protocol matches the suite's subset.","excluded":[{"id":45,"name":"HLE","why":"Not used in the frontier comparison on Jev's page: published HLE numbers cover the full benchmark and are not comparable to this suite's text-only multiple-choice subset."}]},"hashes":{"panel_proposal_sha256":"a9823d5d8ce1dcf81da91f285d9de5b77b8bb1c9a5ad4fa2d51df6c3d0850a21","static_corpus_sha256":"b2b56d6fb636837ca469e689087bdbf373dda8de7638aa2da6793e6eda0792d5","random_baselines_sha256":"b68bb94e1ce0d2a40b4f69d0b7a9acf796c4bc64c973d5c291625aec34629bfa"},"frozen_panel_id":"core25-observed-protocol-v1","also_shown":[{"id":"balanced_raw","label":"raw accuracy (not chance-corrected)"}],"chance_levels":[{"id":25,"name":"GPQA Diamond","area":"knowledge","chance":0.25,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":30,"name":"GSM8K","area":"knowledge","chance":0.175,"label":"Uniform random choice","method":"per track: 4-choice and 10-choice versions corrected separately, then averaged"},{"id":31,"name":"ChessBench","area":"knowledge","chance":0.0819,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":32,"name":"MuSR","area":"knowledge","chance":0.371,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":33,"name":"SATA-Bench","area":"knowledge","chance":0.0131,"label":"Random whole-case guesses","method":"expected score of uniform random choice over each question's options"},{"id":43,"name":"CRUXEval","area":"knowledge","chance":0.3697,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":44,"name":"CLadder","area":"knowledge","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":45,"name":"HLE","area":"knowledge","chance":0.1641,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":57,"name":"MMLU-Pro","area":"knowledge","chance":0.1109,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":58,"name":"BBH","area":"knowledge","chance":0.3101,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":11,"name":"ContractNLI","area":"language","chance":0.3085,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":12,"name":"ANLI","area":"language","chance":0.3324,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":28,"name":"WinoGrande","area":"knowledge","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":29,"name":"HellaSwag","area":"language","chance":0.25,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":38,"name":"ACOS","area":"language","chance":0.031,"label":"Always yes · per-review F1","method":"per-review F1 of answering yes to every aspect-and-sentiment pair (a fair coin scores lower)"},{"id":39,"name":"FinEntity","area":"language","chance":0.3201,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":40,"name":"iSarcasmEval","area":"language","chance":0.2227,"label":"Random choices · F1","method":"track A English: F1 of random guessing on the sarcastic class"},{"id":41,"name":"VAST","area":"language","chance":0.3333,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":42,"name":"NLI4CT","area":"language","chance":0.486,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":59,"name":"RAGTruth","area":"language","chance":0.5177,"label":"Always \"hallucinated\" · F1","method":"F1 of always answering \"hallucinated\" (a fair coin scores lower)"},{"id":4,"name":"BANKING77","area":"retrieval","chance":0.0127,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":5,"name":"CLINC150","area":"retrieval","chance":0.006,"label":"Random choices · F1","method":"expected score of uniform random choice over each question's options"},{"id":36,"name":"BRIGHT","area":"retrieval","chance":0.116,"label":"Random ranking, answerable queries","method":"per query: expected nDCG@10 of a random ranking of that query's candidates, answerable queries only"},{"id":37,"name":"Amazon ESCI","area":"retrieval","chance":0.2027,"label":"Random choices · F1","method":"F1 of random guessing over the fixed label set"},{"id":56,"name":"PhishNChips","area":"retrieval","chance":0.5,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":61,"name":"HoVer","area":"retrieval","chance":0.5,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":1,"name":"BFCL","area":"tools","chance":0.2592,"label":"Random whole-case guesses","method":"all fields of a case right by chance"},{"id":2,"name":"ToolRet","area":"tools","chance":0.1341,"label":"Random ranking, answerable queries","method":"per query: expected nDCG@10 of a random ranking of that query's candidates, answerable queries only"},{"id":3,"name":"API-Bank","area":"tools","chance":0.0189,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":9,"name":"Home appliances","area":"tools","chance":0,"label":"Random whole-case guesses","method":"all fields of a case right by chance, on the kept rows"},{"id":62,"name":"When2Call","area":"tools","chance":0.25,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"},{"id":20,"name":"BPoMP","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":21,"name":"Humicroedit","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":22,"name":"POP909","area":"arts","chance":0.0078,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":23,"name":"cfcolor","area":"arts","chance":0.5,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":50,"name":"Habermas","area":"arts","chance":0.311,"label":"Uniform random choice","method":"expected score of uniform random choice over each question's options"},{"id":64,"name":"New Yorker","area":"arts","chance":0.2,"label":"uniform random choice, mean of 1/options","method":"expected score of uniform random choice over each question's options"}],"chance_paragraph":"The Decision Index is chance-corrected. Every benchmark score is first mapped to (score − chance) / (1 − chance), clipped to 0–1, where chance is what random guessing scores on that benchmark: 0.5 on a yes/no question, 0.25 with four options, the expected nDCG of a random ranking for retrieval, the F1 of random labels for F1-scored tasks. 0 means no better than random guessing and 100 means perfect, so a benchmark made of yes/no questions no longer hands out 50 free points and a benchmark with hundreds of options no longer looks harder than it is. Area scores and the index average these corrected values exactly as before. The raw, uncorrected accuracy is still computed and shown on each model page as raw accuracy (not chance-corrected).","lower_rules":[],"not_in_index":[{"id":6,"name":"RouterBench","note":"Not in the index: the prompt reveals that GPT-4 is the best route, so the top models all pick GPT-4 and score within a fraction of a point of each other. Scores stay visible."},{"id":10,"name":"SGD","note":"Not in the index: build bug, pending re-run. The builder gave every SGD row the whole dialogue, including the turns after the one being classified. Scores stay visible; SGD returns to the index once re-run with the fixed builder."},{"id":24,"name":"MMLU","note":null},{"id":26,"name":"ARC-Easy","note":null},{"id":27,"name":"ARC-Challenge","note":null}],"panel_change":"Decision Index 0.3 keeps the 0.2.1 public panel and rules with these changes: GSM8K is rebuilt, ForecastBench is retired and WinoGrande counts under Knowledge & Reasoning, so the public index averages 37 benchmarks. The headline is the Full score: 60% private tests of the same skills and 40% private tasks from new domains, both mapped onto the public scale with untuned stock models. Since Decision Index 0.4 the public index is shown alongside but does not count toward the Full score. ACOS, ToolRet, BRIGHT, RAGTruth and Home appliances stay rescored as described under Rescored benchmarks.","weights":{"knowledge":0.2585,"language":0.2585,"retrieval":0.2002,"tools":0.1828,"arts":0.1},"weights_pct":{"knowledge":"25.8%","language":"25.8%","retrieval":"20.0%","tools":"18.3%","arts":"10.0%"},"sizing":{"rule":"sqrt","fixed":{"arts":0.1},"text":"Arts & Human Taste is fixed at 10%; the other four areas keep the 0.2.1 weights, set in proportion to √n of their benchmark count at the time (Tools & Automation 18.3%, Retrieval & Classification 20.0%, Knowledge & Reasoning 25.8%, Language Understanding 25.8%). In 0.3 Knowledge & Reasoning holds 11 benchmarks and Language Understanding 9."},"gold":{"weight":1.2,"text":"Gold ★ benchmarks weigh 1.2 inside their area and every other benchmark 1.0, normalized within the area. GSM8K is left out of the gold tier because its distractor options leak the answer.","benchmarks":[{"id":57,"name":"MMLU-Pro","area":"knowledge"},{"id":58,"name":"BBH","area":"knowledge"},{"id":25,"name":"GPQA Diamond","area":"knowledge"},{"id":45,"name":"HLE","area":"knowledge"},{"id":12,"name":"ANLI","area":"language"},{"id":28,"name":"WinoGrande","area":"knowledge"},{"id":29,"name":"HellaSwag","area":"language"},{"id":4,"name":"BANKING77","area":"retrieval"},{"id":5,"name":"CLINC150","area":"retrieval"},{"id":36,"name":"BRIGHT","area":"retrieval"},{"id":1,"name":"BFCL","area":"tools"},{"id":3,"name":"API-Bank","area":"tools"}]},"rescored":[{"id":38,"name":"ACOS","what":"metric","line":"Scored with F1 per review, averaged over reviews: the aspect-and-sentiment pairs a model says yes to are compared with the gold pairs of that review. Whole-review exact match left almost everyone near zero (the best entrant got 9.2% of reviews fully right). Chance is answering yes to every pair (0.031), which beats a fair coin."},{"id":2,"name":"ToolRet","what":"subset","line":"Only queries with at least one relevant tool among their 32 candidates are scored (685 of 1,000): on the others every ranking scores 0. Chance is recomputed per query on the same subset (0.1341); 0.2 displayed 0.0937 while the index used the per-query value."},{"id":36,"name":"BRIGHT","what":"subset","line":"Same rule as ToolRet: 220 of 550 queries have a relevant document among their 32 candidates. Chance 0.1160 on those queries; 0.2 displayed 0.0448."},{"id":59,"name":"RAGTruth","what":"chance","line":"Chance is the F1 of always answering \"hallucinated\" (0.518), a stronger trivial baseline than a fair coin (0.411)."},{"id":9,"name":"Home appliances","what":"subset","line":"24 duplicate test rows (one copy of each pair kept) and 48 test rows identical to rows of the published dev split are dropped; 88 of 160 rows are scored."}]},"rules":{"evaluation_rules":["Native inference helpers, prompts and readout rules are preserved. Mechanical schema translations and any probability conversion are recorded with the adapter.","No prompt tuning, truncation, option pruning, retraining, or correctness-based retry.","Two requests per benchmark/subtrack form the 86-request compatibility pass. It is not a representative accuracy estimate. A model advances only after that pass completes without runtime errors; supported capacity exclusions stay visible.","Entire linked source groups must be complete and supported before they enter the preliminary successful-case scorer. Missing coverage is reported separately. These successful-case metrics must not be directly averaged into an overall index.","Full records retain original state/questions, native outputs, mapped answers, probabilities, errors and CUDA-synchronized request time. Model loading and toy warmup are separate. Compute timing excludes HTTP/network and is not directly equivalent to Jev HTTPS latency.","Decider's eager and compilation-heavy runs are separate. The initial compiled smoke measured compilation at new shapes, not warmed serving latency.","Native abstentions are retained as terminal abstentions; the wrapper never forces them into a listed answer.","Games require fresh episodes using each model's own decisions under the same seeds, observations, options, partners and budgets. **Interactive reproduction runs have not started; do not reuse Jev trajectories as their scores.**"],"hard_rules":[{"rule":"No truncation","detail":"Every request is sent whole. A model whose context window cannot hold a prompt records that request as unsupported; it is never shortened to fit."},{"rule":"No label filtering","detail":"The full option set of every question is offered. No option pruning, no candidate reduction, no label-set narrowing per model."},{"rule":"No prompt tuning","detail":"Each entrant runs with its own native inference helpers, prompts and readout rules as published. Mechanical schema translations and any probability conversion are recorded with the adapter."},{"rule":"No labels in the adapter","detail":"Gold answers never enter model state; requests.jsonl carries the exact payloads with labels omitted."},{"rule":"No correctness-based retry","detail":"Transient transport errors retry up to twice; a valid wrong answer is never retried."},{"rule":"Native abstentions stand","detail":"A model that abstains keeps its abstention as a terminal answer; the wrapper never forces it into a listed option."},{"rule":"Unanswered = wrong","detail":"Unsupported, refused and errored requests score zero against the full denominator. Nobody climbs by skipping the hard cases."}],"complete_run_gate":{"total_requests":120340,"excluded_questions":442,"scoreable_rows":119898,"rule":"A run enters the index only when its status event is \"complete\" and it has processed at least 119,898 rows (the 120,340-request corpus minus the 442 questions excluded for every engine)."},"superseded_policy":"A model appears once, at the configuration that stands. Lifting a cap the author set outside the model, or replacing a checkpoint, retires the earlier run from the ranking; the results file stays on disk, it just no longer competes.","superseded":[{"retired":"nanojev","replaced_by":"nanojev-8k","why":"moved to the 8k-context checkpoint"},{"retired":"jeff","replaced_by":"jeff-uncapped","why":"author-set output cap lifted"},{"retired":"jevfire","replaced_by":"jevfire-uncapped","why":"author-set output cap lifted"},{"retired":"kev-4b","replaced_by":"kev-4b-raised","why":"label window raised"},{"retired":"kev-9b","replaced_by":"kev-9b-raised","why":"label window raised"},{"retired":"kev-0.8b","replaced_by":"kev-0.8b-raised","why":"label window raised"},{"retired":"pngwn-space-uncapped","replaced_by":"pngwn-space-uncapped-full","why":"full run replaced the earlier partial run"},{"retired":"joshua-diffusion","replaced_by":"joshua-diffusion-full","why":"full run replaced the earlier partial run"},{"retired":"decision-nox","replaced_by":"decision-nox-full","why":"full run replaced the earlier partial run"},{"retired":"solomon-v11-bf16","replaced_by":"solomon-v11-bf16-encoding","why":"encoding path replaced the earlier serving path"},{"retired":"razorback","replaced_by":"razorback-one-read","why":"same build serving the same weights as the one-read path; one row per project"},{"retired":"laptop-pcd","replaced_by":"harsha","why":"same decoding harness over the same stock Qwen2.5-1.5B-Instruct as Qwen-2.5-1B-RLCD, identical scores; one row"},{"retired":"decision-kai","replaced_by":"decision-kai-7185f514","why":"newer Decision 1.0 Kai revision 7185f514"},{"retired":"decision-sol","replaced_by":"decision-sol-0665a411","why":"newer Decision 1.0 Sol revision 0665a411"},{"retired":"decision-nox-full","replaced_by":"decision-nox-0bb83350","why":"newer Decision 1.0 Nox revision 0bb83350"},{"retired":"this-that-1.0","replaced_by":"this-that-1.2","why":"newer this-that release"},{"retired":"nimble","replaced_by":"nimble-v2","why":"newer Nimble release (v2)"},{"retired":"razorback-one-read","replaced_by":"razorback-one-read-3e296f08","why":"newer openjev build 0.4.0 @3e296f08"},{"retired":"laya","replaced_by":"laya-1e28ac20","why":"newer laya code revision 1e28ac20 (same weights)"},{"retired":"verdict","replaced_by":"verdict-8af2496e","why":"newer Verdict weights 8af2496e and code 30f15564"},{"retired":"rune-26b-a4b","replaced_by":"rune-26b-a4b-v3","why":"author-submitted v3 checkpoint (kit PR #9), adopted after a 3,000-row reproduction matched it bit for bit; v1 stays on the 0.2 board"},{"retired":"reflex-27b","replaced_by":"reflex-27b-v2","why":"author-submitted full-coverage rerun with wide-choice one-token labels (kit PR #13), adopted after our latency-v1 run agreed with it within reflex's own run-to-run noise; v1 stays on the 0.2 board"},{"retired":"hopper","replaced_by":"hopper-g-1.2","why":"Hopper (G) 1.2, the author-submitted run of the next Hopper adapter (same Qwen3.5-4B base, continued from Hopper 1.0's adapter, same 1.1.1 serving code and map; kit PR #17), adopted after our latency-v1 run agreed with it within the A10G-vs-RTX noise; 1.1.1 stays on the 0.2 board"}],"withdrawn":[{"engine":"kev-0.5b-raised","why":"Withdrawn at the author's request: an older Kev generation (Qwen2.5-0.5B base); the released family is Qwen3.5 0.8B / 4B / 9B.","source":"https://huggingface.co/spaces/multimodalart/jev-decision-index/discussions/10"},{"engine":"kev-0.6b-raised","why":"Withdrawn at the author's request: an older Kev generation (Qwen3-0.6B-Base); the released family is Qwen3.5 0.8B / 4B / 9B.","source":"https://huggingface.co/spaces/multimodalart/jev-decision-index/discussions/10"},{"engine":"kev-8b-raised","why":"Withdrawn at the author's request: an older Kev generation (Qwen3-8B-Base); the released family is Qwen3.5 0.8B / 4B / 9B.","source":"https://huggingface.co/spaces/multimodalart/jev-decision-index/discussions/10"}],"jev_reference_notes":["Metrics exclude failed/incomplete cases; coverage is explicit. No silent replacement of failures.","Different primary metrics are not averaged across benchmarks.","Latency includes HTTPS and concurrent service scheduling; not isolated model inference.","Full raw HTTP responses/payloads and all retry timings are in http-attempts.jsonl.","BPoMP/RouterBench/GSM8K and chunked tasks retain all linked requests for sampled cases."],"interactive":"Games require fresh episodes using each model's own decisions under the same seeds, observations, options, partners and budgets. Interactive reproduction runs have not started; Jev trajectories are never reused as their scores.","removed":[{"engine":"nanojev-8k","why":"NanoJev retired; removed from the board."},{"engine":"reflex-4b","why":"Removed: the author's recommended configuration is the frozen Qwen3.5-4B base without the published LoRA adapter, and that adapter was trained on 800 MMLU-Pro test items. reflex 27B stays."},{"engine":"jqv-targeted-v2","why":"Parked; not run for 0.2."},{"engine":"bev","why":"Cancelled; not run."},{"engine":"jqv","why":"Left out of 0.2: the new index benchmarks were not run for it; its results stay on disk."}]},"entrants":{"principle":"Nothing is inferred from a name. The kind comes from what the run recorded it loaded, the base comes from the Hub card's own base_model field or from the run's recorded runtime model, and anything that resolves to neither is left null rather than guessed.","kinds":[{"kind":"inference technique","rule":"The entrant published no weights of its own: it serves a stock checkpoint from a base-model vendor namespace (Qwen, Google, LiquidAI, Meta, Mistral, Microsoft, NVIDIA, DeepSeek, AI2, HuggingFaceTB, Answer.AI, OpenAI, IBM Granite, TII) through its own decoding or scoring code."},{"kind":"full fine-tune","rule":"The entrant's own repo ships whole model weights (.safetensors, .bin, .pt, .pth, .gguf or .npz) with no adapter files."},{"kind":"LoRA","rule":"The entrant's files include adapter_config.json or adapter_model.* and no separate head, scorer or pointer weights."},{"kind":"LoRA + head","rule":"Adapter files plus a head, scorer or pointer module."},{"kind":"head / adapter","rule":"A head, scorer or pointer module on a frozen backbone the entrant did not train; classed as full fine-tune when no base model is recorded."},{"kind":"fine-tune","rule":"Own repo with weights the file list could not classify further."}],"weights_in_tree":"A GitHub entrant that carries trained weights inside its own checkout (for example a scorer adapter) counts as trained, not as a technique over the stock base.","techniques":[{"technique":"autoregressive","rule":"The executing model class ends in ForCausalLM or ForConditionalGeneration, read from the config of the entrant's own checkpoint or, failing that, of the backbone it rides on."},{"technique":"diffusion","rule":"The model_type or architecture class contains \"diffusion\", however the model is served."},{"technique":"encoder","rule":"A masked-LM or BERT-family class."},{"technique":"GLi family","rule":"GLiClass, GLiNER2 (BoundaryExtractor / extractor) and GLiFormer classes, bucketed together for the badge; the exact published class is kept alongside."}],"parameter_count_precedence":["Backbone parameter count from the Hub safetensors metadata of the base model (a LoRA or a head reports the backbone it rides on, which is what it costs to run).","Otherwise the entrant's own repo's safetensors metadata total.","Otherwise an exact count from each safetensors header (shapes read over a range request, no tensor download).","Otherwise, for .bin-only repos, weight bytes divided by the declared torch_dtype width, flagged params_estimated.","Otherwise the size stated in the model name (9B, 350M, 35B-A3B gives the total), flagged params_from_name.","trained_bytes is the entrant's own published weight bytes, which is what separates a 9B full fine-tune from a 9B backbone carrying a few megabytes of adapter."],"base_model_rule":"base_model is followed from the Hub card until the owner changes (a card often points at a sibling: an NVFP4 quantization declares the unquantized copy; Decision Lex declares Kai), so the row says what the weights descend from. A base under a vendor namespace that is itself a quantization is resolved two more hops to the original model. The chain is kept for the record.","naming":"Each row uses what the project calls itself in its own README or model card, with the author in parentheses where the name alone is ambiguous (five separate entrants are some spelling of \"openjev\"). variant names a serving choice made here, never part of the published name.","counts":{"kind":{"LoRA + head":8,"full fine-tune":24,"inference technique":15,"head / adapter":9,"LoRA":14},"technique":{"autoregressive":55,"encoder":5,"diffusion":4,"GLi family":6},"entrants":70},"entrants":[{"engine":"akash-gemma","name":"system-one-gemma","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":268098176,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"google/gemma-3-270m","base_chain":null,"weights_repo":null,"served_checkpoint":null,"model_url":null,"code_url":"https://github.com/akash-kamat/system-one-gemma","unanswered_share":0},{"engine":"autojev-27b","name":"pplx-decider-v1-27b","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":26085330160,"trained_bytes":52173413928,"base_model":"Qwen/Qwen3.8-27B","base_chain":["Qwen/Qwen3.8-27B"],"weights_repo":"perplexity-ai/pplx-decider-v1-27b","served_checkpoint":null,"model_url":"https://huggingface.co/perplexity-ai/pplx-decider-v1-27b","code_url":"https://huggingface.co/perplexity-ai/pplx-decider-v1-27b/tree/main/source","unanswered_share":0},{"engine":"bosun-v3.1-0.6b","name":"Bosun v3.1 0.6B","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":596049920,"params_from_name":false,"params_estimated":false,"trained_params":10616832,"trained_bytes":41470928,"base_model":"Qwen/Qwen3-0.6B-Base","base_chain":["Qwen/Qwen3-0.6B","Qwen/Qwen3-0.6B-Base"],"weights_repo":"Hanno-Labs/bosun-v3.1-0.6b","served_checkpoint":null,"model_url":"https://huggingface.co/Hanno-Labs/bosun-v3.1-0.6b","code_url":"https://github.com/Hanno-Labs/jev-compatible-server","unanswered_share":0},{"engine":"bosun-v3.1-1.7b","name":"Bosun v3.1 1.7B","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":1720574976,"params_from_name":false,"params_estimated":false,"trained_params":18481152,"trained_bytes":71879720,"base_model":"Qwen/Qwen3-1.7B-Base","base_chain":["Qwen/Qwen3-1.7B","Qwen/Qwen3-1.7B-Base"],"weights_repo":"Hanno-Labs/bosun-v3.1-1.7b","served_checkpoint":null,"model_url":"https://huggingface.co/Hanno-Labs/bosun-v3.1-1.7b","code_url":"https://github.com/Hanno-Labs/jev-compatible-server","unanswered_share":0},{"engine":"clm-v0.1-8b","name":"CLM-v0.1-8B","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":8190735360,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":75557149,"base_model":"Qwen/Qwen3-8B-Base","base_chain":["Qwen/Qwen3-8B","Qwen/Qwen3-8B-Base"],"weights_repo":"Contrastive-LM/CLM-v0.1-8B","served_checkpoint":"Contrastive-LM/CLM-v0.1-8B","model_url":"https://huggingface.co/Contrastive-LM/CLM-v0.1-8B","code_url":"https://github.com/Contrastive-LM/CLM","unanswered_share":0},{"engine":"decider-2b-fp8-http","name":"Decider 2B","variant":"FP8","kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_from_name":false,"params_estimated":false,"trained_params":1881825088,"trained_bytes":3763692048,"base_model":"Qwen/Qwen3.5-2B-Base","base_chain":["Qwen/Qwen3.5-2B-Base"],"weights_repo":"Mapika/decider-2b","served_checkpoint":"Mapika/decider-2b","model_url":"https://huggingface.co/Mapika/decider-2b","code_url":null,"unanswered_share":3.32e-05},{"engine":"decider-35b-nvfp4","name":"Decider 35B-A3B","variant":"NVFP4","kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":35951822704,"params_from_name":false,"params_estimated":false,"trained_params":17851937408,"trained_bytes":21013206328,"base_model":"Qwen/Qwen3.5-35B-A3B-Base","base_chain":["Mapika/decider-35b-a3b","Qwen/Qwen3.5-35B-A3B-Base"],"weights_repo":"Mapika/decider-35b-a3b-nvfp4","served_checkpoint":"Mapika/decider-35b-a3b-nvfp4","model_url":"https://huggingface.co/Mapika/decider-35b-a3b-nvfp4","code_url":null,"unanswered_share":0},{"engine":"decider-4b","name":"Decider 4B","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4205751296,"trained_bytes":8411558400,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B-Base"],"weights_repo":"Mapika/decider-4b","served_checkpoint":null,"model_url":"https://huggingface.co/Mapika/decider-4b","code_url":"https://github.com/Mapika/decider","unanswered_share":0},{"engine":"decider-chat-gemma4-31b","name":"Decider chat · Gemma-4-31B","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":32682372656,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"google/gemma-4-31B","base_chain":["google/gemma-4-31B"],"weights_repo":null,"served_checkpoint":"google/gemma-4-31B-it","model_url":null,"code_url":"https://github.com/Mapika/decider","unanswered_share":0},{"engine":"decider-chat-qwen3.6-27b","name":"Decider chat · Qwen3.6-27B","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3.6-27B","base_chain":null,"weights_repo":null,"served_checkpoint":"Qwen/Qwen3.6-27B","model_url":null,"code_url":"https://github.com/Mapika/decider","unanswered_share":0},{"engine":"decision-eos-0.8b","name":"Decision 1.0 Eos","variant":null,"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_from_name":false,"params_estimated":false,"trained_params":753446208,"trained_bytes":1509034472,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_chain":["Qwen/Qwen3.5-0.8B","Qwen/Qwen3.5-0.8B-Base"],"weights_repo":"llm-semantic-router/Decision-1.0-Eos-0.8B","served_checkpoint":null,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Eos-0.8B","code_url":null,"unanswered_share":1.67e-05},{"engine":"decision-kai-7185f514","name":"Decision 1.0 Kai","variant":null,"kind":"head / adapter","technique":"encoder","technique_name":"encoder","served_params":307797035,"params_from_name":false,"params_estimated":true,"trained_params":571909635,"trained_bytes":2287687932,"base_model":"jhu-clsp/mmBERT-base","base_chain":["llm-semantic-router/Vela-1.0-Encoder-307M","jhu-clsp/mmBERT-base"],"weights_repo":"llm-semantic-router/Decision-1.0-Kai","served_checkpoint":null,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Kai","code_url":null,"unanswered_share":0.0027},{"engine":"decision-lex","name":"Decision 1.0 Lex","variant":null,"kind":"head / adapter","technique":"encoder","technique_name":"encoder","served_params":307797035,"params_from_name":false,"params_estimated":true,"trained_params":571909635,"trained_bytes":2287687932,"base_model":"jhu-clsp/mmBERT-base","base_chain":["llm-semantic-router/Decision-1.0-Kai","llm-semantic-router/Vela-1.0-Encoder-307M","jhu-clsp/mmBERT-base"],"weights_repo":"llm-semantic-router/decision-1.0-lex","served_checkpoint":null,"model_url":"https://huggingface.co/llm-semantic-router/decision-1.0-lex","code_url":null,"unanswered_share":0.0027},{"engine":"decision-lux-9b","name":"Decision 1.0 Lux","variant":null,"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_from_name":false,"params_estimated":false,"trained_params":7940895744,"trained_bytes":15890261552,"base_model":"Qwen/Qwen3.5-9B-Base","base_chain":["Qwen/Qwen3.5-9B","Qwen/Qwen3.5-9B-Base"],"weights_repo":"llm-semantic-router/Decision-1.0-Lux-9B","served_checkpoint":null,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Lux-9B","code_url":null,"unanswered_share":0},{"engine":"decision-nox-0bb83350","name":"Decision 1.0 Nox","variant":null,"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4208383488,"trained_bytes":8422078976,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B","Qwen/Qwen3.5-4B-Base"],"weights_repo":"llm-semantic-router/Decision-1.0-Nox","served_checkpoint":null,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Nox","code_url":null,"unanswered_share":0},{"engine":"decision-sol-0665a411","name":"Decision 1.0 Sol","variant":null,"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_from_name":false,"params_estimated":false,"trained_params":1883930944,"trained_bytes":3772109600,"base_model":"Qwen/Qwen3.5-2B-Base","base_chain":["Qwen/Qwen3.5-2B","Qwen/Qwen3.5-2B-Base"],"weights_repo":"llm-semantic-router/Decision-1.0-Sol","served_checkpoint":null,"model_url":"https://huggingface.co/llm-semantic-router/Decision-1.0-Sol","code_url":null,"unanswered_share":0},{"engine":"djev","name":"djev","variant":null,"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"google/diffusiongemma-26B-A4B-it","base_chain":null,"weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it","model_url":null,"code_url":"https://github.com/Davipar/djev-dev","unanswered_share":0.0097},{"engine":"gliner-base","name":"GLiNER 2.5 base","variant":null,"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":193581591,"params_from_name":false,"params_estimated":false,"trained_params":193581591,"trained_bytes":774366564,"base_model":null,"base_chain":null,"weights_repo":"fastino/gliner2.5-base-v1","served_checkpoint":null,"model_url":"https://huggingface.co/fastino/gliner2.5-base-v1","code_url":null,"unanswered_share":7.48e-05},{"engine":"gliner-decide","name":"GLiNER2.5-Decide","variant":null,"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":486444053,"params_from_name":false,"params_estimated":false,"trained_params":486444053,"trained_bytes":1945828140,"base_model":"fastino/gliner2-large-v1","base_chain":["fastino/gliner2-large-v1"],"weights_repo":"fastino/GLiNER2.5-Decide","served_checkpoint":null,"model_url":"https://huggingface.co/fastino/GLiNER2.5-Decide","code_url":"https://github.com/fastino-ai/GLiNER2","unanswered_share":0},{"engine":"gliner-multi","name":"GLiNER 2.5 multilingual","variant":null,"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":287355159,"params_from_name":false,"params_estimated":false,"trained_params":287355159,"trained_bytes":1149461028,"base_model":null,"base_chain":null,"weights_repo":"fastino/gliner2.5-multi-v1","served_checkpoint":null,"model_url":"https://huggingface.co/fastino/gliner2.5-multi-v1","code_url":null,"unanswered_share":7.48e-05},{"engine":"gliner-small","name":"GLiNER 2.5 small","variant":null,"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiNER2","served_params":73881879,"params_from_name":false,"params_estimated":false,"trained_params":73881879,"trained_bytes":295567700,"base_model":null,"base_chain":null,"weights_repo":"fastino/gliner2.5-small-v1","served_checkpoint":null,"model_url":"https://huggingface.co/fastino/gliner2.5-small-v1","code_url":null,"unanswered_share":7.48e-05},{"engine":"harsha","name":"Qwen-2.5-1B-RLCD","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":1543714304,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen2.5-1.5B","base_chain":["Qwen/Qwen2.5-1.5B"],"weights_repo":null,"served_checkpoint":null,"model_url":null,"code_url":"https://huggingface.co/harshatheg/Qwen-2.5-1B-RLCD","unanswered_share":0.0032},{"engine":"hopper-g-1.2","name":"Hopper (G) 1.2","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":32464896,"trained_bytes":129927008,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B","Qwen/Qwen3.5-4B-Base"],"weights_repo":"HopitAI/hopper-g","served_checkpoint":null,"model_url":"https://huggingface.co/HopitAI/hopper-g","code_url":"https://github.com/hopit-ai/hopper","unanswered_share":0},{"engine":"jeff-uncapped","name":"jeff","variant":null,"kind":"inference technique","technique":"GLi family","technique_name":"GLiFormer","served_params":575683963,"params_from_name":false,"params_estimated":true,"trained_params":null,"trained_bytes":null,"base_model":"knowledgator/gliformer-large-v1","base_chain":null,"weights_repo":null,"served_checkpoint":"knowledgator/gliformer-large-v1","model_url":null,"code_url":"https://github.com/logan-markewich/jeff","unanswered_share":0},{"engine":"jev-omni","name":"Jev-Omni","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":11959730224,"params_from_name":false,"params_estimated":false,"trained_params":11959730224,"trained_bytes":23923515487,"base_model":"google/gemma-4-12B","base_chain":["google/gemma-4-12B-it","google/gemma-4-12B"],"weights_repo":"akhilaaa3/Jev-Omni","served_checkpoint":null,"model_url":"https://huggingface.co/akhilaaa3/Jev-Omni","code_url":null,"unanswered_share":0},{"engine":"jevfire-uncapped","name":"Jevfire","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3.8-27B","base_chain":["Qwen/Qwen3.8-27B"],"weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B-FP8","model_url":null,"code_url":"https://github.com/kikoncuo/jevfire","unanswered_share":0},{"engine":"jevk5","name":"JevK5","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4205751296,"trained_bytes":8411558400,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B-Base"],"weights_repo":"alibiserikbay/JevK5","served_checkpoint":null,"model_url":"https://huggingface.co/alibiserikbay/JevK5","code_url":"https://github.com/allebee/jevk5","unanswered_share":0},{"engine":"jobe","name":"Jobe","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B-Base"],"weights_repo":null,"served_checkpoint":"Qwen/Qwen3.5-4B","model_url":null,"code_url":"https://github.com/MantisShrimpdev/jobe","unanswered_share":0},{"engine":"joshua-diffusion-full","name":"open-jev (JoshuaSP)","variant":null,"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"google/diffusiongemma-26B-A4B-it","base_chain":null,"weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it","model_url":null,"code_url":"https://github.com/JoshuaSP/open-jev","unanswered_share":0},{"engine":"kev-0.8b-raised","name":"Kev 0.8B","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_from_name":false,"params_estimated":false,"trained_params":10822656,"trained_bytes":45441727,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_chain":null,"weights_repo":"jaredpalmer/kev-0.8b","served_checkpoint":"jaredpalmer/kev-0.8b","model_url":"https://huggingface.co/jaredpalmer/kev-0.8b","code_url":"https://github.com/jaredpalmer/kev","unanswered_share":0},{"engine":"kev-4b-raised","name":"Kev 4B","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":32464896,"trained_bytes":135172799,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":null,"weights_repo":"jaredpalmer/kev-4b","served_checkpoint":"jaredpalmer/kev-4b","model_url":"https://huggingface.co/jaredpalmer/kev-4b","code_url":"https://github.com/jaredpalmer/kev","unanswered_share":0},{"engine":"kev-9b-raised","name":"Kev 9B","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_from_name":false,"params_estimated":false,"trained_params":43278336,"trained_bytes":181572591,"base_model":"Qwen/Qwen3.5-9B-Base","base_chain":null,"weights_repo":"jaredpalmer/kev-9b","served_checkpoint":"jaredpalmer/kev-9b","model_url":"https://huggingface.co/jaredpalmer/kev-9b","code_url":"https://github.com/jaredpalmer/kev","unanswered_share":0},{"engine":"laya-1e28ac20","name":"Laya","variant":null,"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":421293830,"params_from_name":false,"params_estimated":false,"trained_params":421293830,"trained_bytes":2329053944,"base_model":null,"base_chain":null,"weights_repo":"convaiinnovations/laya","served_checkpoint":null,"model_url":"https://huggingface.co/convaiinnovations/laya","code_url":"https://github.com/NandhaKishorM/laya","unanswered_share":0.0027},{"engine":"lfm2600","name":"LFM2.5-2.6B-RLCD","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2697198592,"params_from_name":false,"params_estimated":false,"trained_params":2697198592,"trained_bytes":5394427456,"base_model":"LiquidAI/LFM2.5-2.6B-Base","base_chain":["LiquidAI/LFM2.5-2.6B","LiquidAI/LFM2.5-2.6B-Base"],"weights_repo":"monotykamary/LFM2.5-2.6B-RLCD","served_checkpoint":null,"model_url":"https://huggingface.co/monotykamary/LFM2.5-2.6B-RLCD","code_url":null,"unanswered_share":0.0044},{"engine":"lfm350-sdpa","name":"LFM2.5-350M-RLCD","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":354483968,"params_from_name":false,"params_estimated":false,"trained_params":354483968,"trained_bytes":708984464,"base_model":null,"base_chain":null,"weights_repo":"notnotsamuel/LFM2.5-350M-RLCD","served_checkpoint":null,"model_url":"https://huggingface.co/notnotsamuel/LFM2.5-350M-RLCD","code_url":null,"unanswered_share":0},{"engine":"lumma-fev-0.1b","name":"Lumma-Fev-0.1B","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":153412928,"params_from_name":false,"params_estimated":false,"trained_params":154102848,"trained_bytes":309928008,"base_model":"FrontiersMind/Nandi-Mini-150M","base_chain":["FrontiersMind/Nandi-Mini-150M"],"weights_repo":"FrontiersMind/Lumma-fev-0.1b","served_checkpoint":null,"model_url":"https://huggingface.co/FrontiersMind/Lumma-fev-0.1b","code_url":null,"unanswered_share":0.0175},{"engine":"lumma-fev-0.6b","name":"Lumma-Fev-0.6B","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":649281964,"params_from_name":false,"params_estimated":false,"trained_params":649282476,"trained_bytes":1300071072,"base_model":"FrontiersMind/Lumma-0.6B-Base","base_chain":["FrontiersMind/Lumma-0.6B-Base"],"weights_repo":"FrontiersMind/Lumma-fev-0.6b","served_checkpoint":null,"model_url":"https://huggingface.co/FrontiersMind/Lumma-fev-0.6b","code_url":null,"unanswered_share":3.34e-05},{"engine":"metask-jev-4b","name":"Metask-Jev-4B","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4539265536,"trained_bytes":9078620536,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B-Base"],"weights_repo":"wayfind/metask-jev-4b-policy-mix","served_checkpoint":null,"model_url":"https://huggingface.co/wayfind/metask-jev-4b-policy-mix","code_url":"https://github.com/metask-ai/metask-jev","unanswered_share":0.1205},{"engine":"mini-jev","name":"mini-jev","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4022468096,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3-4B-Instruct-2507","base_chain":null,"weights_repo":null,"served_checkpoint":"Qwen/Qwen3-4B-Instruct-2507","model_url":null,"code_url":"https://github.com/r-ms/mini-jev","unanswered_share":0.1205},{"engine":"mojev","name":"MoJev","variant":null,"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_from_name":false,"params_estimated":false,"trained_params":854036544,"trained_bytes":1965507264,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_chain":["Qwen/Qwen3.5-0.8B","Qwen/Qwen3.5-0.8B-Base"],"weights_repo":"MoLeMo-Lab/mojev","served_checkpoint":null,"model_url":"https://huggingface.co/MoLeMo-Lab/mojev","code_url":"https://github.com/MoLeMo-Lab/mojev","unanswered_share":0},{"engine":"nimble-v2","name":"Bespoke Nimble 9B v2","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_from_name":false,"params_estimated":false,"trained_params":43278336,"trained_bytes":173188512,"base_model":"Qwen/Qwen3.5-9B-Base","base_chain":["Qwen/Qwen3.5-9B","Qwen/Qwen3.5-9B-Base"],"weights_repo":"bespokelabs/Bespoke-Nimble-9B-v2","served_checkpoint":null,"model_url":"https://huggingface.co/bespokelabs/Bespoke-Nimble-9B-v2","code_url":"https://github.com/bespokelabsai/nimble","unanswered_share":0},{"engine":"openvons","name":"openvons","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4022468096,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3-4B-Instruct-2507","base_chain":null,"weights_repo":null,"served_checkpoint":"Qwen/Qwen3-4B-Instruct-2507","model_url":null,"code_url":"https://github.com/genai-craft/openvons","unanswered_share":0},{"engine":"pngwn-space-uncapped-full","name":"open-jev (pngwn)","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":30476800,"trained_bytes":121956232,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":null,"weights_repo":"pngwn/system-one-qwen3.5-4b-scorer-v2b","served_checkpoint":null,"model_url":"https://huggingface.co/pngwn/system-one-qwen3.5-4b-scorer-v2b","code_url":null,"unanswered_share":0},{"engine":"razorback-one-read-3e296f08","name":"openjev (razorback16)","variant":"one read, NVFP4","kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"google/diffusiongemma-26B-A4B-it","base_chain":["google/diffusiongemma-26B-A4B-it"],"weights_repo":null,"served_checkpoint":"nvidia/diffusiongemma-26B-A4B-it-NVFP4","model_url":null,"code_url":"https://github.com/vllm-project/vllm/pull/57250","unanswered_share":0},{"engine":"reflex-27b-v2","name":"reflex 27B","variant":"FP8 · wide choice","kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3.8-27B","base_chain":["Qwen/Qwen3.8-27B"],"weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B-FP8","model_url":null,"code_url":"https://github.com/kshetrajna12/reflex","unanswered_share":0},{"engine":"rune-26b-a4b-v3","name":"Surogate Rune 26B-A4B v3","variant":"bf16","kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":25805936206,"params_from_name":false,"params_estimated":false,"trained_params":25805936206,"trained_bytes":51612008580,"base_model":"google/gemma-4-26B-A4B-it","base_chain":null,"weights_repo":"surogate/rune-26b-a4b-GGUF","served_checkpoint":"bd4a7cbbed66af1a95dfbac12cabf52e6e677411","model_url":"https://huggingface.co/surogate/rune-26b-a4b-GGUF","code_url":"https://github.com/invergent-ai/surogate","unanswered_share":8.34e-06},{"engine":"semif","name":"SemIf","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B-Base"],"weights_repo":null,"served_checkpoint":null,"model_url":null,"code_url":"https://github.com/TheoLeeCJ/openjev","unanswered_share":0.1273},{"engine":"solomon-v11-bf16-encoding","name":"Solomon v1.1","variant":null,"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":225444865,"trained_bytes":903468342,"base_model":"Qwen/Qwen3.8-27B","base_chain":null,"weights_repo":"DoccyHealth/Solomon","served_checkpoint":null,"model_url":"https://huggingface.co/DoccyHealth/Solomon","code_url":null,"unanswered_share":0.2285},{"engine":"tev1-0.8b","name":"Tev1-0.8B-experimental","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_from_name":false,"params_estimated":false,"trained_params":873438784,"trained_bytes":1746942600,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_chain":["Qwen/Qwen3.5-0.8B","Qwen/Qwen3.5-0.8B-Base"],"weights_repo":"togethercomputer/Tev1-0.8B-experimental","served_checkpoint":null,"model_url":"https://huggingface.co/togethercomputer/Tev1-0.8B-experimental","code_url":"https://github.com/togethercomputer/tev1","unanswered_share":0.1229},{"engine":"tev1-4b","name":"Tev1-4B-experimental","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4659865088,"trained_bytes":9319828096,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B","Qwen/Qwen3.5-4B-Base"],"weights_repo":"togethercomputer/Tev1-4B-experimental","served_checkpoint":null,"model_url":"https://huggingface.co/togethercomputer/Tev1-4B-experimental","code_url":"https://github.com/togethercomputer/tev1","unanswered_share":0.1229},{"engine":"this-that-1.2","name":"this-that 1.2","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":1881825088,"params_from_name":false,"params_estimated":false,"trained_params":1881825088,"trained_bytes":3763692048,"base_model":"Mapika/decider-2b","base_chain":null,"weights_repo":"flock-io/this-that-model-1.2","served_checkpoint":null,"model_url":"https://huggingface.co/flock-io/this-that-model-1.2","code_url":"https://github.com/FLock-io/this-that-model","unanswered_share":0},{"engine":"verdict-8af2496e","name":"Verdict","variant":null,"kind":"full fine-tune","technique":"GLi family","technique_name":"GLiClass","served_params":151378176,"params_from_name":false,"params_estimated":false,"trained_params":151378176,"trained_bytes":605529340,"base_model":"knowledgator/gliclass-modern-base-v2.0","base_chain":["knowledgator/gliclass-modern-base-v2.0"],"weights_repo":"heman10x/rlcd-modernbert-151m","served_checkpoint":null,"model_url":"https://huggingface.co/heman10x/rlcd-modernbert-151m","code_url":"https://github.com/Heman10x-NGU/Verdict-open-jev","unanswered_share":0.5908},{"engine":"vllm-pr57250","name":"mmastrac diffusiongemma (vLLM PR 57250)","variant":null,"kind":"inference technique","technique":"diffusion","technique_name":"diffusion","served_params":25823778864,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"google/diffusiongemma-26B-A4B-it","base_chain":null,"weights_repo":null,"served_checkpoint":"google/diffusiongemma-26B-A4B-it","model_url":null,"code_url":"https://github.com/vllm-project/vllm/pull/57250","unanswered_share":0.1207},{"engine":"winnow-12b-q8","name":"Winnow-12B","variant":"Q8_0","kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":11959730224,"params_from_name":false,"params_estimated":false,"trained_params":11907350576,"trained_bytes":12669646592,"base_model":"google/gemma-4-12B","base_chain":["google/gemma-4-12B-it","google/gemma-4-12B"],"weights_repo":"EldanRing/Winnow-12B","served_checkpoint":"EldanRing/Winnow-12B","model_url":"https://huggingface.co/EldanRing/Winnow-12B","code_url":"https://github.com/EldanRing/winnow-inference","unanswered_share":0},{"engine":"winnow-e4b-q8","name":"Winnow-E4B","variant":"Q8_0","kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":7996156490,"params_from_name":false,"params_estimated":false,"trained_params":7518069290,"trained_bytes":8005437472,"base_model":"google/gemma-4-E4B","base_chain":["google/gemma-4-E4B-it","google/gemma-4-E4B"],"weights_repo":"EldanRing/Winnow-E4B","served_checkpoint":"EldanRing/Winnow-E4B","model_url":"https://huggingface.co/EldanRing/Winnow-E4B","code_url":"https://github.com/EldanRing/winnow-inference","unanswered_share":0},{"engine":"xor","name":"Xor","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":35951822704,"params_from_name":false,"params_estimated":false,"trained_params":35107181936,"trained_bytes":70214495944,"base_model":"Qwen/Qwen3.6-35B-A3B","base_chain":["Qwen/Qwen3.6-35B-A3B"],"weights_repo":"juspay/xor","served_checkpoint":null,"model_url":"https://huggingface.co/juspay/xor","code_url":null,"unanswered_share":0.1209},{"engine":"lev","name":"lev","variant":null,"kind":"LoRA + head","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":42467328,"trained_bytes":184601885,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B-Base"],"weights_repo":"interfaze-ai/lev","served_checkpoint":null,"model_url":"https://huggingface.co/interfaze-ai/lev","code_url":"https://github.com/Abhinavexists/lev","unanswered_share":0},{"engine":"intern-decision-0.8b","name":"Intern-Decision-0.8B","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_from_name":false,"params_estimated":false,"trained_params":852985920,"trained_bytes":1706034736,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_chain":["Qwen/Qwen3.5-0.8B","Qwen/Qwen3.5-0.8B-Base"],"weights_repo":"internlm/Intern-Decision-0.8B","served_checkpoint":null,"model_url":"https://huggingface.co/internlm/Intern-Decision-0.8B","code_url":"https://github.com/internlm/Intern-Decision","unanswered_share":0},{"engine":"intern-decision-2b","name":"Intern-Decision-2B","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":2274069824,"params_from_name":false,"params_estimated":false,"trained_params":2213241664,"trained_bytes":4426562832,"base_model":"Qwen/Qwen3.5-2B-Base","base_chain":["Qwen/Qwen3.5-2B","Qwen/Qwen3.5-2B-Base"],"weights_repo":"internlm/Intern-Decision-2B","served_checkpoint":null,"model_url":"https://huggingface.co/internlm/Intern-Decision-2B","code_url":"https://github.com/internlm/Intern-Decision","unanswered_share":0},{"engine":"intern-decision-4b","name":"Intern-Decision-4B","variant":null,"kind":"full fine-tune","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4539265536,"trained_bytes":9078626736,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B","Qwen/Qwen3.5-4B-Base"],"weights_repo":"internlm/Intern-Decision-4B","served_checkpoint":null,"model_url":"https://huggingface.co/internlm/Intern-Decision-4B","code_url":"https://github.com/internlm/Intern-Decision","unanswered_share":0},{"engine":"simple-jev-qwen3.8-27b","name":"simple-jev · Qwen3.8-27B (featherless)","variant":null,"kind":"inference technique","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":null,"trained_bytes":null,"base_model":"Qwen/Qwen3.8-27B","base_chain":null,"weights_repo":null,"served_checkpoint":"Qwen/Qwen3.8-27B","model_url":null,"code_url":"https://github.com/featherless-ai/simple-jev","unanswered_share":0},{"engine":"neohorse-jev-4b","name":"NeoHorse-Jev-4B","variant":null,"kind":"head / adapter","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4540576768,"trained_bytes":9083860816,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["TokenRhythm/NeoHorse-1-4B","Qwen/Qwen3.5-4B","Qwen/Qwen3.5-4B-Base"],"weights_repo":"TokenRhythm/NeoHorse-Jev-4B","served_checkpoint":null,"model_url":"https://huggingface.co/TokenRhythm/NeoHorse-Jev-4B","code_url":"https://github.com/TokenRhythm/NeoHorse/tree/main/jev","unanswered_share":0},{"engine":"julia-1","name":"Julia 1","variant":null,"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":140908331,"params_from_name":false,"params_estimated":true,"trained_params":144292870,"trained_bytes":577189056,"base_model":"jhu-clsp/mmBERT-small","base_chain":["jhu-clsp/mmBERT-small"],"weights_repo":"SupersonicLabs/Julia-1","served_checkpoint":null,"model_url":"https://huggingface.co/SupersonicLabs/Julia-1","code_url":null,"unanswered_share":8.34e-05},{"engine":"jebadiah-27b","name":"Jebadiah 27B","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":116727808,"trained_bytes":null,"base_model":"Qwen/Qwen3.8-27B","base_chain":null,"weights_repo":"frontier-infra/jebadiah-27b","served_checkpoint":"1c0d794fc24b24b15175c6a912654fa5cd76e1bb","model_url":"https://huggingface.co/frontier-infra/jebadiah-27b","code_url":"https://github.com/getainode/jebadiah","unanswered_share":0},{"engine":"lavoir","name":"Lavoir","variant":null,"kind":"full fine-tune","technique":"encoder","technique_name":"encoder","served_params":395881664,"params_from_name":false,"params_estimated":false,"trained_params":421562382,"trained_bytes":1686272004,"base_model":"answerdotai/ModernBERT-large","base_chain":null,"weights_repo":"moganai/lavoir","served_checkpoint":"4c5eaeb99b2368f1416b9893c7aacc58235aa09f","model_url":"https://huggingface.co/moganai/lavoir","code_url":"https://github.com/moganai/lavoir","unanswered_share":0.0027},{"engine":"jpt-0.8b","name":"JPT-0.8B","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":873438784,"params_from_name":false,"params_estimated":false,"trained_params":10822656,"trained_bytes":1706035272,"base_model":"Qwen/Qwen3.5-0.8B-Base","base_chain":["Qwen/Qwen3.5-0.8B","Qwen/Qwen3.5-0.8B-Base"],"weights_repo":"kirp/jpt-0.8b","served_checkpoint":"5921ae03fbaff98fcbe781c1ff3f1200dc897fd1","model_url":"https://huggingface.co/kirp/jpt-0.8b","code_url":"https://github.com/tic-top/llm2jev","unanswered_share":0},{"engine":"jpt-4b","name":"JPT-4B","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":32464896,"trained_bytes":9078627496,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B","Qwen/Qwen3.5-4B-Base"],"weights_repo":"kirp/jpt-4b","served_checkpoint":"277ae37b7a6398a8ec3b86712ac16c1991ab6ef1","model_url":"https://huggingface.co/kirp/jpt-4b","code_url":"https://github.com/tic-top/llm2jev","unanswered_share":0},{"engine":"jpt-9b","name":"JPT-9B","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":9653104368,"params_from_name":false,"params_estimated":false,"trained_params":43278336,"trained_bytes":18819728184,"base_model":"Qwen/Qwen3.5-9B-Base","base_chain":["Qwen/Qwen3.5-9B","Qwen/Qwen3.5-9B-Base"],"weights_repo":"kirp/jpt-9b","served_checkpoint":"7114b0c3d9bea6b82dfa2d0691e8d5562cd26d4e","model_url":"https://huggingface.co/kirp/jpt-9b","code_url":"https://github.com/tic-top/llm2jev","unanswered_share":0},{"engine":"eikos-27b-fp8","name":"Eikos-27B","variant":"FP8","kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":27781427952,"params_from_name":false,"params_estimated":false,"trained_params":27781427952,"trained_bytes":31220328864,"base_model":"Qwen/Qwen3.8-27B","base_chain":null,"weights_repo":"caiovicentino1/Eikos-27B-FP8","served_checkpoint":null,"model_url":"https://huggingface.co/caiovicentino1/Eikos-27B-FP8","code_url":"https://github.com/caiovicentino/eikos","unanswered_share":0},{"engine":"jet-v6.2","name":"Jet v6.2","variant":null,"kind":"LoRA","technique":"autoregressive","technique_name":"autoregressive","served_params":4659865088,"params_from_name":false,"params_estimated":false,"trained_params":4205751296,"trained_bytes":8411558448,"base_model":"Qwen/Qwen3.5-4B-Base","base_chain":["Qwen/Qwen3.5-4B","Qwen/Qwen3.5-4B-Base"],"weights_repo":"michaljach/jet","served_checkpoint":null,"model_url":"https://huggingface.co/michaljach/jet","code_url":"https://github.com/michaljach/jet","unanswered_share":0.002}]},"latency":{"reproductions":{"summary":"Latency is measured on its own, after scoring: every entrant runs the same 750 rows (stratified by benchmark and prompt length across the 0.2 scope, plus 10 untimed warm-up rows) as one process sending one request at a time on 1 x NVIDIA RTX PRO 6000, using the fastest inference path the model or its code supports. The figure is the wall time of one row, request in to all answers out, measured the same way for in-process and HTTP engines; board medians and p95s come from this run, not from the scoring runs.","aggregate":"Median and p95 over the timed rows that returned an answer; per-benchmark medians are shown only where the sample holds at least 5 rows of that benchmark. A bootstrap on the full single-process runs of ten entrants put the sample median within 5% and the p95 within 10% of a full run at 95% confidence. Earlier board figures came from the scoring runs, some timed with 6 to 12 processes sharing a card or with queue waits removed; they are no longer shown.","protocol":{"protocol":"latency-v1","frozen_utc":"2026-09-26","seed":20260926,"sample":{"rows":"selected-rows.jsonl","sha256":"d6666eb50235b0cd77a5b07391b7ac551dea5a3e89f2802ccf92620f41a63235","timed":750,"warmup":10,"design":"benchmark+length: systematic sample (random start, seed 20260926) over the 151,024-row live scope sorted by benchmark then payload length, so each benchmark gets rows in proportion to its share of the scope and each benchmark is spread across its length range","scope":"0.2 live scope: release-v2 rows (ACOS subset) minus SimpleBench and excluded questions, plus the 7 added catalogs","per_catalog":{"1":9,"2":5,"3":2,"4":16,"5":27,"6":50,"9":0,"10":13,"11":0,"12":16,"20":25,"21":13,"22":10,"23":25,"24":70,"25":1,"26":11,"27":6,"28":7,"29":49,"30":13,"31":25,"32":4,"33":8,"36":3,"37":25,"38":8,"39":4,"40":23,"41":15,"42":28,"43":2,"44":25,"45":3,"48":50,"50":8,"56":10,"57":60,"58":27,"59":14,"61":20,"62":18,"64":2},"refusals":"every entrant runs the same 760 rows; rows it refuses on its board run are recorded but not timed"},"load":"one process, one request at a time, no concurrency, on exactly one RTX PRO 6000 (96 GB); no multi-GPU entrants","path":"fastest inference path the model or its code supports (CUDA graphs, fused/fla kernels, flash attention, compiled paths, vLLM/SGLang, prefix/KV reuse, batching questions inside a row); rows that do not fit the fastest path fall back to the next-fastest and are recorded; the path is listed per entrant","timing":"end-to-end wall time of one row (request in, all answers out), total_wall_ms from run.py --suite latency-v1 (CUDA-synchronized in-process, or one sequential HTTP client against the author server); job/server startup and model load excluded","report":"median and p95 over the 750 timed rows, plus per-benchmark medians (catalogs with fewer than 5 sampled rows are shown as too few)","parity":"answers compared with the entrant board rows: choice flips and max |dp|; informational only, a faster path is not rejected for drift","jev":"network API latency with one sequential client if the jev-release-v1 credentials still work; otherwise the old figure, flagged","evidence":{"method":"bootstrap (B=4000) of the design at each n on per-row wall times from full runs that were single-process (one runner per card or per HF Job; in-file row concurrency 1, AutoJev one shard per card); error = 95th percentile of |sample stat / full-run stat - 1|; criterion median within 5% and p95 within 10% for every probe","probes":["lfm350-sdpa","laya-1e28ac20","decision-eos-0.8b","harsha","this-that-1.2","mini-jev","kev-4b-raised","winnow-12b-q8","openvons","pplx-decider-v1-27b"],"reference":{"lfm350-sdpa":{"median":25.9357,"p95":172.8058,"rows":151024},"laya-1e28ac20":{"median":18.2852,"p95":49.7916,"rows":151007},"decision-eos-0.8b":{"median":36.0447,"p95":76.2549,"rows":151021},"harsha":{"median":41.8783,"p95":73.3379,"rows":151024},"this-that-1.2":{"median":38.5695,"p95":64.3355,"rows":151024},"mini-jev":{"median":63.8445,"p95":162.631,"rows":136524},"kev-4b-raised":{"median":52.1934,"p95":144.7569,"rows":151024},"winnow-12b-q8":{"median":67.0422,"p95":355.6466,"rows":151024},"openvons":{"median":17.0598,"p95":104.556,"rows":151024},"autojev-27b":{"median":96.9983,"p95":1123.6942,"rows":151024}},"table":[{"design":"benchmark+length","n":100,"worst_median_err95":0.0806,"worst_p95_err95":0.3214,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"mini-jev","passes":false},{"design":"benchmark+length","n":200,"worst_median_err95":0.0739,"worst_p95_err95":0.2846,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"benchmark+length","n":300,"worst_median_err95":0.0595,"worst_p95_err95":0.1142,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"this-that-1.2","passes":false},{"design":"benchmark+length","n":400,"worst_median_err95":0.0554,"worst_p95_err95":0.1135,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"this-that-1.2","passes":false},{"design":"benchmark+length","n":500,"worst_median_err95":0.0518,"worst_p95_err95":0.0904,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"this-that-1.2","passes":false},{"design":"benchmark+length","n":600,"worst_median_err95":0.0519,"worst_p95_err95":0.0746,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"this-that-1.2","passes":false},{"design":"benchmark+length","n":750,"worst_median_err95":0.0493,"worst_p95_err95":0.0677,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"mini-jev","passes":true},{"design":"benchmark+length","n":1000,"worst_median_err95":0.0454,"worst_p95_err95":0.0467,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"mini-jev","passes":true},{"design":"length","n":100,"worst_median_err95":0.0924,"worst_p95_err95":0.3048,"worst_median_entrant":"winnow-12b-q8","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"length","n":200,"worst_median_err95":0.0736,"worst_p95_err95":0.2538,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"length","n":300,"worst_median_err95":0.0721,"worst_p95_err95":0.1758,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"length","n":400,"worst_median_err95":0.0629,"worst_p95_err95":0.1674,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"length","n":500,"worst_median_err95":0.0596,"worst_p95_err95":0.1513,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"length","n":600,"worst_median_err95":0.0569,"worst_p95_err95":0.1554,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"length","n":750,"worst_median_err95":0.054,"worst_p95_err95":0.1205,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false},{"design":"length","n":1000,"worst_median_err95":0.0531,"worst_p95_err95":0.1029,"worst_median_entrant":"pplx-decider-v1-27b","worst_p95_entrant":"decision-eos-0.8b","passes":false}],"chosen_n":750,"median_only_n":{"all probes":750,"all probes except AutoJev-27B (bimodal, two-card shards)":200},"rejected":{"eligible-rows pool (rows every entrant answered)":"biased: p95 low by 6-12% and pplx-decider-v1-27b median high by 7% at any n, because refusal-driven eligibility drops the longest rows; see bootstrap-eligible.json","benchmark-only stratification":"needs n=2000; see bootstrap.json","length-only stratification":"fails p95 at n=1000"}}},"methods":[{"method":"latency-v1 single process: GLiNER2 Classifier.classify in bf16 (board fp32; bf16 is equal on median and lower on p95), fleet venv torch 2.14.0+cu130, board guard lifts","engines":["gliner-base","gliner-multi","gliner-small"]},{"method":"latency-v1 single process: author DecisionEngine.predict on the whole row, SDPA, max_length 262144 and the board lifts; transformers fast path enabled with causal-conv1d 1.7.0 (fla 0.4.2 gated delta + CUDA conv). The board ran the torch fallback because causal-conv1d was absent; that run is kept as latency-v1-full.torch-fallback. venv-intern torch 2.9.1+cu128","engines":["intern-decision-0.8b","intern-decision-2b","intern-decision-4b"]},{"method":"latency-v1 single process: author RegistryRuntime.decide with the row-batched backend (all questions of a row scored together, prompts rendered exactly as model.predict); board used exact per-question mode; bf16, torch 2.14.0+cu130","engines":["bosun-v3.1-0.6b","bosun-v3.1-1.7b"]},{"method":"latency-v1 single process: author DecisionModel.from_pretrained(...).decide (bf16, FLA gated-delta kernels with sm_120 autotune instead of the gfx942 l2norm profile, questions batched in-row), venv-decision torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2; same as board","engines":["decision-nox-0bb83350","decision-sol-0665a411"]},{"method":"latency-v1 single process: native Kev serving at the board checkpoint (raw probabilities, no temperature, as on the board): merged bf16 backbone, fp32 pointer head, sdpa, native prefix cache 4, FLA kernels; venv-kev torch 2.8.0+cu128. A first run on the newer checkpoint (kev-models.json head) was set aside.","engines":["kev-4b-raised","kev-9b-raised"]},{"method":"latency-v1 single process: author model.decide, all questions in one forward, bf16, sdpa backbone (outer attention eager as shipped), torch 2.14.0+cu130","engines":["lumma-fev-0.1b","lumma-fev-0.6b"]},{"method":"latency-v1 single process: in-process vLLM 0.29.0 LLM.generate with all questions of the row in one call, CUDA graphs, gpu util 0.9 (single process)","engines":["tev1-0.8b","tev1-4b"]},{"method":"latency-v1 single process: author infer.score, fp32 (the only precision infer.load_trained_model offers), one padded batch of all options per question, SDPA default; max_len 32768 lift as board; OOM retry sub-batches of 8","engines":["akash-gemma"]},{"method":"latency-v1 single process: author DecisionModel forward (bf16, SDPA, one pass per question over the shared state) on one card; latency definition: In-process wall time of the DecisionModel forward: one pass per question over the shared state, CUDA-synchronized.","engines":["pplx-decider-v1-27b"]},{"method":"latency-v1 single process: clm.Engine.answer against the owned vLLM pooling server with CUDA graphs (JEV_CLM_GRAPHS=1; the author script uses --enforce-eager: eager measured 58.5 / 133.2 ms vs graphs 46.8 / 130.9), prefix caching on, gpu util 0.85; eager run kept as latency-v1-full.eager","engines":["clm-v0.1-8b"]},{"method":"latency-v1 single process: author decider.serve through decider_memory_app (DECIDER_COMPILE=1, DECIDER_FP8=1, CUDA graphs, DECIDER_MAX_FWD_TOKENS 65536), same server as board; one sequential HTTP client (run_decider_http_suite.py --concurrency 1) instead of concurrency 8","engines":["decider-2b-fp8-http"]},{"method":"latency-v1 single process: in-process vLLM 0.29.0 LLM, NVFP4 MoE, CUDA graphs, prefix caching off (board config; a prefix-cache run was slower and flipped answers), gpu util 0.9, venv-vllm torch 2.13.0+cu130","engines":["decider-35b-nvfp4"]},{"method":"latency-v1 single process: author decider.serve v1.2.1 with DECIDER_FP8=1 and DECIDER_COMPILE=1 on top of CUDA graphs (89 captured), one sequential HTTP client; the board ran graphs only. FP8+compile: 12.6 / 188.5 ms; graphs only: 13.6 / 172.7 ms (kept as latency-v1-full.stock). Chosen on the median","engines":["decider-4b"]},{"method":"latency-v1 single process: decider2.serve_chat_adaptive over serve_chat_lowmem as board (graph server serve_chat OOMs on 96 GB); one sequential HTTP client; card venv torch 2.14/transformers 5.17 instead of the job's pytorch 2.10 image","engines":["decider-chat-gemma4-31b"]},{"method":"latency-v1 single process: author decider.serve v1.2.1 chat layout (bf16, graphs), raised admission limits as board; one sequential HTTP client (engine-launched server, not attached)","engines":["decider-chat-qwen3.6-27b"]},{"method":"latency-v1 single process: author DecisionModel.decide (bf16, FLA gated-delta kernels, questions of a row batched at physical batch 8), venv-decision torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2; same as board (no causal-conv1d CUDA kernel, as on the board)","engines":["decision-eos-0.8b"]},{"method":"latency-v1 single process: author encoder DecisionModel (Decision 1.0 decision-kai), predict_1k cap lifted (uncapped engine), venv-decision-legacy torch 2.13.0+cu130 / transformers 4.57.6","engines":["decision-kai-7185f514"]},{"method":"latency-v1 single process: author encoder DecisionModel (Decision 1.0 decision-lex), predict_1k cap lifted (uncapped engine), venv-decision-legacy torch 2.13.0+cu130 / transformers 4.57.6","engines":["decision-lex"]},{"method":"latency-v1 single process: author DecisionModel.decide (bf16, FLA gated-delta kernels with sm_120 autotune, questions batched in-row), input limit raised to the backbone window as on the board, venv-decision torch 2.13.0+cu130","engines":["decision-lux-9b"]},{"method":"latency-v1 single process: board DjevLong: djev in-process adapter over pinned vLLM 0.29.1rc1.dev347, prefix caching, max-model-len 262144","engines":["djev"]},{"method":"latency-v1 single process: author AutoExtractor.classify_text with TF32 matmuls on (JEV_DECIDE_TF32=1; 23.3 / 228.6 ms vs 24.0 / 251.3 ms in fp32, kept as latency-v1-full.fp32), encoder attention eager as shipped, venv-q torch 2.14.0+cu130 (+peft)","engines":["gliner-decide"]},{"method":"latency-v1 single process: author engine_torch (bf16, SDPA, shared prefix + batched candidate suffixes), fleet venv torch 2.14.0+cu130; same as board","engines":["harsha"]},{"method":"latency-v1 single process: hopper-decisions 1.1.1 at tag g-1.2.0 with the HopitAI/hopper-g adapter (tournament shortlist for choice questions over 26 options, package default), Decider(cuda_graphs=True) package opt-in CUDA graphs per length bucket, eager outside captured buckets; author pins torch 2.8.0+cu128 / transformers 5.17.0 / fla 0.5.2 / causal-conv1d 1.7.0; eager variant kept as latency-v1-full.eager","engines":["hopper-g-1.2"]},{"method":"latency-v1 single process: author jeff encoder with flash attention, long-context lift as board (compile off in the package); venv-jeff","engines":["jeff-uncapped"]},{"method":"latency-v1 single process: author JevOmni.predict per question, single process (no shared-card lock); torch 2.14.0+cu130 / transformers 5.17.0","engines":["jev-omni"]},{"method":"latency-v1 single process: author DecisionEngine over vLLM 0.29.0 OpenAI server, Qwen3.8-27B-FP8, fp8 KV, prefix caching, CUDA graphs, max-model-len 262144 (board lift), gpu util 0.9","engines":["jevfire-uncapped"]},{"method":"latency-v1 single process: JevK5 0.2.2 lifted (JEVK5_LIFT=1, knockout readout), CUDA graphs on (package default JevK5(graphs=True)), fla kernels; venv torch 2.14.0+cu130 / transformers 5.17.0 / fla 0.5.2","engines":["jevk5"]},{"method":"latency-v1 single process: author jobe.score per question with SDPA attention (published default eager; sdpa is faster: median 53.5 vs 55.9 ms, p95 2910 vs 3088 ms); JOBE_LIFT=1 board lifts; venv-q torch 2.14.0+cu130. The eager run is kept as latency-v1-full.eager","engines":["jobe"]},{"method":"latency-v1 single process: open-jev[inference] diffusiongemma engine, bf16, chunked-expert path as board; venv-joshua","engines":["joshua-diffusion-full"]},{"method":"latency-v1 single process: native Kev serving: merged bf16 backbone, fp32 pointer head, sdpa, native prefix cache 4, FLA kernels, head.pt T=2.406 applied as on the board; venv-kev torch 2.8.0+cu128","engines":["kev-0.8b-raised"]},{"method":"latency-v1 single process: laya Agent.accelerate(use_graphs=True): TileLang fused kernels + CUDA graphs per shape bucket, built with the CUDA 13.0 nvcc from pip (nvidia-cuda-nvcc 13.0) because the card image ships nvcc 12.3 without sm_120a; measured with a warm TileLang JIT cache (a first pass with a cold cache had 72 rows of ~5.3 s kernel compilation, kept as latency-v1-full.fast-coldjit); p95 still includes first-use CUDA graph capture per bucket. Stock path (19.8 / 51.2 ms) kept as latency-v1-full.stock","engines":["laya-1e28ac20"]},{"method":"latency-v1 single process: author pcd Engine.constrained, board guard lifts, fleet venv torch 2.14.0+cu130","engines":["lfm2600"]},{"method":"latency-v1 single process: author rlcd Engine with SDPA attention (board path), fleet venv torch 2.14.0+cu130","engines":["lfm350-sdpa"]},{"method":"latency-v1 single process: author serve.py /v1/systemone handler in-process (Flask test client), fla kernels; torch 2.14.0+cu130 / transformers 5.17.0 / fla 0.5.2","engines":["metask-jev-4b"]},{"method":"latency-v1 single process: author mini-jev engine in-process, bf16, fleet venv torch 2.14.0+cu130 (same as board)","engines":["mini-jev"]},{"method":"latency-v1 single process: mojev.serve.Engine.answer, one packed forward over the state and every question (eager, author Space setting); torch 2.14.0+cu130","engines":["mojev"]},{"method":"latency-v1 single process: ParallelScorer.score (one full-prompt forward per question), author pins torch 2.8.0+cu128 / transformers 5.17.0 / fla 0.5.2 / causal-conv1d 1.7.0, Triton autotune cache from the board HF Job","engines":["nimble-v2"]},{"method":"latency-v1 single process: author client over vLLM 0.29.0 OpenAI server, Qwen3-4B-Instruct-2507 bf16, prefix caching, CUDA graphs, gpu util 0.8","engines":["openvons"]},{"method":"latency-v1 single process: Space cached scorer (shared prefix, chunked branch forwards), v2b merged LoRA bf16, sdpa, uncapped guards as board; fleet venv torch 2.14.0+cu130 (causal-conv1d absent: torch conv fallback)","engines":["pngwn-space-uncapped-full"]},{"method":"latency-v1 single process: openjev 0.4 API over vLLM 0.29.1rc1.dev551 NVFP4, async scheduling, prefix caching, TRITON_ATTN, one read per question group, one sequential HTTP client","engines":["razorback-one-read-3e296f08"]},{"method":"latency-v1 single process: author reflex frontend (serving-source archive 1dfaa87f: d08a8c6 + admission/readout overlays, wide_choice.py) over SGLang 0.5.20 from the pinned image b27fce60, --permutations 2, pressure admission 6, published SGLang flags with the triton attention backend (fa3 is Hopper-only), one sequential client, one RTX PRO 6000 HF Job","engines":["reflex-27b-v2"]},{"method":"latency-v1 single process: surogate v1.5.3 release wheel (pinned rune-v3-audit setup: uv py3.12.14, torch 2.11.0+cu130, noble glibc shim) serving the rune-v3 bf16 safetensors, surogate serve --max-model-len 262144 --kv-capacity 262144 --max-num-seqs 8, POST /v1/decisions with the whole row in one request (shared state prefill), one sequential client via run.py (latency_run_extra.py wrapper, engine latency_rune.RuneV3Latency)","engines":["rune-26b-a4b-v3"]},{"method":"latency-v1 single process: author score_shared with shared prefix, bf16, causal-conv1d 1.7.0 CUDA kernel (JEV_SEMIF_REQUIRE_CUDA_CONV=1), board lifts; fleet venv torch 2.14.0+cu130","engines":["semif"]},{"method":"latency-v1 single process: Solomon v1.1 bf16 serving binding with the encoding-fix path (solomon-v11-bf16-empty-context engine), sdpa (flash-attn optional, absent), venv-solomon torch 2.13.0+cu130 / transformers 5.17.0 / fla 0.5.2 as the binding requires","engines":["solomon-v11-bf16-encoding"]},{"method":"latency-v1 single process: TypedDecider.decide one forward per row, bf16, fla chunk_gated_delta_rule + causal-conv1d 1.7.0 CUDA kernel (board used the torch conv fallback), sdpa full attention, torch 2.14.0+cu130","engines":["this-that-1.2"]},{"method":"latency-v1 single process: author DecisionEngine (ModernBERT 151M) evaluate, max_length 8192 lift as board; venv-vl torch 2.13.0+cu130 from the rerun freeze. Native abstentions are not timed (357 of 750 rows answered)","engines":["verdict-8af2496e"]},{"method":"latency-v1 single process: structured_server.py over vLLM PR 57250 nightly, prefix caching, default attention backend, one sequential HTTP client","engines":["vllm-pr57250"]},{"method":"latency-v1 single process: winnow-inference llama.cpp CUDA server (6c2b3c0 + 255-label lift, WINNOW_LABELS_LIFT=1), Winnow-12B Q8_0 GGUF fully on GPU, n_ctx 131072, batch 2048 / ubatch 1024, one slot, one sequential HTTP client; same build and settings as the board","engines":["winnow-12b-q8"]},{"method":"latency-v1 single process: winnow-inference llama.cpp CUDA server (77d1458 labels-lift build), Winnow-E4B Q8_0 GGUF fully on GPU, n_ctx 131072, batch 2048 / ubatch 1024, one server and one sequential HTTP client (board: 4 servers per card)","engines":["winnow-e4b-q8"]},{"method":"latency-v1 single process: author xor server.py (raised body caps) over SGLang image 6bcaa47d, TP=1 on one RTX PRO 6000 HF Job, OPENJEV_CACHE=0 as board, one sequential client","engines":["xor"]},{"method":"latency-v1 single process: author DecisionEngine.system_one single batched forward, bf16, LEV_LOGITS_KEEP=last (board setting), torch 2.14.0+cu130","engines":["lev"]},{"method":"latency-v1 single process: author hf-server/hf_server.py (bf16, fla kernels, examples_binary policy, max-model-len 65536, batch tokens 65536), one sequential client (JEV_ATTACH); 203 of 760 rows ran out of GPU memory at the server default batch and were answered in-process by the same hf_server service with max_batch_size 32 (simplejev_ladder.py, as on the board for heavy rows); causal-conv1d absent (torch conv fallback, as on the board)","engines":["simple-jev-qwen3.8-27b"]},{"method":"latency-v1 single process: author neohorse_decision 1.0.0 wheel in its own env (py3.12.10, torch 2.8.0+cu128, triton 3.7.1), board guard lifts, one process","engines":["neohorse-jev-4b"]},{"method":"latency-v1 single process: author julia runtime (supersonic-julia 0.1.0, FastEngine) on CUDA with its optional compile_model=True (torch.compile dynamic=True; recompiles on new shapes are inside the timed rows), BF16 autocast, board guard lifts, one process; torch 2.14.0+cu130, transformers 5.0.0. Other variants kept as latency-v1-full.: eager (median 12.4 / p95 35.0 ms), marker_only_head (12.3 / 39.5), compile+marker_only_head (5.9 / 14.6)","engines":["julia-1"]},{"method":"latency-v1 single process: author-submitted full run, not re-timed by us: kit 19ad28e runner (one process, rows strictly serial) with the PR #8 JebadiahEngine in-process on one RTX PRO 6000 (HF Job), bf16 SDPA + fla 0.5.2 Gated DeltaNet kernels, prefix_cache=true (shared system+state prefix once per row, question suffixes from cached KV/DeltaNet state), batch_tokens=32768, max_batch=64 (questions batched inside a row only); torch 2.8.0+cu128, transformers 5.17.0","engines":["jebadiah-27b"]},{"method":"latency-v1 single process: author-submitted LavoirEngine (di_lavoir_engine.py, native=true) in-process on one RTX PRO 6000 Space (rtx-13): ModernBERT-large encoder with SDPA attention (pinned by lavoir/model.py), bf16 autocast, a row's questions length-sorted and batched (64 questions / 65,536 padded tokens per forward); torch 2.14.0+cu130, transformers 5.17.0, laya 0.3.11, lavoir af79b0a","engines":["lavoir"]},{"method":"latency-v1 single process: llm2jev 0.6.1 (chat prompt, thinking off, T=1.140) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","engines":["jpt-0.8b"]},{"method":"latency-v1 single process: llm2jev 0.6.1 (chat prompt, thinking off, T=1.036) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","engines":["jpt-4b"]},{"method":"latency-v1 single process: llm2jev 0.6.1 (chat prompt, thinking off, T=1.087) over SGLang 0.5.9 with the card flags (--context-length 32768 --mamba-scheduler-strategy extra_buffer; radix cache and CUDA graphs on) and --attention-backend triton (required by SGLang 0.5.9 for Qwen3.5 hybrid GDN on Blackwell); all questions of a row in one /generate call; one sequential client; one RTX PRO 6000 Space; venv torch 2.9.1+cu128, transformers 4.57.1, nvidia-cudnn-cu12 9.16.0.29 (SGLang start-up requirement)","engines":["jpt-9b"]},{"method":"latency-v1 single process: model repo serve.py 1.2 (caiovicentino1/Eikos-27B-FP8@e40a92f, letter readout, calib.json T=1, max_one_pass 160) over vLLM 0.30.0 with the submission's launch flags (serve_vllm.sh flags, --max-model-len 65536, --enable-prefix-caching --mamba-cache-mode align, --max-num-seqs 64, gpu util 0.85); one sequential client; one RTX PRO 6000 Space; venv torch 2.13.0+cu130, transformers 5.17.0 (fleet/eikos-27b-fp8-freeze.txt)","engines":["eikos-27b-fp8"]},{"method":"latency-v1 single process: kit PR #20 engine submissions/jet/engine.py @21092e8 (sha256 357454d0) unchanged over michaljach/jet@fbc3d2d format.py/runtime.py: BF16, SDPA, FLA chunk gated-delta kernel, PyTorch causal-conv fallback as the runtime specifies, one forward per question, no cache; the only path the published runtime has; venv torch 2.11.0+cu128, transformers 5.17.0, fla 0.5.2, triton 3.6.0 (the model repo requirements.txt); fresh process, cold autotune cache; warm-cache rerun kept as latency-v1-full.rerun (47.1 / 319.0 ms, bit-identical answers)","engines":["jet-v6.2"]}]},"jev":{"source":"artifacts/benchmark-suite/jev-release-v1/results.jsonl","ok_rows":132422,"total_wall_ms":{"median":949.5,"mean":1335.2,"p95":3743.3,"max":151038.9,"n":132422},"http_wall_ms":{"median":252.8,"mean":276.1,"p95":436.6,"max":5434.6,"n":132422},"note":"Remote hosted API measured over HTTP. total_wall_ms includes client retries and network; http_wall_ms is the request itself. Local GPU reproductions have no network leg, so these are not a controlled speed comparison.","summary":"Jev is TypeSafe's hosted API: a network round-trip from our lab with one sequential HTTPS client on the same latency sample, not comparable to on-card latency. Its answers on this sample drifted from the Sep-19 board rows (20 choice flips); Jev's scores are unchanged."},"comparability":"The leaderboard plots the reproductions' single-process time against Jev's HTTPS round trip on one axis for orientation only. The two are not a controlled speed comparison: Jev's number includes the network and shared service scheduling, the reproductions' numbers include none of that."},"answer_gaps":[{"engine":"verdict-8af2496e","name":"Verdict","variant":null,"total":120340,"unanswered":71101,"share":0.5908,"reasons":[{"reason":"chose its own abstain option instead of an answer","count":56045,"share":0.4657},{"reason":"supports only 2–24 answer options per question","count":14733,"share":0.1224},{"reason":"prompt longer than its 8,192-token context window","count":323,"share":0.0027}]},{"engine":"solomon-v11-bf16-encoding","name":"Solomon v1.1","variant":null,"total":120328,"unanswered":27498,"share":0.2285,"reasons":[{"reason":"supports only 2–8 answer options per question","count":27080,"share":0.2251},{"reason":"request shape outside the model's schema","count":415,"share":0.0034},{"reason":"request larger than its native request budget","count":3,"share":2.49e-05}]},{"engine":"semif","name":"SemIf","variant":null,"total":120340,"unanswered":15317,"share":0.1273,"reasons":[{"reason":"supports only 2–16 answer options per question","count":15317,"share":0.1273}]},{"engine":"tev1-0.8b","name":"Tev1-0.8B-experimental","variant":null,"total":119888,"unanswered":14732,"share":0.1229,"reasons":[{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 151","count":5500,"share":0.0459},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 77","count":3080,"share":0.0257},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'chord' has 129","count":2000,"share":0.0167},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'tool' has 53","count":508,"share":0.0042}]},{"engine":"tev1-4b","name":"Tev1-4B-experimental","variant":null,"total":119898,"unanswered":14732,"share":0.1229,"reasons":[{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 151","count":5500,"share":0.0459},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'q1' has 77","count":3080,"share":0.0257},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'chord' has 129","count":2000,"share":0.0167},{"reason":"Tev1 answers with one of the labels A-X (2-24 options); question 'tool' has 53","count":508,"share":0.0042}]},{"engine":"xor","name":"Xor","variant":null,"total":119898,"unanswered":14500,"share":0.1209,"reasons":[{"reason":"xor server 422: q1: 2..26 options, got 151","count":5500,"share":0.0459},{"reason":"xor server 422: q1: 2..26 options, got 77","count":3080,"share":0.0257},{"reason":"xor server 422: chord: 2..26 options, got 129","count":2000,"share":0.0167},{"reason":"xor server 422: tool: 2..26 options, got 53","count":508,"share":0.0042}]},{"engine":"vllm-pr57250","name":"mmastrac diffusiongemma (vLLM PR 57250)","variant":null,"total":120340,"unanswered":14522,"share":0.1207,"reasons":[{"reason":"supports at most 26 answer options per question","count":14500,"share":0.1205},{"reason":"structured_server: question 'q_78d37c943d': label 'C' is not a single token","count":1,"share":8.31e-06},{"reason":"structured_server: question 'q_c3e60b168d': label 'C' is not a single token","count":1,"share":8.31e-06},{"reason":"structured_server: question 'q_dbd1ddcc9b': label 'D' is not a single token","count":1,"share":8.31e-06}]},{"engine":"mini-jev","name":"mini-jev","variant":null,"total":120341,"unanswered":14501,"share":0.1205,"reasons":[{"reason":"supports only 2–26 answer options per question","count":14500,"share":0.1205},{"reason":"ran out of GPU memory on the longest prompts","count":1,"share":8.31e-06}]},{"engine":"metask-jev-4b","name":"Metask-Jev-4B","variant":null,"total":120340,"unanswered":14500,"share":0.1205,"reasons":[{"reason":"metask serve.py HTTP 500: decision: enum choices must be 1–26 nonempty strings.","count":14500,"share":0.1205}]},{"engine":"lumma-fev-0.1b","name":"Lumma-Fev-0.1B","variant":null,"total":119888,"unanswered":2093,"share":0.0175,"reasons":[{"reason":"question longer than its 2,048-token context window","count":2093,"share":0.0175}]},{"engine":"djev","name":"djev","variant":null,"total":120340,"unanswered":1165,"share":0.0097,"reasons":[{"reason":"more questions per request than its answer canvas holds","count":1165,"share":0.0097}]},{"engine":"lfm2600","name":"LFM2.5-2.6B-RLCD","variant":null,"total":120340,"unanswered":533,"share":0.0044,"reasons":[{"reason":"request exceeds conservative GPU memory budget; reduce prompt or branch batch size","count":533,"share":0.0044}]},{"engine":"harsha","name":"Qwen-2.5-1B-RLCD","variant":null,"total":120340,"unanswered":380,"share":0.0032,"reasons":[{"reason":"prompt longer than its context window (no truncation allowed)","count":380,"share":0.0032}]},{"engine":"decision-lex","name":"Decision 1.0 Lex","variant":null,"total":120340,"unanswered":326,"share":0.0027,"reasons":[{"reason":"ran out of GPU memory on the longest prompts","count":319,"share":0.0027},{"reason":"prompt longer than its 8,192-token context window","count":7,"share":5.82e-05}]},{"engine":"lavoir","name":"Lavoir","variant":null,"total":120340,"unanswered":324,"share":0.0027,"reasons":[{"reason":"instructions longer than its native budget (no truncation allowed)","count":318,"share":0.0026},{"reason":"prompt longer than its context window (no truncation allowed)","count":6,"share":4.99e-05}]},{"engine":"laya-1e28ac20","name":"Laya","variant":null,"total":120226,"unanswered":320,"share":0.0027,"reasons":[{"reason":"instructions longer than its native budget (no truncation allowed)","count":314,"share":0.0026},{"reason":"prompt longer than its context window (no truncation allowed)","count":5,"share":4.16e-05},{"reason":"answer options longer than its per-option budget (no truncation allowed)","count":1,"share":8.32e-06}]},{"engine":"decision-kai-7185f514","name":"Decision 1.0 Kai","variant":null,"total":120237,"unanswered":319,"share":0.0027,"reasons":[{"reason":"ran out of GPU memory on the longest prompts","count":315,"share":0.0026},{"reason":"prompt longer than its 8,192-token context window","count":4,"share":3.33e-05}]},{"engine":"jet-v6.2","name":"Jet v6.2","variant":null,"total":120340,"unanswered":243,"share":0.002,"reasons":[{"reason":"Complete prompt has 26996 tokens; declared limit 16384","count":7,"share":5.82e-05},{"reason":"Complete prompt has 26999 tokens; declared limit 16384","count":6,"share":4.99e-05},{"reason":"Complete prompt has 27017 tokens; declared limit 16384","count":5,"share":4.15e-05},{"reason":"Complete prompt has 27003 tokens; declared limit 16384","count":5,"share":4.15e-05}]},{"engine":"julia-1","name":"Julia 1","variant":null,"total":119888,"unanswered":10,"share":8.34e-05,"reasons":[{"reason":"prompt longer than its 8,192-token context window (no truncation allowed)","count":10,"share":8.34e-05}]},{"engine":"gliner-base","name":"GLiNER 2.5 base","variant":null,"total":120340,"unanswered":9,"share":7.48e-05,"reasons":[{"reason":"prompt text with parentheses, which the native GLiNER prompt syntax cannot carry","count":9,"share":7.48e-05}]},{"engine":"gliner-multi","name":"GLiNER 2.5 multilingual","variant":null,"total":120340,"unanswered":9,"share":7.48e-05,"reasons":[{"reason":"prompt text with parentheses, which the native GLiNER prompt syntax cannot carry","count":9,"share":7.48e-05}]},{"engine":"gliner-small","name":"GLiNER 2.5 small","variant":null,"total":120340,"unanswered":9,"share":7.48e-05,"reasons":[{"reason":"prompt text with parentheses, which the native GLiNER prompt syntax cannot carry","count":9,"share":7.48e-05}]},{"engine":"lumma-fev-0.6b","name":"Lumma-Fev-0.6B","variant":null,"total":119888,"unanswered":4,"share":3.34e-05,"reasons":[{"reason":"question longer than its 12,288-token context window","count":4,"share":3.34e-05}]},{"engine":"decider-2b-fp8-http","name":"Decider 2B","variant":"FP8","total":120344,"unanswered":4,"share":3.32e-05,"reasons":[{"reason":"runtime error","count":4,"share":3.32e-05}]},{"engine":"decision-eos-0.8b","name":"Decision 1.0 Eos","variant":null,"total":119898,"unanswered":2,"share":1.67e-05,"reasons":[{"reason":"prompt longer than its 16,384-token context window","count":2,"share":1.67e-05}]},{"engine":"rune-26b-a4b-v3","name":"Surogate Rune 26B-A4B v3","variant":"bf16","total":119898,"unanswered":1,"share":8.34e-06,"reasons":[{"reason":"runtime error","count":1,"share":8.34e-06}]},{"engine":"akash-gemma","name":"system-one-gemma","variant":null,"total":120237,"unanswered":0,"share":0,"reasons":[]},{"engine":"autojev-27b","name":"pplx-decider-v1-27b","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"bosun-v3.1-0.6b","name":"Bosun v3.1 0.6B","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"bosun-v3.1-1.7b","name":"Bosun v3.1 1.7B","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"clm-v0.1-8b","name":"CLM-v0.1-8B","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"decider-35b-nvfp4","name":"Decider 35B-A3B","variant":"NVFP4","total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"decider-4b","name":"Decider 4B","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"decider-chat-gemma4-31b","name":"Decider chat · Gemma-4-31B","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"decider-chat-qwen3.6-27b","name":"Decider chat · Qwen3.6-27B","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"decision-lux-9b","name":"Decision 1.0 Lux","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"decision-nox-0bb83350","name":"Decision 1.0 Nox","variant":null,"total":120171,"unanswered":0,"share":0,"reasons":[]},{"engine":"decision-sol-0665a411","name":"Decision 1.0 Sol","variant":null,"total":120226,"unanswered":0,"share":0,"reasons":[]},{"engine":"gliner-decide","name":"GLiNER2.5-Decide","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"hopper-g-1.2","name":"Hopper (G) 1.2","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"jeff-uncapped","name":"jeff","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"jev-omni","name":"Jev-Omni","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"jevfire-uncapped","name":"Jevfire","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"jevk5","name":"JevK5","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"jobe","name":"Jobe","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"joshua-diffusion-full","name":"open-jev (JoshuaSP)","variant":null,"total":120025,"unanswered":0,"share":0,"reasons":[]},{"engine":"kev-0.8b-raised","name":"Kev 0.8B","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"kev-4b-raised","name":"Kev 4B","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"kev-9b-raised","name":"Kev 9B","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"lfm350-sdpa","name":"LFM2.5-350M-RLCD","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"mojev","name":"MoJev","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"nimble-v2","name":"Bespoke Nimble 9B v2","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"openvons","name":"openvons","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"pngwn-space-uncapped-full","name":"open-jev (pngwn)","variant":null,"total":120139,"unanswered":0,"share":0,"reasons":[]},{"engine":"razorback-one-read-3e296f08","name":"openjev (razorback16)","variant":"one read, NVFP4","total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"reflex-27b-v2","name":"reflex 27B","variant":"FP8 · wide choice","total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"this-that-1.2","name":"this-that 1.2","variant":null,"total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"winnow-12b-q8","name":"Winnow-12B","variant":"Q8_0","total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"winnow-e4b-q8","name":"Winnow-E4B","variant":"Q8_0","total":119898,"unanswered":0,"share":0,"reasons":[]},{"engine":"lev","name":"lev","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"intern-decision-0.8b","name":"Intern-Decision-0.8B","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"intern-decision-2b","name":"Intern-Decision-2B","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"intern-decision-4b","name":"Intern-Decision-4B","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"simple-jev-qwen3.8-27b","name":"simple-jev · Qwen3.8-27B (featherless)","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"neohorse-jev-4b","name":"NeoHorse-Jev-4B","variant":null,"total":119888,"unanswered":0,"share":0,"reasons":[]},{"engine":"jebadiah-27b","name":"Jebadiah 27B","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"jpt-0.8b","name":"JPT-0.8B","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"jpt-4b","name":"JPT-4B","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"jpt-9b","name":"JPT-9B","variant":null,"total":120340,"unanswered":0,"share":0,"reasons":[]},{"engine":"eikos-27b-fp8","name":"Eikos-27B","variant":"FP8","total":120340,"unanswered":0,"share":0,"reasons":[]}],"changelog":[{"edition":"v0.4.1","date":"2026-10-21","notes":["We expanded and refreshed the private new-domain tests to cover a wider range of real decisions and measure them more accurately. The added tasks are harder, so new-domain scores sit lower than before."]},{"edition":"v0.4","date":"2026-10-19","notes":["Full score = 0.60 × private tests of the same skills + 0.40 × private tasks from new domains, both mapped onto the public scale using untuned stock models.","Public benchmarks are still run and shown next to every model, but they no longer count toward the Full score: the ranking rests entirely on tests nobody can train on.","The new-domain part uses a revised set of private tasks, kept to the ones that rank untuned stock models the same way the rest of the index does.","Ties are unchanged: models within 0.9 points of the first model in their group share a rank."]},{"edition":"v0.3.2","date":"2026-10-12","notes":["Scoring bug fix: three same-skill sets (the GPQA, MMLU-Pro and BBH skill sets) were left out of the same-skill score for 37 models scored since the 0.3 launch. Every model now is assessed against all \"same skill\" sets","New chart: public vs private score, model by model, under How the Full score works."]},{"edition":"v0.3.1","date":"2026-10-09","notes":["Vision board: every model now reads images at up to 1.6 MP (1,638,400 pixels). Models whose own image path shrank images further (to 262k pixels, or 280 image tokens) were re-run with that cap lifted; Gemma-4 models use 560 image tokens (about 1.3 MP), the nearest setting below 1.6 MP. Engines that cannot be set this way keep their own budget, shown per model.","The text board is unchanged."]},{"edition":"v0.3","date":"2026-10-06","notes":["Full score = 0.20 × public + 0.50 × private tests of the same skills + 0.30 × private tasks from new domains. The private parts are mapped onto the public scale using untuned stock models.","Most of the score comes from tests nobody can train on, so a public-only advantage counts for little.","Models within 0.9 points of the first model in their group share a rank: a gap that small could come from which questions happened to be sampled. Tied models share a rank; the next group takes the next number, so ranks count groups, not models.","Private tests of equivalent competence for most public tasks, built from fresh data and kept private, plus a set of private decision tasks from new domains.","Public index rebuilt: GSM8K rebuilt, ForecastBench retired, WinoGrande moved to Knowledge & Reasoning.","Vision board added: 0.5 × public vision benchmarks + 0.5 × private tests of the same skills.","Superseded versions leave the board when a newer version of the same model is scored: pplx-decider v1 → v1.1, RSI-Jev v4.0 → v5.0, EXAONE-4.0-1.2B-JEV v0.1 and v0.2 → v0.3, among others."]},{"edition":"v0.2.1","date":"2026-09-28","notes":["New area weights: Arts & Human Taste is fixed at 10%; the other four areas share the rest in proportion to the square root of their benchmark count: Tools & Automation 18.3%, Retrieval & Classification 20.0%, Knowledge & Reasoning 25.8%, Language Understanding 25.8%.","Gold ★ benchmarks weigh 1.2 inside their area, the rest 1.0: MMLU-Pro, BBH, GPQA Diamond, HLE, ANLI, WinoGrande, HellaSwag, BANKING77, CLINC150, BRIGHT, BFCL, API-Bank, ForecastBench.","SGD leaves the index until it is re-run with a fixed builder, and RouterBench leaves because its prompt gives away the best route. Both stay on the board.","ACOS is now scored with F1 per review instead of all-or-nothing, and stays in the index.","ToolRet and BRIGHT count only queries with at least one relevant candidate among the 32 (685 of 1,000 and 220 of 550); chance is recomputed on the same queries.","RAGTruth chance is now always answering \"hallucinated\" (F1 0.518) instead of a fair coin (0.411).","Home appliances drops 24 duplicate test rows and 48 rows identical to published dev rows (88 of 160 kept).","Surogate Rune 26B-A4B v3 replaces v1 on the board: the author's submitted run, adopted after our 3,000-row reproduction matched it exactly.","reflex 27B's full-coverage rerun replaces its 26-option run on the board: the author's submitted run with wide-choice labels, adopted after our single-card latency run agreed with it within reflex's own run-to-run noise.","Hopper (G) 1.2 replaces Hopper 1.1.1 on the board: the author's submitted run of the next Hopper adapter (same base and serving code), adopted after our single-card latency run agreed with it within the A10G-vs-RTX noise."]},{"edition":"v0.2","date":"2026-09-27","notes":["Scores are now chance-corrected: 0 means random guessing and 100 means perfect, so yes/no questions no longer give free points.","30 new models on the board, plus newer versions of 14 existing ones.","The index now counts 40 benchmarks, up from 19: 7 brand-new ones (MMLU-Pro, BBH, When2Call, RAGTruth, HoVer, New Yorker caption matching, PhishNChips), plus 15 we already ran that now count, such as HLE, ANLI, WinoGrande, HellaSwag, BANKING77 and CLINC150.","MMLU left the index: top models have nearly maxed it out, and MMLU-Pro takes its place. ARC-Easy and ARC-Challenge are still shown but don't count.","ToolRet and BRIGHT are run on a subset of their data that doesn't skew results but is faster and cheaper to run."]},{"edition":"v0.1","date":"2026-09-22","notes":["First public edition: 31 open reproductions plus Jev jev-1.13.0 on the frozen 132,422-request suite.","442 defective or out-of-reach questions excluded for every engine alike; 131,980 scoreable rows.","Panel: core25 with the six interactive environments left out (19 static benchmarks, five equal-weight areas); ChessBench folded into Knowledge & Reasoning; Language & Retrieval split into Language Understanding and Retrieval & Classification.","iSarcasmEval headline is track A English sarcasm F1; other tracks listed under each result.","Only the balanced raw index is shown; chance-normalized and breadth variants are computed but hidden.","Point estimates only; paired cluster-bootstrap intervals not yet calculated."]}],"added_benchmarks":[{"id":56,"name":"PhishNChips","dataset":"PhishNChips phishing decisions","area":"retrieval","requests":2000,"metric":"accuracy","chance":0.5,"chance_note":"uniform random choice, mean of 1/options","source":null,"suite":"calibration-extension-v1","framing":"Says whether an email and its link are phishing or legitimate.","licence":"see calibration-extension-v1 README","jev":0.8375,"entrants_scored":70,"references":[{"kind":"dataset","url":"https://huggingface.co/datasets/AreLit/PhishNChips"},{"kind":"code","url":"https://github.com/anisselbd/jev-phishing-bench"}],"explainer":"Says whether an email and its link are phishing or legitimate.","in_panel":true},{"id":57,"name":"MMLU-Pro","dataset":"MMLU-Pro","area":"knowledge","requests":12032,"metric":"accuracy","chance":0.1109,"chance_note":"uniform random choice, mean of 1/options","source":{"huggingface":{"repo":"TIGER-Lab/MMLU-Pro","revision":"b189ec765aa7ed75c8acfea42df31fdae71f97be"}},"suite":"candidates-v3/batch-1","framing":"state = question; instructions = `CHOICE`; criteria = letters `A`.. mapped to the published options in order (3–10 options; 9,981 items have 10). Gold = published `answer`. One item has duplicate option text at the source and is kept as published.","licence":"MIT","jev":0.8213,"entrants_scored":70,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2406.01574"},{"kind":"dataset","url":"https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro"}],"explainer":"Harder MMLU successor: questions across 14 subjects with up to ten options.","in_panel":true},{"id":58,"name":"BBH","dataset":"BBH fixed-option tasks","area":"knowledge","requests":5507,"metric":"accuracy","chance":0.3101,"chance_note":"uniform random choice, mean of 1/options","source":{"git":{"url":"https://github.com/suzgunmirac/BIG-Bench-Hard","commit":"9ee07bd481feebf959a6b59d61ea57bdcf30964d"}},"suite":"candidates-v3/batch-1","framing":"The 27 BBH task files minus the four with free-form answers (`dyck_languages`, `multistep_arithmetic_two`, `object_counting`, `word_sorting`) leaves 23 files = 19 task families (the three `logical_deduction_*` and three `tracking_shuffled_objects_*` files are one family each). state = the task input verbatim, including its own `Options:` block; instructions = `CHOICE`. Criteria: the `(A) text` lines parsed from that block (key `(A)`, the published target form), or the `- Yes / - No`, `- valid / - invalid` lines, or for the three tasks that list no options the task's fixed answer set (`boolean_expressions` True/False, `sports_understanding` yes/no, `web_of_lies` Yes/No). No option is invented. Four source items are malformed and excluded before any call, so 5,507 of 5,511 remain: `movie_recommendation:163` (target \"Monsters, Inc\" was split across options A and B), `ruin_names:99` and `ruin_names:144` (targets containing commas were split into several options, so the gold is not an option), `snarks:88` (input truncated to a single option).","licence":"MIT (repository LICENSE); carries the BIG-bench canary, never place in training corpora","jev":0.9319,"entrants_scored":70,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2210.09261"},{"kind":"dataset","url":"https://github.com/suzgunmirac/BIG-Bench-Hard"}],"explainer":"23 tricky reasoning tasks from BIG-Bench (logic, dates, tracking objects, fallacies), answered with no working.","in_panel":true},{"id":59,"name":"RAGTruth","dataset":"RAGTruth response-level hallucination","area":"language","requests":2700,"metric":"F1 on hallucinated class","chance":0.5177,"chance_note":"Always \"hallucinated\" · F1","source":{"git":{"url":"https://github.com/ParticleMedia/RAGTruth","commit":"c103204b9ce28d6bbad859304bf30de72b8ed8fe"}},"suite":"candidates-v3/batch-2","framing":"All 2,700 test responses (900 each of QA, Summary, Data2txt; six responder models). state = `{prompt, response}`, where `prompt` is the exact generation prompt from `source_info.jsonl` (it carries the passages, article or structured data) and `response` is the evaluated response. Question is a `noul`: instructions `The response contains content that is not supported by the context in the prompt.`; criteria true `The response contains content that is not supported by the context in the prompt.`, false `All content of the response is supported by the context in the prompt.` Gold = hallucinated iff the response has at least one annotated span (`len(labels) > 0`, exactly the RAGTruth baseline's `is_halu`); 943 positives. Spans flagged `implicit_true` or `due_to_null` count, as in the baseline. Responses marked `incorrect_refusal`/`truncated` are kept. Headline metric: F1 on the hallucinated class (prediction = P(yes) ≥ 0.5), plus accuracy.","licence":"MIT annotations and code; contexts include CNN/Daily Mail, MS MARCO (non-commercial) and the Yelp Open Dataset (redistribution forbidden): built locally, rows not republished","jev":0.7512,"entrants_scored":70,"references":[{"kind":"dataset","url":"https://github.com/ParticleMedia/RAGTruth"}],"explainer":"Says yes or no: does an AI-written answer contain anything its source text doesn't support? Scored with F1.","in_panel":true},{"id":61,"name":"HoVer","dataset":"HoVer claim verification","area":"retrieval","requests":4000,"metric":"accuracy","chance":0.5,"chance_note":"uniform random choice, mean of 1/options","source":{"git":{"url":"https://github.com/hover-nlp/hover","commit":"39b84697f196308f398a251a7aea9b82ae0f0562"},"hover_wiki_db":{"url":"https://nlp.cs.unc.edu/data/hover/wiki_wo_links.db","sha256":"c37ee397916ec0bffacfe8902db454a5cda88a7a188409217b2e15231fe5ee2f"}},"suite":"candidates-v3/batch-3","framing":"All 4,000 dev claims (2,000 SUPPORTED, 2,000 NOT_SUPPORTED). The dataset provides evidence as `supporting_facts` = (Wikipedia title, sentence id) pairs into the HoVer authors' own Wikipedia introduction database (`wiki_wo_links.db`, distributed by the HoVer authors and pinned by sha256). Sentence ids refer to a CoreNLP split that is not distributed, so rather than re-splitting, state carries the **complete introduction of every document named in `supporting_facts`**: `{claim, evidence: [{title, text}]}`, documents in first-mention order. Only dataset-provided text is used; no retrieval, no external text, and NOT_SUPPORTED claims get their own annotated documents just like SUPPORTED ones. This is the oracle-document setting, easier than HoVer's open retrieval setting. instructions `Is the claim supported by the evidence?`; criteria `SUPPORTED` → `The evidence supports the claim.`, `NOT_SUPPORTED` → `The evidence does not support the claim.` (the original label strings as keys).","licence":"CC BY-SA 4.0 data, MIT code, Wikipedia text CC BY-SA: share-alike applies to republished rows","jev":0.7675,"entrants_scored":70,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2011.03088"},{"kind":"dataset","url":"https://github.com/hover-nlp/hover"}],"explainer":"Decides whether a claim linking two to four Wikipedia articles is supported by their introductions.","in_panel":true},{"id":62,"name":"When2Call","dataset":"When2Call MCQ","area":"tools","requests":3652,"metric":"accuracy","chance":0.25,"chance_note":"uniform random choice, mean of 1/options","source":{"huggingface":{"repo":"nvidia/When2Call","revision":"0582f7749df63a96fdc3070932e83e72396ace53"}},"suite":"candidates-v3/batch-1","framing":"All 3,652 `mcq` test items. state = `{tools, question}` with `tools` = the item's `tools` field parsed from JSON strings (not `orig_tools`; 258 items have no tools). instructions `Which response should the assistant give to the user's question, given the available tools?`; criteria `A`..`D` = the four published answers in published order (direct, tool_call, request_for_info, cannot_answer). Category names are not on the wire. Gold is never `A` (direct) by the dataset's construction. Reported: accuracy, macro-F1 over the three gold categories, and tool-call rate when no tools are supplied.","licence":"CC BY 4.0","jev":0.8165,"entrants_scored":70,"references":[{"kind":"paper","url":"https://aclanthology.org/2025.naacl-long.174"},{"kind":"dataset","url":"https://huggingface.co/datasets/nvidia/When2Call"}],"explainer":"Given a question and an assistant's tools, picks the right move: answer, call a tool, ask for details, or decline.","in_panel":true},{"id":64,"name":"New Yorker","dataset":"New Yorker caption matching","area":"arts","requests":528,"metric":"accuracy","chance":0.2,"chance_note":"uniform random choice, mean of 1/options","source":{"huggingface":{"repo":"jmhessel/newyorker_caption_contest","revision":"d81cbab7d0392708d5371d3a4960e69261824db4"}},"suite":"candidates-v3/batch-1","framing":"`matching` config (fold 0) test, 528 items. Text only, no pixels: state = `{scene, description, uncanny_description, entities}` = the dataset's `image_location`, `image_description`, `image_uncanny_description`, `entities` (Wikipedia URLs as published). instructions `Which caption was written for this cartoon?`; criteria `A`..`E` = `caption_choices` in order.","licence":"CC BY 4.0 annotations; cartoons not used, text descriptions only","jev":0.7254,"entrants_scored":70,"references":[{"kind":"paper","url":"https://arxiv.org/abs/2209.06293"},{"kind":"dataset","url":"https://huggingface.co/datasets/jmhessel/newyorker_caption_contest"}],"explainer":"Given a text description of a New Yorker cartoon, picks the caption written for it from five.","in_panel":true}],"reproduce":{"github":"https://github.com/apolinario/decision-index","lab_scripts":["evaluation/reproductions/run.py","evaluation/reproductions/report_model.py","evaluation/reproductions/answer_gaps.py","evaluation/reproductions/compute_indices.py","evaluation/reproductions/entrant_metadata.py","evaluation/reproductions/build_leaderboard.py","evaluation/reproductions/build_methodology.py"],"outputs":["data/index-v0.2.1.json","data/methodology-v0.2.1.json"]},"notes":{"lifts":[{"engine":"winnow-12b-q8","name":"Winnow-12B","lift_class":"labels-255","rows_answered_by_lift":10582,"line":"Winnow-12B: answer-label stop in the author's native engine (native/engine.h) raised from 64 to 255 labels; same label loop and single-token filter, first 64 labels unchanged; 10,582 previously refused rows answered by the lift."},{"engine":"semif","name":"SemIf","lift_class":"direct-fallback-max-tokens","rows_answered_by_lift":2855,"line":"SemIf: max_tokens raised from the 4,096 keyword default to the 262,144-token Qwen3.5-4B window; rows that fail the shared-prefix precondition or run out of memory in the shared batch take the author's direct path; 2,855 previously refused rows answered by the lift."},{"engine":"lfm2600","name":"LFM2.5-2.6B-RLCD","lift_class":"pcd-limits","rows_answered_by_lift":11910,"line":"LFM2.5-2.6B-RLCD: pcd Limits request budgets raised (prompt 122,880 plus value 8,192 tokens inside the 131,072-token window; field, candidate and character budgets opened); batch sizes unchanged; 11,910 previously refused rows answered by the lift."},{"engine":"pngwn-space-uncapped-full","name":"open-jev (pngwn)","lift_class":"stale-question-limit","rows_answered_by_lift":26,"line":"open-jev (pngwn): rows refused under a question limit the uncapped path no longer applies were re-answered, and rows missing from the release run were answered; 26 previously refused rows answered by the lift."},{"engine":"jevfire-uncapped","name":"Jevfire","lift_class":"context-262144","rows_answered_by_lift":292,"line":"Jevfire: context window raised from 16,384 to the 262,144-token Qwen3.8-27B-FP8 window (vLLM max-model-len); 292 previously refused rows answered by the lift."},{"engine":"djev","name":"djev","lift_class":"context-262144","rows_answered_by_lift":388,"line":"djev: max_model_len raised from 32,768 to 262,144, canvas from 128 to the checkpoint's declared 256, question cap from 32 to 256; API character caps opened; 388 previously refused rows answered by the lift."},{"engine":"jeff-uncapped","name":"jeff","lift_class":"tokenizer-budget-lean-encoder","rows_answered_by_lift":564,"line":"jeff: tokenizer max_len of 16,384 lifted (the encoder has no absolute position limit); rows that ran out of GPU memory re-answered with a lean encoder path that skips two unused n x n tensors; 564 previously refused rows answered by the lift."},{"engine":"gliner-base","name":"GLiNER 2.5 base","lift_class":"maxlen-paren-chunked-attention","rows_answered_by_lift":19525,"line":"GLiNER 2.5 base: harness max_len guard removed (upstream sets none); \"(\" and \")\" removed from GLiNER2's reserved characters (markers are routed by position; rows without parentheses bit-identical); memory-bound rows re-answered with query-chunked DeBERTa attention; 19,525 previously refused rows answered by the lift."},{"engine":"gliner-small","name":"GLiNER 2.5 small","lift_class":"maxlen-paren-chunked-attention","rows_answered_by_lift":19525,"line":"GLiNER 2.5 small: harness max_len guard removed (upstream sets none); \"(\" and \")\" removed from GLiNER2's reserved characters (markers are routed by position; rows without parentheses bit-identical); memory-bound rows re-answered with query-chunked DeBERTa attention; 19,525 previously refused rows answered by the lift."},{"engine":"gliner-multi","name":"GLiNER 2.5 multilingual","lift_class":"maxlen-paren-chunked-attention","rows_answered_by_lift":19852,"line":"GLiNER 2.5 multilingual: harness max_len guard removed (upstream sets none); \"(\" and \")\" removed from GLiNER2's reserved characters (markers are routed by position; rows without parentheses bit-identical); memory-bound rows re-answered with query-chunked DeBERTa attention; 19,852 previously refused rows answered by the lift."},{"engine":"akash-gemma","name":"system-one-gemma","lift_class":"max-len-32768-full","rows_answered_by_lift":41425,"line":"system-one-gemma: max_len raised from the 384 keyword default to the 32,768-token gemma-3-270m window; out-of-memory retries score options in sub-batches of 8 (0 flips, max |dp| 2.4e-6 on 30 rows); 41,425 previously refused rows answered by the lift."},{"engine":"jobe","name":"Jobe","lift_class":null,"rows_answered_by_lift":15504,"line":"Jobe: questions over 16 options answered through the author's score_text path; rows that ran out of GPU memory re-run with query-chunked eager attention (0 flips, max |dp| 0.061 on 30 rows); 15,504 previously refused rows answered by the lift."},{"engine":"hopper","name":"Hopper","lift_class":null,"rows_answered_by_lift":14500,"line":"Hopper: questions over 26 options answered by the author's v1.1.1 runtime (same weights and map; discussion #14): a tournament deals the menu into ceil(n/26) chunks with a fixed seed, keeps the 10 best, decides among them with one more pass, and spreads 5% of the mass uniformly over the whole menu; on this card v1.1.1 and v1.1.0 are bit-identical on 106 in-range rows (both 51/106 bit-identical to the rtx-3 release rows, max |dp| 0.078, 2 flips, a card/autotune effect), and in-range rows stay verbatim from the release run; 14,500 previously refused rows answered by the lift."},{"engine":"jevk5","name":"JevK5","lift_class":null,"rows_answered_by_lift":15317,"line":"JevK5: guard lifted; 15,317 previously refused rows answered by the lift."},{"engine":"decision-kai-7185f514","name":"Decision 1.0 Kai","lift_class":null,"rows_answered_by_lift":18969,"line":"Decision 1.0 Kai: rows refused at the 1,024-token predict_1k product cap answered with the bundle's native 8,192-token packing (the author points users to predict_1k); empty-state check reduced to a type check; 18,969 previously refused rows answered by the lift."},{"engine":"decision-lex","name":"Decision 1.0 Lex","lift_class":null,"rows_answered_by_lift":20791,"line":"Decision 1.0 Lex: rows refused at the 1,024-token predict_1k product cap answered with the bundle's native 8,192-token packing (the author points users to predict_1k); empty-state check reduced to a type check; 20,791 previously refused rows answered by the lift."},{"engine":"neohorse-jev-4b","name":"NeoHorse-Jev-4B","lift_class":"in-run","rows_answered_by_lift":null,"line":"NeoHorse-Jev-4B: four serving guards of the neohorse_decision runtime shipped in the bundle lifted (DecisionEngine documents them as deployment protection limits): max_state 2,048 and max_branch 8,192 raised to the 262,144-token backbone window (strict encode kept, so nothing is truncated), the 16-questions-per-request cap and the 32,768-token expanded-request budget opened. 9,388 of 151,024 rows need a lift (6,038 over the state cap, mostly RouterBench, POP909-CL and API-Bank; 3,401 over 16 questions: ACOS, ToolRet, BRIGHT, Home appliance, ContractNLI and 2 others; 183 over the token budget; 98 over the branch cap; longest row 22,163 tokens). Every in-range row checked is bit-identical to the stock defaults (74 smoke rows, 200 sampled full-run rows). The author's single padded forward is always tried first; 69 rows that ran out of GPU memory, or hit the torch conv1d 32-bit index limit (one 32-question ToolRet row at 22,163 tokens), were re-run with their question rows split in halves at the same padded width."},{"engine":"simple-jev-qwen3.8-27b","name":"simple-jev · Qwen3.8-27B (featherless)","lift_class":"in-run","rows_answered_by_lift":null,"line":"simple-jev · Qwen3.8-27B (featherless): served exactly as the user specified (examples_binary, --max-model-len 65,536, --max-batch-tokens 65,536, 256 branches, 255 options); 2 ToolRet rows whose question branch exceeds 65,536 tokens were re-answered with both limits raised to the 262,144-token Qwen3.8-27B window (the flag is an admission check, nothing is truncated either way). The server batches up to 32 question branches per forward and does not fit that next to the 27B weights on ToolRet/BRIGHT rows, so those 1,550 rows ran through the same service in-process, stepping the batch size down (32, 16, 8 ...) only after an out-of-memory; batch size moves probabilities by up to 0.05 with no choice flips on 7 smoke rows (208 questions)."},{"engine":"mojev","name":"MoJev","lift_class":"in-run","rows_answered_by_lift":null,"line":"MoJev: keeps its field_tokens_max lift: per-question prompt cap raised from the 32-token training default to the 262,144-token backbone window (the collate would otherwise cut instructions), context_tokens from 16,384 to 262,144."},{"engine":"winnow-e4b-q8","name":"Winnow-E4B","lift_class":"in-run","rows_answered_by_lift":null,"line":"Winnow-E4B: runs on the Winnow-12B lift: answer-label stop in the author's native engine (native/engine.h) raised from 64 to 255 labels; same label loop and single-token filter, first 64 labels unchanged; 10,582 release rows with more than 64 options (CLINC150+OOS, BANKING77, POP909-CL, ChessBench) answered, and 732 of 732 in-range rows re-run on the pinned build are bit-identical."},{"engine":"lumma-fev-0.6b","name":"Lumma-Fev-0.6B","lift_class":"in-run","rows_answered_by_lift":null,"line":"Lumma-Fev-0.6B: MAX_ROW in the author's modeling_fev.py raised from 8,192 to above the 12,288-token backbone window, so the author's own limits() lowers it to the window (a state is cut at 11,648 tokens by its QUESTION_RESERVE rule and each question row keeps the rest); every in-range smoke row is bit-identical to the stock limits."},{"engine":"tev1-0.8b","name":"Tev1-0.8B-experimental","lift_class":"in-run","rows_answered_by_lift":null,"line":"Tev1-0.8B-experimental: lifts of the Tev1-4B engine: an empty question is sent as \"\" (decide.py's nonempty-question check is client validation), noul questions are sent as two options A \"yes\" / B \"no\", and the context is the 262,144-token backbone window (no serving cap is published); questions over 24 options stay refused because the A-X alphabet is the trained answer format."},{"engine":"intern-decision-0.8b","name":"Intern-Decision-0.8B","lift_class":"in-run","rows_answered_by_lift":null,"line":"Intern-Decision-0.8B: three serving guards of the inference.py shipped with the checkpoint lifted: DecisionEngine max_length raised from 8,192 to the 262,144-token backbone window; the 1-16 questions check in validate_request (the HTTP service limit, per the release's docs/DATA.md) opened by running the author's own validation over chunks of 16, so the whole row is still one skeleton and one forward pass; the 62-symbol answer alphabet (A-Z, a-z, 0-9) extended with the 562 two-letter labels AA-ZZ that stay one token (the Winnow and razorback order and filter; the model was trained on the 62). 14,493 of 151,024 rows need a lift (CLINC150+OOS, BANKING77, POP909-CL and 3 ChessBench rows over 62 options; ACOS, ToolRet, BRIGHT, Home appliance, ContractNLI and 3 others over 16 questions; 1,681 rows over 8,192 tokens, longest 33,135); every in-range smoke row is bit-identical to the stock module."},{"engine":"intern-decision-2b","name":"Intern-Decision-2B","lift_class":"in-run","rows_answered_by_lift":null,"line":"Intern-Decision-2B: three serving guards of the inference.py shipped with the checkpoint lifted: DecisionEngine max_length raised from 8,192 to the 262,144-token backbone window; the 1-16 questions check in validate_request (the HTTP service limit, per the release's docs/DATA.md) opened by running the author's own validation over chunks of 16, so the whole row is still one skeleton and one forward pass; the 62-symbol answer alphabet (A-Z, a-z, 0-9) extended with the 562 two-letter labels AA-ZZ that stay one token (the Winnow and razorback order and filter; the model was trained on the 62). 14,493 of 151,024 rows need a lift (CLINC150+OOS, BANKING77, POP909-CL and 3 ChessBench rows over 62 options; ACOS, ToolRet, BRIGHT, Home appliance, ContractNLI and 3 others over 16 questions; 1,681 rows over 8,192 tokens, longest 33,135); every in-range smoke row is bit-identical to the stock module."},{"engine":"intern-decision-4b","name":"Intern-Decision-4B","lift_class":"in-run","rows_answered_by_lift":null,"line":"Intern-Decision-4B: three serving guards of the inference.py shipped with the checkpoint lifted: DecisionEngine max_length raised from 8,192 to the 262,144-token backbone window; the 1-16 questions check in validate_request (the HTTP service limit, per the release's docs/DATA.md) opened by running the author's own validation over chunks of 16, so the whole row is still one skeleton and one forward pass; the 62-symbol answer alphabet (A-Z, a-z, 0-9) extended with the 562 two-letter labels AA-ZZ that stay one token (the Winnow and razorback order and filter; the model was trained on the 62). 14,493 of 151,024 rows need a lift (CLINC150+OOS, BANKING77, POP909-CL and 3 ChessBench rows over 62 options; ACOS, ToolRet, BRIGHT, Home appliance, ContractNLI and 3 others over 16 questions; 1,681 rows over 8,192 tokens, longest 33,135); every in-range smoke row is bit-identical to the stock module."},{"engine":"julia-1","name":"Julia 1","lift_class":"in-run","rows_answered_by_lift":null,"line":"Julia 1: four serving guards of the julia runtime shipped in the model repo lifted, keeping its strict (never-truncate) encoding: the 2-20 options check in validate_row opened (the head scores each [MASK] option marker independently and has no per-option parameters), the 48-token option clip and its strict refusal removed, the question-plus-options budget raised from the card's head_length 512 to 8,187 (max_length - 5, the largest the runtime accepts; a budget, not a reserved region), and the strict refusal on a literal mask token replaced by the runtime's own clean(). Both functions are recompiled from the pinned source with only those edits. 24,706 of 151,024 rows need a lift (15,060 over 20 options: CLINC150+OOS, BANKING77, ChessBench, POP909-CL, API-Bank, HLE; 7,708 with an option over 48 tokens: HellaSwag, Habermas, When2Call, BPoMP and 6 others; 1,954 over the 512 head budget: MuSR, ToolRet, BRIGHT, MMLU, HLE, GPQA; 2 others). The 8,192-token encoder window (max_position_embeddings) is kept, so 18 rows over it are refused, never cut (11 ToolRet, 3 BRIGHT, 3 POP909-CL, 1 HLE). Every in-range row checked is bit-identical to the card settings (70 smoke rows, 200 sampled full-run rows)."},{"engine":"clm-v0.1-8b","name":"CLM-v0.1-8B","lift_class":"in-run","rows_answered_by_lift":null,"line":"CLM-v0.1-8B: full-text run: max_model_len raised from the script's 2,048 to the 40,960-token Qwen3-8B window with truncation off; over-window rows refused, not cut (the truncated 2,048-token variant is not used)."},{"engine":"rune-26b-a4b","name":"Surogate Rune 26B-A4B","lift_class":"candidate-harness","rows_answered_by_lift":2000,"line":"Surogate Rune 26B-A4B: PhishNChips harness mapping fixed: every row was refused because the surogate decisions endpoint requires a noul question's criteria to be exactly true/false and five of its noul questions carry an empty criteria object, and it also refuses a null option description (verdict_alt_minimal); empty noul criteria are now sent as {\"false\": \"False\", \"true\": \"True\"} (the runner's default noul wording, as Bosun's empty-criteria fix sends the server default) and a null description as its option key; questions without either are sent byte-identical, and 82/82 answered release smoke rows re-ran bit-identical; 2,000 previously refused rows answered by the fix."},{"engine":"decision-kai-7185f514","name":"Decision 1.0 Kai","lift_class":"candidate-harness","rows_answered_by_lift":1676,"line":"Decision 1.0 Kai: new benchmarks (MMLU-Pro, When2Call, RAGTruth): rows refused at the 1,024-token predict_1k product cap answered with the bundle's native 8,192-token packing at 7185f514, as in its 0.2 lift; 30/30 in-range rows re-ran bit-identical; 1,676 previously refused rows answered by the fix."}],"lifts_not_applied":[{"engine":"solomon-v11-bf16-encoding","line":"Solomon v1.1: lift not applied; the lifted run is built on the earlier solomon-v11-bf16 serving path, not on the encoding-fix path the board uses; on release-v2 rows it answers 422 rows the encoding run refused but refuses 4,768 rows the encoding run answered, so it would lower coverage; the release encoding-fix run stays until a lift over that path exists."}],"run_variation":[{"engine":"decider-35b-nvfp4","line":"Decider 35B-A3B: the release run and a rerun in the rebuilt release environment agree on 97.7% of choices (420 of 430; 9 of the 10 flips are near-ties); two reruns on one card are bit-identical in choice; the perturbation-suite control rows agree with the scored run on 96.9% (1,642 of 1,695)."},{"engine":"razorback-one-read-3e296f08","line":"openjev (razorback16): the openjev 0.4.0 build is not deterministic on the author's default flags; two smoke passes agree on 85.3% of choices (367 of 430), 95.6% with vLLM batch-invariant mode, which no documented option makes the default; the perturbation-suite control rows agree with the scored run on 92.6% (1,570 of 1,695). The full run uses the author defaults."},{"engine":"jevfire-uncapped","line":"Jevfire: the perturbation-suite control rows (unchanged source rows, same build, vLLM serving on the author's settings) agree with the scored run on 97.6% (1,655 of 1,695) of choices."},{"engine":"vllm-pr57250","line":"mmastrac diffusiongemma (vLLM PR 57250): the perturbation-suite control rows (unchanged source rows, same build, vLLM PR 57250 serving on the author's settings) agree with the scored run on 99.6% (1,688 of 1,695) of choices."},{"engine":"clm-v0.1-8b","line":"CLM-v0.1-8B: a rerun of 17 sampled rows on the scored serving settings reproduces 62 of 62 labels (max |dp| 0.011)."},{"engine":"tev1-4b","line":"Tev1-4B-experimental: two smoke passes on the scored vLLM settings agree on 100.0% (421 of 421) of labels."},{"engine":"tev1-0.8b","line":"Tev1-0.8B-experimental: two smoke passes on the scored vLLM settings agree on 100.0% (421 of 421) of labels and are bit-identical; further single-process reruns, a second card and five concurrent processes per card (the scored setting) differ from the first pass on at most 2 of 77 rows, each by one bf16 step in a label logit (max |dp| 0.031, no label changes)."},{"engine":"gliner-decide","line":"GLiNER2.5-Decide: two smoke passes agree on 100.0% (430 of 430) of labels, and every probability is bit-identical between the passes and across the two cards the run was split over."},{"engine":"winnow-e4b-q8","line":"Winnow-E4B: two smoke passes agree on 100.0% (424 of 424) of labels, and every probability is bit-identical between the passes, across cards, with four server processes per card, and between the pinned and label-lifted builds on every row with at most 64 options."},{"engine":"lumma-fev-0.1b","line":"Lumma-Fev-0.1B: two smoke passes agree on 100.0% (428 of 428) of labels, and every probability is bit-identical between the passes, with six concurrent processes on one card, and against the author's stock row limits on every row those limits accept."},{"engine":"lumma-fev-0.6b","line":"Lumma-Fev-0.6B: two smoke passes agree on 100.0% (430 of 430) of labels, and every probability is bit-identical between the passes, with six concurrent processes on one card, and against the author's stock row limits on every row those limits accept."},{"engine":"simple-jev-qwen3.8-27b","line":"simple-jev · Qwen3.8-27B (featherless): two smoke passes agree on 100.0% (302 of 302) of labels and every response is bit-identical; 1, 4 and 8 concurrent clients on one server return bit-identical answers on 164 release rows, all six cards reproduce 20 stored smoke rows bit for bit, and plain curl requests return the stored responses byte for byte."},{"engine":"lev","line":"lev: two smoke passes agree on 100.0% (430 of 430) of labels and are bit-identical. The run used three processes per card; rows that ran out of GPU memory in the author's single batched forward were re-run in smaller chunks, which is not bit-identical (on 60 smoke rows forced through that path: max |dp| 0.052, 2 label flips)."},{"engine":"jev-omni","line":"Jev-Omni: two smoke passes agree on 100.0% (430 of 430) of labels, and every probability is bit-identical between the passes, with three concurrent processes on one card, across the two cards, and against the author's unmodified load_jev_omni() on the card's four verification cases and 68 suite questions."},{"engine":"intern-decision-0.8b","line":"Intern-Decision-0.8B: two smoke passes agree on 100.0% (430 of 430) of labels, and every probability is bit-identical between the passes, with 10 concurrent processes on one card, and against the published guards on every row they accept."},{"engine":"intern-decision-2b","line":"Intern-Decision-2B: two smoke passes agree on 100.0% (430 of 430) of labels, and every probability is bit-identical between the passes, with 8 concurrent processes on one card, and against the published guards on every row they accept."},{"engine":"intern-decision-4b","line":"Intern-Decision-4B: two smoke passes agree on 100.0% (430 of 430) of labels, and every probability is bit-identical between the passes, with 6 concurrent processes on one card, and against the published guards on every row they accept."},{"engine":"neohorse-jev-4b","line":"NeoHorse-Jev-4B: two smoke passes agree on 100.0% (430 of 430) of labels and every probability is bit-identical between the passes; with four concurrent processes per card, 200 sampled in-range full-run rows are bit-identical to a fresh single process of the stock runtime."},{"engine":"julia-1","line":"Julia 1: two smoke passes agree on 100.0% (430 of 430) of labels and every logit is bit-identical between the passes, across 8 concurrent processes on one card, and against the card settings on every row they accept; with 16 processes per card, 200 sampled in-range full-run rows are bit-identical to a fresh single process of the stock runtime."},{"engine":"xor","line":"Xor: no repeat of the single-GPU run on the same settings is on file, so its run-to-run variation is not measured."}],"contamination":["reflex 4B (removed from the board): confirmed leakage. Its published LoRA adapter trained on 800 MMLU-Pro test items; on those items it scores 65.8% against 52.1% on held-out items, a gap of 13.7 points (95% CI 9.8 to 17.5). The author's recommended configuration is the frozen Qwen3.5-4B base without that adapter, so the adapter entry is not shown. reflex 27B stays.","Eikos-27B: 979 FinEntity test items it was trained on (author-disclosed) count as wrong answers; its FinEntity score is still computed over the full benchmark, and its area and index scores follow. caiovicentino1/eikos-decisions (declared on the Eikos-27B card) trains on FinEntity with the human entity labels (1,994 rows, used_in eikos-27b). All 979 suite FinEntity documents appear verbatim as training states (966 in the train split, 13 in the validation split), and every one of their 2,129 entity questions has a training question on the same entity. User decision 2026-09-27: every FinEntity row counts as wrong for this entrant.","Jet v6.2: every training file of its weight lineage (V6.0 train_v5_r2, V6.1 targeted, V6.2 focused) was rebuilt byte-identical to the hashes in github michaljach/jet with the author's own builders; 5 of our test items appear in it with the same label (2 BANKING77, 3 CLINC150), train-split duplicates of test utterances that passed the author's exclusion filter. Those items count as wrong answers; each benchmark is still computed over the full set. 4 RouterBench queries also occur in its V6.1 data (GSM8K train questions inside UltraFeedback prompts), but RouterBench's label is a routing choice that the training text does not carry, so they are not counted. Details: contamination/jet-v6.2-training-overlap.json.","JevK5: 980 MMLU-Pro test items it was trained on (author-disclosed) count as wrong answers; its MMLU-Pro score is still computed over the full benchmark, and its area and index scores follow. Union of the author-published training ids for JevK5 v0.2 (980) and v0.1 (480), mapped to catalog-57 run ids in contamination/jevk5-mmlu-pro-items.json; v0.2 descends from v0.1 training, so both count as trained-on.","lev: its declared training data (29 sources on 26 public datasets in Abhinavexists/lev sources.py) was rebuilt with the author's own builder; 45 of our test items appear in its 200,000 training records (22 BANKING77, 12 CLINC150+OOS, 1 ANLI, 9 ARC-Easy, 1 HellaSwag), mostly exact duplicates between the public train and test splits. Those items count as wrong answers; each benchmark is still computed over the full set. 42 RouterBench queries also occur in its training data (GSM8K train questions via UltraFeedback), but RouterBench's label is a routing choice that the training text does not carry, so they are not counted. Details: contamination/lev-training-overlap.json.","Lumma-Fev-0.1B: its model card declares no training data, but FrontiersMind's own decision datasets (new_kevv_dataset, v2_kevv_dataset) contain 3,077 of our 3,080 BANKING77 test items with their labels, and the card reports Banking77 among its four headline benchmarks. Those items count as wrong answers in its BANKING77 score, still computed over the full benchmark. Details: contamination/lumma-fev-*-overlap.json.","Lumma-Fev-0.6B: its model card declares no training data, but FrontiersMind's own decision datasets (new_kevv_dataset, v2_kevv_dataset) contain 3,077 of our 3,080 BANKING77 test items with their labels, and the card reports Banking77 among its four headline benchmarks. Those items count as wrong answers in its BANKING77 score, still computed over the full benchmark. Details: contamination/lumma-fev-*-overlap.json.","open-jev (pngwn): its training set, pngwn/typed-decisions-v2-system-one, was built from the MMLU-Pro test split, so 9,542 of our 12,032 MMLU-Pro items were trained on for two epochs, and 5,308 MMLU test questions reappear among them. It scores 0.929 on the trained MMLU-Pro items and 0.614 on the 2,490 it never saw, a gap of +31 points that no other entrant shows. Those contaminated items count as wrong answers in its MMLU-Pro and MMLU scores, which are still computed over the full benchmarks. Details: contamination/open-jev-mmlu-pro.md.","Shuffle test (options reordered, keys kept or renamed, on MMLU, GPQA Diamond, ARC-Challenge, HellaSwag and MMLU-Pro): 12 entrants tested, no entrant flagged (no drop of 5 points or more with a 95% CI excluding zero). 6 smaller drops under 5 points have a CI excluding zero: open-jev (pngwn) HellaSwag; AutoJev-27B MMLU-Pro; Winnow-12B MMLU; Winnow-12B HellaSwag; open-jev (JoshuaSP) MMLU-Pro; mmastrac diffusiongemma (vLLM PR 57250) HellaSwag.","Declared overlap, 57 MMLU-Pro: test items in training: jevk5: 980 author-published ids (v0.1 and v0.2 union), counted as wrong in its MMLU-Pro score; open-jev: 9,542 test items (typed-decisions-v2-system-one), counted as wrong in its MMLU-Pro score.","Declared overlap, 24 MMLU: test items in training: open-jev: 5,308 test questions, via MMLU-Pro ori_mmlu items, counted as wrong in its MMLU score. other splits of the same dataset in training: decider family, this-that, hopper, jqv-targeted-v2 (auxiliary_train).","Declared overlap, 26/27 ARC-Easy/Challenge: test items in training: 15 rows (12 Easy, 3 Challenge) duplicated in ARC train and MMLU auxiliary_train: counts for decider family, hopper, jevk5, jqv, this-that. other splits of the same dataset in training: decider family, hopper, jevk5, kev-8b (not on board).","Declared overlap, 29 HellaSwag: other splits of the same dataset in training: decider family, this-that (train split; ours is validation).","Declared overlap, 28 WinoGrande: other splits of the same dataset in training: decider family, this-that. used for checkpoint selection: pplx-decider-v1-27b used WinoGrande rows for checkpoint selection.","Declared overlap, 4 BANKING77: test items in training: 2 rows duplicated in the train split: counts for decider family, kev, jevk5, decision family, reflex. other splits of the same dataset in training: decider family, kev, jevk5, decision family, reflex.","Declared overlap, 5 CLINC150: other splits of the same dataset in training: decider family, hopper, decision family, reflex.","Declared overlap, 58 BBH / 59 RAGTruth: used for checkpoint selection: pplx-decider-v1-27b checkpoint selection used 750 BBH and 1,500 RAGTruth rows."],"other":["Contrastive rankers such as CLM-v0.1-8B do poorly when the option texts are bare labels (yes/no, NotMentioned): scored by similarity, the label carries almost nothing to rank, and CLM picks NotMentioned on all of ContractNLI, yes on 99% of BFCL decisions and no on all of ForecastBench.","Xor runs on a single RTX PRO 6000 with tensor parallelism 1, as every entrant runs on one card for fairness; the author validated the model at tensor parallelism 2 on two cards, and only the tensor-parallel split and the static memory fraction differ from that configuration."],"board_changes":{"replaced":[{"retired":"decision-kai","replaced_by":"decision-kai-7185f514","why":"newer Decision 1.0 Kai revision 7185f514","line":"Decision 1.0 Kai (decision-kai-7185f514) replaces decision-kai: newer Decision 1.0 Kai revision 7185f514."},{"retired":"decision-sol","replaced_by":"decision-sol-0665a411","why":"newer Decision 1.0 Sol revision 0665a411","line":"Decision 1.0 Sol (decision-sol-0665a411) replaces decision-sol: newer Decision 1.0 Sol revision 0665a411."},{"retired":"decision-nox-full","replaced_by":"decision-nox-0bb83350","why":"newer Decision 1.0 Nox revision 0bb83350","line":"Decision 1.0 Nox (decision-nox-0bb83350) replaces decision-nox-full: newer Decision 1.0 Nox revision 0bb83350."},{"retired":"this-that-1.0","replaced_by":"this-that-1.2","why":"newer this-that release","line":"this-that 1.2 (this-that-1.2) replaces this-that-1.0: newer this-that release."},{"retired":"nimble","replaced_by":"nimble-v2","why":"newer Nimble release (v2)","line":"Bespoke Nimble 9B v2 (nimble-v2) replaces nimble: newer Nimble release (v2)."},{"retired":"razorback-one-read","replaced_by":"razorback-one-read-3e296f08","why":"newer openjev build 0.4.0 @3e296f08","line":"openjev (razorback16) (razorback-one-read-3e296f08) replaces razorback-one-read: newer openjev build 0.4.0 @3e296f08."},{"retired":"laya","replaced_by":"laya-1e28ac20","why":"newer laya code revision 1e28ac20 (same weights)","line":"Laya (laya-1e28ac20) replaces laya: newer laya code revision 1e28ac20 (same weights)."},{"retired":"verdict","replaced_by":"verdict-8af2496e","why":"newer Verdict weights 8af2496e and code 30f15564","line":"Verdict (verdict-8af2496e) replaces verdict: newer Verdict weights 8af2496e and code 30f15564."},{"retired":"rune-26b-a4b","replaced_by":"rune-26b-a4b-v3","why":"author-submitted v3 checkpoint (kit PR #9), adopted after a 3,000-row reproduction matched it bit for bit; v1 stays on the 0.2 board","line":"Surogate Rune 26B-A4B v3 (rune-26b-a4b-v3) replaces rune-26b-a4b: author-submitted v3 checkpoint (kit PR #9), adopted after a 3,000-row reproduction matched it bit for bit; v1 stays on the 0.2 board."},{"retired":"reflex-27b","replaced_by":"reflex-27b-v2","why":"author-submitted full-coverage rerun with wide-choice one-token labels (kit PR #13), adopted after our latency-v1 run agreed with it within reflex's own run-to-run noise; v1 stays on the 0.2 board","line":"reflex 27B (reflex-27b-v2) replaces reflex-27b: author-submitted full-coverage rerun with wide-choice one-token labels (kit PR #13), adopted after our latency-v1 run agreed with it within reflex's own run-to-run noise; v1 stays on the 0.2 board."},{"retired":"hopper","replaced_by":"hopper-g-1.2","why":"Hopper (G) 1.2, the author-submitted run of the next Hopper adapter (same Qwen3.5-4B base, continued from Hopper 1.0's adapter, same 1.1.1 serving code and map; kit PR #17), adopted after our latency-v1 run agreed with it within the A10G-vs-RTX noise; 1.1.1 stays on the 0.2 board","line":"Hopper (G) 1.2 (hopper-g-1.2) replaces hopper: Hopper (G) 1.2, the author-submitted run of the next Hopper adapter (same Qwen3.5-4B base, continued from Hopper 1.0's adapter, same 1.1.1 serving code and map; kit PR #17), adopted after our latency-v1 run agreed with it within the A10G-vs-RTX noise; 1.1.1 stays on the 0.2 board."}],"removed":[{"engine":"nanojev-8k","why":"NanoJev retired; removed from the board."},{"engine":"reflex-4b","why":"Removed: the author's recommended configuration is the frozen Qwen3.5-4B base without the published LoRA adapter, and that adapter was trained on 800 MMLU-Pro test items. reflex 27B stays."},{"engine":"jqv-targeted-v2","why":"Parked; not run for 0.2."},{"engine":"bev","why":"Cancelled; not run."},{"engine":"jqv","why":"Left out of 0.2: the new index benchmarks were not run for it; its results stay on disk."}]}}} \ No newline at end of file diff --git a/data/orgs/Cloudflare.png b/data/orgs/Cloudflare.png deleted file mode 100644 index b25ad57e19495885a0be6ce0ddc7095c33dc3d50..0000000000000000000000000000000000000000 Binary files a/data/orgs/Cloudflare.png and /dev/null differ diff --git a/data/orgs/DangerLabs.png b/data/orgs/DangerLabs.png deleted file mode 100644 index f5b364e46f8ac74b97b2a8e7ac8d25e93a10172b..0000000000000000000000000000000000000000 Binary files a/data/orgs/DangerLabs.png and /dev/null differ diff --git a/data/orgs/DoccyHealth.png b/data/orgs/DoccyHealth.png deleted file mode 100644 index 99f4146e0a707595b8ab59a37987fa389a81840c..0000000000000000000000000000000000000000 Binary files a/data/orgs/DoccyHealth.png and /dev/null differ diff --git a/data/orgs/FINAL-Bench.png b/data/orgs/FINAL-Bench.png deleted file mode 100644 index 7b315af76b455b6c2969d50ced7395147f624725..0000000000000000000000000000000000000000 Binary files a/data/orgs/FINAL-Bench.png and /dev/null differ diff --git a/data/orgs/FrontiersMind.png b/data/orgs/FrontiersMind.png deleted file mode 100644 index b54442000fa6cfe91dea600b31bdb4aff113b191..0000000000000000000000000000000000000000 Binary files a/data/orgs/FrontiersMind.png and /dev/null differ diff --git a/data/orgs/Hanno-Labs.png b/data/orgs/Hanno-Labs.png deleted file mode 100644 index 2956c5fecfb1e31df09fc2f61ab81089c3148dcc..0000000000000000000000000000000000000000 Binary files a/data/orgs/Hanno-Labs.png and /dev/null differ diff --git a/data/orgs/HopitAI.png b/data/orgs/HopitAI.png deleted file mode 100644 index 250f10e4a45bd13c3228de53c51e19c73ea6d417..0000000000000000000000000000000000000000 Binary files a/data/orgs/HopitAI.png and /dev/null differ diff --git a/data/orgs/LatentMesh-Labs.png b/data/orgs/LatentMesh-Labs.png deleted file mode 100644 index 1650146e395ac4e8b8ba342a5977dc7c6fc7364d..0000000000000000000000000000000000000000 Binary files a/data/orgs/LatentMesh-Labs.png and /dev/null differ diff --git a/data/orgs/Leptos-ai.png b/data/orgs/Leptos-ai.png deleted file mode 100644 index 4df401beda7e5412ac1d7735fb2704f60fc8c73f..0000000000000000000000000000000000000000 Binary files a/data/orgs/Leptos-ai.png and /dev/null differ diff --git a/data/orgs/LiquidAI.png b/data/orgs/LiquidAI.png deleted file mode 100644 index 0308cf31ebab37b409587a308f5ef33bf634d60a..0000000000000000000000000000000000000000 Binary files a/data/orgs/LiquidAI.png and /dev/null differ diff --git a/data/orgs/MISA-core.png b/data/orgs/MISA-core.png deleted file mode 100644 index 337f7958b69cdd61caba49ee54c08e7c3bc71689..0000000000000000000000000000000000000000 Binary files a/data/orgs/MISA-core.png and /dev/null differ diff --git a/data/orgs/MachineFi.png b/data/orgs/MachineFi.png deleted file mode 100644 index 0ee3af004cc6f58f6a6c57797f98f48ec6d980f6..0000000000000000000000000000000000000000 Binary files a/data/orgs/MachineFi.png and /dev/null differ diff --git a/data/orgs/Maincode.png b/data/orgs/Maincode.png deleted file mode 100644 index 30931e185c283d0b3d300250b4bdcd7579f13528..0000000000000000000000000000000000000000 Binary files a/data/orgs/Maincode.png and /dev/null differ diff --git a/data/orgs/Nace-DeepInfra.png b/data/orgs/Nace-DeepInfra.png deleted file mode 100644 index 49196d5a5b2bec7185262a032c5f641faa26c65b..0000000000000000000000000000000000000000 Binary files a/data/orgs/Nace-DeepInfra.png and /dev/null differ diff --git a/data/orgs/OmniJev.png b/data/orgs/OmniJev.png deleted file mode 100644 index caa889f4a083bfd0c3c6af40991812a07744e55d..0000000000000000000000000000000000000000 Binary files a/data/orgs/OmniJev.png and /dev/null differ diff --git a/data/orgs/OneShotOrg.png b/data/orgs/OneShotOrg.png deleted file mode 100644 index d0dbf9dd468e12e03574704e216f9942c214639b..0000000000000000000000000000000000000000 Binary files a/data/orgs/OneShotOrg.png and /dev/null differ diff --git a/data/orgs/PelaAI.png b/data/orgs/PelaAI.png deleted file mode 100644 index 21df5a66ac9d566dab72cce2fe5739f721c63bc2..0000000000000000000000000000000000000000 Binary files a/data/orgs/PelaAI.png and /dev/null differ diff --git a/data/orgs/PixilabAI.png b/data/orgs/PixilabAI.png deleted file mode 100644 index b0c86cebcd44c44b56931e9326aa5cb2547bd623..0000000000000000000000000000000000000000 Binary files a/data/orgs/PixilabAI.png and /dev/null differ diff --git a/data/orgs/RBDash-Team.png b/data/orgs/RBDash-Team.png deleted file mode 100644 index b52ddc64c36ba94427cd7f9a2fa7b5397bfc8a46..0000000000000000000000000000000000000000 Binary files a/data/orgs/RBDash-Team.png and /dev/null differ diff --git a/data/orgs/StandardThinking.png b/data/orgs/StandardThinking.png deleted file mode 100644 index c61dd655df81f871ca9c4577426504de12c07ac9..0000000000000000000000000000000000000000 Binary files a/data/orgs/StandardThinking.png and /dev/null differ diff --git a/data/orgs/StrandsAgents.png b/data/orgs/StrandsAgents.png deleted file mode 100644 index 8664ac9f2c4faa2656e4f42381c41c55bfca64b7..0000000000000000000000000000000000000000 Binary files a/data/orgs/StrandsAgents.png and /dev/null differ diff --git a/data/orgs/SupersonicLabs.png b/data/orgs/SupersonicLabs.png deleted file mode 100644 index cca652f5ba77261956280b9f9ff4d3955d9a0228..0000000000000000000000000000000000000000 Binary files a/data/orgs/SupersonicLabs.png and /dev/null differ diff --git a/data/orgs/TokenRhythm.png b/data/orgs/TokenRhythm.png deleted file mode 100644 index 68e31ea9de9cc22bdf529a62d51b886a05d40131..0000000000000000000000000000000000000000 Binary files a/data/orgs/TokenRhythm.png and /dev/null differ diff --git a/data/orgs/Valen-Team.png b/data/orgs/Valen-Team.png deleted file mode 100644 index 2abf50901f262d34fcf8d53da5f08f85e5781a2f..0000000000000000000000000000000000000000 Binary files a/data/orgs/Valen-Team.png and /dev/null differ diff --git a/data/orgs/akivet.png b/data/orgs/akivet.png deleted file mode 100644 index 2877bc5114ad96a6a2823e3f16bfaa3e21a1075d..0000000000000000000000000000000000000000 Binary files a/data/orgs/akivet.png and /dev/null differ diff --git a/data/orgs/autotrust.png b/data/orgs/autotrust.png deleted file mode 100644 index ab9bf7835b1907435a963ba7eb3b5a6dea0f7cf9..0000000000000000000000000000000000000000 Binary files a/data/orgs/autotrust.png and /dev/null differ diff --git a/data/orgs/baselabs.png b/data/orgs/baselabs.png deleted file mode 100644 index 60aa32add74359eb64954806e307931b922c7fb0..0000000000000000000000000000000000000000 Binary files a/data/orgs/baselabs.png and /dev/null differ diff --git a/data/orgs/bespokelabs.png b/data/orgs/bespokelabs.png deleted file mode 100644 index bebd043dc9bd1d24a293dfdc123e6f75d3af09a9..0000000000000000000000000000000000000000 Binary files a/data/orgs/bespokelabs.png and /dev/null differ diff --git a/data/orgs/blackmallangcow.png b/data/orgs/blackmallangcow.png deleted file mode 100644 index 4fecb33a5dff99a17b05470787cffb593b655628..0000000000000000000000000000000000000000 Binary files a/data/orgs/blackmallangcow.png and /dev/null differ diff --git a/data/orgs/corners-ai.png b/data/orgs/corners-ai.png deleted file mode 100644 index 3d6d01e5344cb84e7dc9d56af0777ac5e05f895b..0000000000000000000000000000000000000000 Binary files a/data/orgs/corners-ai.png and /dev/null differ diff --git a/data/orgs/cortex-agent-llc.png b/data/orgs/cortex-agent-llc.png deleted file mode 100644 index d3adc1ebf90f66e3b9c15d4994d9bb9966c0b99b..0000000000000000000000000000000000000000 Binary files a/data/orgs/cortex-agent-llc.png and /dev/null differ diff --git a/data/orgs/decision-tune.png b/data/orgs/decision-tune.png deleted file mode 100644 index a6c2d90894be13629388a3d34841d0c762b61be4..0000000000000000000000000000000000000000 Binary files a/data/orgs/decision-tune.png and /dev/null differ diff --git a/data/orgs/empiriolabsai.png b/data/orgs/empiriolabsai.png deleted file mode 100644 index 905729b36e6d31e3bdd44f46832f40fbe1ebad36..0000000000000000000000000000000000000000 Binary files a/data/orgs/empiriolabsai.png and /dev/null differ diff --git a/data/orgs/fastino.png b/data/orgs/fastino.png deleted file mode 100644 index 29df7a57998890d558523e7670746705453bd0fd..0000000000000000000000000000000000000000 Binary files a/data/orgs/fastino.png and /dev/null differ diff --git a/data/orgs/felhen-ai.png b/data/orgs/felhen-ai.png deleted file mode 100644 index e41f6a1b1a6f809ca1222d963ea56c65311d44ad..0000000000000000000000000000000000000000 Binary files a/data/orgs/felhen-ai.png and /dev/null differ diff --git a/data/orgs/flock-io.png b/data/orgs/flock-io.png deleted file mode 100644 index b21a5ebe97dccd039364a68ca4e3ef98470064ea..0000000000000000000000000000000000000000 Binary files a/data/orgs/flock-io.png and /dev/null differ diff --git a/data/orgs/flymy-ai.png b/data/orgs/flymy-ai.png deleted file mode 100644 index 9649e67acf7daf62758f655b3d7b097baef2424f..0000000000000000000000000000000000000000 Binary files a/data/orgs/flymy-ai.png and /dev/null differ diff --git a/data/orgs/frontier-infra.png b/data/orgs/frontier-infra.png deleted file mode 100644 index 4bd95f3bc842eec115166d6f42031611c54bb2c3..0000000000000000000000000000000000000000 Binary files a/data/orgs/frontier-infra.png and /dev/null differ diff --git a/data/orgs/h2oai.png b/data/orgs/h2oai.png deleted file mode 100644 index e6d6fd45d3a8da0bcc2d3facd548c59a21beb855..0000000000000000000000000000000000000000 Binary files a/data/orgs/h2oai.png and /dev/null differ diff --git a/data/orgs/interfaze-ai.png b/data/orgs/interfaze-ai.png deleted file mode 100644 index e63bfec9003b5b0a7967c0f110d8caa9cc46b9e1..0000000000000000000000000000000000000000 Binary files a/data/orgs/interfaze-ai.png and /dev/null differ diff --git a/data/orgs/internlm.png b/data/orgs/internlm.png deleted file mode 100644 index 622cf6e2d621089698491deb087e66d8ccbc119e..0000000000000000000000000000000000000000 Binary files a/data/orgs/internlm.png and /dev/null differ diff --git a/data/orgs/juspay.png b/data/orgs/juspay.png deleted file mode 100644 index 682913fbfd563272040c908e0ba2f43b1ef20a43..0000000000000000000000000000000000000000 Binary files a/data/orgs/juspay.png and /dev/null differ diff --git a/data/orgs/knowledgator.png b/data/orgs/knowledgator.png deleted file mode 100644 index 9a9c1f2e50d610ba7f33db49380a27f1e5d16a4c..0000000000000000000000000000000000000000 Binary files a/data/orgs/knowledgator.png and /dev/null differ diff --git a/data/orgs/misa-jsc.png b/data/orgs/misa-jsc.png deleted file mode 100644 index e215b139fb0f51302db1c5187b6c8355ce1facdd..0000000000000000000000000000000000000000 Binary files a/data/orgs/misa-jsc.png and /dev/null differ diff --git a/data/orgs/moganai.png b/data/orgs/moganai.png deleted file mode 100644 index 40d078f6393cb939954492406de205488cc66572..0000000000000000000000000000000000000000 Binary files a/data/orgs/moganai.png and /dev/null differ diff --git a/data/orgs/nace-ai.png b/data/orgs/nace-ai.png deleted file mode 100644 index cdd3fcad2bc176edd4feb62f21932460c1d5629c..0000000000000000000000000000000000000000 Binary files a/data/orgs/nace-ai.png and /dev/null differ diff --git a/data/orgs/olaverse.png b/data/orgs/olaverse.png deleted file mode 100644 index 9fa1aac61e003148b367301d9f6470d764067542..0000000000000000000000000000000000000000 Binary files a/data/orgs/olaverse.png and /dev/null differ diff --git a/data/orgs/org2ai.png b/data/orgs/org2ai.png deleted file mode 100644 index a0c576424de3c94f37034ef2527b68b6ea35a4ce..0000000000000000000000000000000000000000 Binary files a/data/orgs/org2ai.png and /dev/null differ diff --git a/data/orgs/perplexity-ai.png b/data/orgs/perplexity-ai.png deleted file mode 100644 index 5d0a1b32dd38f76093aeae8825a3ff3529a5e3c9..0000000000000000000000000000000000000000 Binary files a/data/orgs/perplexity-ai.png and /dev/null differ diff --git a/data/orgs/run-saina.png b/data/orgs/run-saina.png deleted file mode 100644 index 404d6f1ffc0fa57d637c666d5737d5e12062c2f6..0000000000000000000000000000000000000000 Binary files a/data/orgs/run-saina.png and /dev/null differ diff --git a/data/orgs/shisa-ai.png b/data/orgs/shisa-ai.png deleted file mode 100644 index f1ea033580f9f31ad166dfb142938a4991f994f6..0000000000000000000000000000000000000000 Binary files a/data/orgs/shisa-ai.png and /dev/null differ diff --git a/data/orgs/startlux-models.png b/data/orgs/startlux-models.png deleted file mode 100644 index 70a71cd9451543838ba71b3eb661977f994fb366..0000000000000000000000000000000000000000 Binary files a/data/orgs/startlux-models.png and /dev/null differ diff --git a/data/orgs/sthanika-ai.png b/data/orgs/sthanika-ai.png deleted file mode 100644 index 7cc81072519be518fd521285b9a75be1025b2adc..0000000000000000000000000000000000000000 Binary files a/data/orgs/sthanika-ai.png and /dev/null differ diff --git a/data/orgs/surogate.png b/data/orgs/surogate.png deleted file mode 100644 index 1f3e3dbe37b4a6aec5c5235361aa68fac7ee9153..0000000000000000000000000000000000000000 Binary files a/data/orgs/surogate.png and /dev/null differ diff --git a/data/orgs/tasksource.png b/data/orgs/tasksource.png deleted file mode 100644 index 898e0a00487af4fb8f0353bf3bf1d5207a0e2367..0000000000000000000000000000000000000000 Binary files a/data/orgs/tasksource.png and /dev/null differ diff --git a/data/orgs/togethercomputer.png b/data/orgs/togethercomputer.png deleted file mode 100644 index a84257486fc0c95e1463afad96be9be883eb2c39..0000000000000000000000000000000000000000 Binary files a/data/orgs/togethercomputer.png and /dev/null differ diff --git a/data/orgs/torchcast-ai.png b/data/orgs/torchcast-ai.png deleted file mode 100644 index cb37b23862d4b27cd1101a1da88b463c04f56842..0000000000000000000000000000000000000000 Binary files a/data/orgs/torchcast-ai.png and /dev/null differ diff --git a/data/orgs/turion-ai.png b/data/orgs/turion-ai.png deleted file mode 100644 index c72292a5da895f803f0aa0c246684c0df3be1190..0000000000000000000000000000000000000000 Binary files a/data/orgs/turion-ai.png and /dev/null differ diff --git a/data/orgs/typesafe-ai.png b/data/orgs/typesafe-ai.png deleted file mode 100644 index 4744bf3b259fff71bac04d6ea1d800b810561876..0000000000000000000000000000000000000000 Binary files a/data/orgs/typesafe-ai.png and /dev/null differ diff --git a/data/orgs/vllm-sr.png b/data/orgs/vllm-sr.png deleted file mode 100644 index cab15269e06e17283c5b77837069b6b926aaf433..0000000000000000000000000000000000000000 Binary files a/data/orgs/vllm-sr.png and /dev/null differ diff --git a/data/orgs/wayfind.png b/data/orgs/wayfind.png deleted file mode 100644 index 5437c57852107c58cabda599fd25e0e47ffd44b0..0000000000000000000000000000000000000000 Binary files a/data/orgs/wayfind.png and /dev/null differ diff --git a/data/own.json b/data/own.json deleted file mode 100644 index cf0c3c11f1dd2989d1f1b65d44fae704c8880f8a..0000000000000000000000000000000000000000 --- a/data/own.json +++ /dev/null @@ -1 +0,0 @@ -{"formula":"v0.4","parts":{"public":0,"same_skills":0.6,"new_domains":0.4},"areas":[{"id":"knowledge","label":"Knowledge & Reasoning","w":0.258494},{"id":"language","label":"Language Understanding","w":0.258494},{"id":"retrieval","label":"Retrieval & Classification","w":0.200229},{"id":"tools","label":"Tools & Automation","w":0.182783},{"id":"arts","label":"Arts & Human Taste","w":0.1}],"eq":{"mB":23.4973,"sB":16.539081,"mS":24.268626,"sS":16.350451,"mO":16.884661,"sO":13.391589},"models":[{"e":"decider-chat-gemma4-31b","a":{"knowledge":[45.456,1,40.790417,0.9],"language":[59.492766,1,49.972553,0.893617],"retrieval":[63.081818,1,63.74303,1],"tools":[75.573704,1,70.741111,1],"arts":[42.176667,1,37.393333,0.833333]},"o":54.008625},{"e":"rune-26b-a4b-v3","a":{"knowledge":[45.954,1,38.28203,0.9],"language":[61.932128,1,52.724255,0.893617],"retrieval":[63.520909,1,62.640606,1],"tools":[71.221852,1,69.272593,1],"arts":[46.503333,1,43.683333,0.833333]},"o":43.2045},{"e":"decision-2.0-vega-27b","a":{"knowledge":[45.957667,1,34.551857,0.9],"language":[59.037234,1,48.552553,0.893617],"retrieval":[60.880303,1,61.410606,1],"tools":[73.535185,1,77.634444,1],"arts":[42.031667,1,38.418333,0.833333]},"o":48.116875},{"e":"kev-27b","a":{"knowledge":[45.6655,1,35.015063,0.9],"language":[62.598936,1,51.679787,0.893617],"retrieval":[55.749394,1,55.99,1],"tools":[72.645926,1,77.698519,1],"arts":[42.665,1,37.981667,0.833333]},"o":51.50825},{"e":"blink-v0.3-26b-a4b-nvfp4","a":{"knowledge":[45.124333,1,37.18073,0.9],"language":[63.563404,1,53.26766,0.893617],"retrieval":[63.023636,1,63.429394,1],"tools":[70.043333,1,69.613704,1],"arts":[46.975,1,43.956667,0.833333]},"o":42.046625},{"e":"simple-jev-qwen3.8-27b","a":{"knowledge":[36.916667,1,31.11834,0.9],"language":[61.223617,1,50.519574,0.893617],"retrieval":[63.25303,1,63.822727,1],"tools":[76.209259,1,75.920741,1],"arts":[40.815,1,38.206667,0.833333]},"o":46.362875},{"e":"jev-27b","a":{"knowledge":[41.406667,1,35.671767,0.9],"language":[56.444468,1,46.121277,0.893617],"retrieval":[53.741515,1,58.860303,1],"tools":[73.445185,1,74.08963,1],"arts":[41.605,1,39.518333,0.833333]},"o":48.606375},{"e":"eikos-27b-fp8","a":{"knowledge":[39.978833,1,34.365067,0.9],"language":[52.557447,1,49.641915,0.893617],"retrieval":[55.865455,1,59.684242,1],"tools":[74.434444,1,78.642593,1],"arts":[41.416667,1,39.675,0.833333]},"o":49.43},{"e":"jade","a":{"knowledge":[41.244667,1,33.79223,0.9],"language":[56.456809,1,50.88234,0.893617],"retrieval":[54.398788,1,56.408182,1],"tools":[73.177407,1,73.802222,1],"arts":[38.968333,1,38.125,0.833333]},"o":44.920875},{"e":"jev-gemma4-26b-a4b","a":{"knowledge":[43.736333,1,35.38982,0.9],"language":[61.23,1,50.288511,0.893617],"retrieval":[67.884242,1,63.612424,1],"tools":[69.725926,1,66.581852,1],"arts":[45.558333,1,40.893333,0.833333]},"o":39.8625},{"e":"xor-1.2","a":{"knowledge":[35.471667,1,32.759497,0.9],"language":[56.862553,1,48.528511,0.893617],"retrieval":[54.457273,1,58.236061,1],"tools":[71.190741,1,71.828148,1],"arts":[44.961667,1,40.471667,0.833333]},"o":41.81525},{"e":"decider-chat-qwen3.6-27b","a":{"knowledge":[38.304167,1,34.74049,0.9],"language":[55.938936,1,47.36617,0.893617],"retrieval":[52.231515,1,54.186061,1],"tools":[71.423333,1,70.071111,1],"arts":[36.955,1,36.175,0.833333]},"o":45.7225},{"e":"clef","a":{"knowledge":[52.804333,1,33.227877,0.9],"language":[61.268936,1,43.149787,0.893617],"retrieval":[62.839697,1,60.132121,1],"tools":[80.714815,1,75.689259,1],"arts":[48.885,1,37.703333,0.833333]},"o":45.06175},{"e":"joshua-diffusion-full","a":{"knowledge":[32.463833,1,29.99149,0.9],"language":[54.309362,1,46.786383,0.893617],"retrieval":[58.373939,1,54.592727,1],"tools":[70.164815,1,67.168148,1],"arts":[31.981667,1,34.59,0.833333]},"o":37.939625},{"e":"winnow-12b-q8","a":{"knowledge":[34.754667,1,28.724727,0.9],"language":[56.985957,1,45.602979,0.893617],"retrieval":[54.047273,1,55.020909,1],"tools":[71.022222,1,63.23037,1],"arts":[31.376667,1,29.813333,0.833333]},"o":34.25825},{"e":"jevfire-uncapped","a":{"knowledge":[30.253833,1,22.75185,0.9],"language":[53.895319,1,40.383404,0.893617],"retrieval":[56.216061,1,54.123939,1],"tools":[72.332222,1,72.411111,1],"arts":[35.68,1,31.828333,0.833333]},"o":31.428},{"e":"decision-2.0-lux-9b","a":{"knowledge":[35.457333,1,27.847143,0.9],"language":[46.848511,1,41.640426,0.893617],"retrieval":[55.652424,1,54.722121,1],"tools":[60.36963,1,65.298889,1],"arts":[33.078333,1,31.836667,0.833333]},"o":34.022875},{"e":"nimble-v3","a":{"knowledge":[40.96,1,30.18736,0.9],"language":[60.680426,1,49.157447,0.893617],"retrieval":[55.477273,1,53.296061,1],"tools":[84.244444,1,73.957037,1],"arts":[44.05,1,39.715,0.833333]},"o":38.195125},{"e":"jevk5-9b-v0.3.3","a":{"knowledge":[31.644167,1,29.5824,0.9],"language":[54.675532,1,48.355957,0.893617],"retrieval":[49.984848,1,52.492424,1],"tools":[64.144444,1,61.176667,1],"arts":[35.698333,1,29.76,0.833333]},"o":30.791375},{"e":"jpt-9b","a":{"knowledge":[30.921167,1,26.791723,0.9],"language":[57.678936,1,42.007234,0.893617],"retrieval":[44.585455,1,42.861818,1],"tools":[67.024074,1,61.387407,1],"arts":[28.781667,1,28.226667,0.833333]},"o":34.981},{"e":"jev-9b","a":{"knowledge":[30.717333,1,27.485467,0.9],"language":[44.151915,1,36.902766,0.893617],"retrieval":[46.450606,1,48.927879,1],"tools":[62.058148,1,57.188889,1],"arts":[32.825,1,29.346667,0.833333]},"o":31.84725},{"e":"sieve-9b","a":{"knowledge":[28.9625,1,23.340527,0.9],"language":[45.554468,1,40.954894,0.893617],"retrieval":[46.065152,1,46.348485,1],"tools":[61.297778,1,62.467037,1],"arts":[21.865,1,25.345,0.833333]},"o":30.95525},{"e":"kev-9b-v2","a":{"knowledge":[30.797167,1,26.35679,0.9],"language":[46.678298,1,40.078723,0.893617],"retrieval":[46.794848,1,47.95,1],"tools":[56.858519,1,59.024444,1],"arts":[17.506667,1,18.411667,0.833333]},"o":29.901125},{"e":"jev-omni","a":{"knowledge":[26.091333,1,21.800603,0.9],"language":[51.335319,1,43.463191,0.893617],"retrieval":[35.825152,1,39.932121,1],"tools":[60.770741,1,63.711852,1],"arts":[27.425,1,27.153333,0.833333]},"o":36.0445},{"e":"decision-2.0-nox-4b","a":{"knowledge":[31.370167,1,26.178033,0.9],"language":[46.381064,1,38.195106,0.893617],"retrieval":[52.437879,1,52.73303,1],"tools":[60.118519,1,58.799259,1],"arts":[26.248333,1,27.136667,0.833333]},"o":27.9115},{"e":"decider-35b-nvfp4","a":{"knowledge":[35.856167,1,29.433963,0.9],"language":[52.746809,1,43.729362,0.893617],"retrieval":[54.664242,1,52.577576,1],"tools":[56.537407,1,48.737407,1],"arts":[35.18,1,36.165,0.833333]},"o":38.1185},{"e":"djev","a":{"knowledge":[25.934833,1,20.819623,0.9],"language":[50.033617,1,43.834255,0.893617],"retrieval":[41.770303,1,45.364545,1],"tools":[51.681111,1,53.354815,1],"arts":[34.406667,1,31.676667,0.833333]},"o":25.221625},{"e":"jpt-4b","a":{"knowledge":[26.513833,1,25.101327,0.9],"language":[53.73234,1,37.383191,0.893617],"retrieval":[45.006061,1,40.997273,1],"tools":[57.196296,1,56.548148,1],"arts":[26.111667,1,26.036667,0.833333]},"o":29.929875},{"e":"nimble-v2","a":{"knowledge":[30.4765,1,24.614213,0.9],"language":[44.30617,1,40.128936,0.893617],"retrieval":[34.927879,1,36.678182,1],"tools":[59.212222,1,53.404815,1],"arts":[27.56,1,29.028333,0.833333]},"o":35.7375},{"e":"hopper-g-1.2","a":{"knowledge":[24.5905,1,24.471,0.9],"language":[45.66617,1,36.112553,0.893617],"retrieval":[45.417576,1,47.056061,1],"tools":[60.064444,1,62.529259,1],"arts":[29.166667,1,28.888333,0.833333]},"o":28.390375},{"e":"razorback-one-read-3e296f08","a":{"knowledge":[25.694,1,21.338577,0.9],"language":[45.503617,1,41.786383,0.893617],"retrieval":[37.01697,1,38.242727,1],"tools":[47.186667,1,54.378148,1],"arts":[33.068333,1,30.416667,0.833333]},"o":26.4285},{"e":"kev-4b-r10","a":{"knowledge":[26.299833,1,21.644387,0.9],"language":[41.782766,1,32.368298,0.893617],"retrieval":[42.481212,1,42.223939,1],"tools":[55.01037,1,50.574815,1],"arts":[17.9,1,19.81,0.833333]},"o":25.395375},{"e":"lev","a":{"knowledge":[22.783,1,19.46748,0.9],"language":[39.131064,1,26.983404,0.893617],"retrieval":[47.920303,1,42.879697,1],"tools":[57.524074,1,58.358148,1],"arts":[30.815,1,33.346667,0.833333]},"o":25.2255},{"e":"solomon-v11-bf16-encoding","a":{"knowledge":[27.805,1,20.013473,0.9],"language":[56.663404,1,48.815319,0.893617],"retrieval":[25.972727,1,27.62697,1],"tools":[46.968148,1,46.07,1],"arts":[24.466667,1,26.141667,0.833333]},"o":41.250125},{"e":"jevk5-v0.3","a":{"knowledge":[25.9425,1,23.86624,0.9],"language":[46.56617,1,41.834894,0.893617],"retrieval":[44.468485,1,46.14697,1],"tools":[56.937037,1,53.400741,1],"arts":[30.825,1,28.43,0.833333]},"o":21.9145},{"e":"neohorse-jev-4b","a":{"knowledge":[23.606,1,21.624643,0.9],"language":[39.621702,1,31.158298,0.893617],"retrieval":[43.908788,1,45.187576,1],"tools":[58.879259,1,56.124074,1],"arts":[11.025,1,13.665,0.833333]},"o":29.228125},{"e":"ryotide-qwen-v0.3.1","a":{"knowledge":[20.954667,1,17.149413,0.9],"language":[35.988511,1,30.24383,0.893617],"retrieval":[47.520606,1,46.359394,1],"tools":[55.084815,1,52.174074,1],"arts":[29.811667,1,30.303333,0.833333]},"o":18.83325},{"e":"intern-decision-4b","a":{"knowledge":[22.962667,1,21.34866,0.9],"language":[42.544468,1,37.694468,0.893617],"retrieval":[40.341515,1,40.111515,1],"tools":[55.656296,1,56.208519,1],"arts":[28.181667,1,27.971667,0.833333]},"o":29.414},{"e":"winnow-e4b-q8","a":{"knowledge":[22.763833,1,18.77626,0.9],"language":[46.770851,1,34.082979,0.893617],"retrieval":[43.79697,1,41.348788,1],"tools":[62.509259,1,53.952222,1],"arts":[26.933333,1,24.133333,0.833333]},"o":20.15875},{"e":"clef-flash","a":{"knowledge":[52.975833,1,29.854243,0.9],"language":[47.218511,1,36.7,0.893617],"retrieval":[52.155758,1,41.42,1],"tools":[81.867037,1,66.235556,1],"arts":[48.406667,1,33.22,0.833333]},"o":34.998375},{"e":"decider-4b","a":{"knowledge":[27.697167,1,24.380443,0.9],"language":[43.985532,1,30.841702,0.893617],"retrieval":[44.674242,1,42.785152,1],"tools":[58.624444,1,48.222222,1],"arts":[30.041667,1,29.308333,0.833333]},"o":26.93475},{"e":"vllm-pr57250","a":{"knowledge":[24.9885,1,21.585173,0.9],"language":[46.479574,1,43.076383,0.893617],"retrieval":[24.169091,1,23.794545,1],"tools":[33.985185,1,37.2,1],"arts":[31.695,1,28.228333,0.833333]},"o":27.384625},{"e":"intelif-qwen3-4b","a":{"knowledge":[19.471833,1,18.969643,0.9],"language":[31.211915,1,33.252553,0.893617],"retrieval":[39.547879,1,37.272121,1],"tools":[50.962963,1,55.355556,1],"arts":[21.34,1,23.235,0.833333]},"o":18.4335},{"e":"jobe","a":{"knowledge":[18.453667,1,15.358783,0.9],"language":[34.852128,1,32.738511,0.893617],"retrieval":[36.690303,1,38.113939,1],"tools":[48.805185,1,48.654074,1],"arts":[28.868333,1,29.761667,0.833333]},"o":18.388875},{"e":"gevva-e4b","a":{"knowledge":[17.912,1,15.786803,0.9],"language":[27.611489,1,25.941277,0.893617],"retrieval":[39.969091,1,33.517879,1],"tools":[56.936296,1,48.931481,1],"arts":[22.616667,1,24.18,0.833333]},"o":20.147125},{"e":"pngwn-space-uncapped-full","a":{"knowledge":[22.7915,1,22.175077,0.9],"language":[34.680426,1,33.514681,0.893617],"retrieval":[29.113939,1,24.742424,1],"tools":[45.73037,1,37.29,1],"arts":[14.451667,1,17.618333,0.833333]},"o":25.18625},{"e":"tev1-4b","a":{"knowledge":[24.345333,1,21.180777,0.9],"language":[39.18383,1,37.525319,0.893617],"retrieval":[15.48,1,19.29697,1],"tools":[41.828519,1,44.730741,1],"arts":[23.476667,1,23.383333,0.833333]},"o":28.371625},{"e":"openvons","a":{"knowledge":[19.491167,1,18.601737,0.9],"language":[30.730426,1,28.96234,0.893617],"retrieval":[25.368788,1,30.71697,1],"tools":[44.64963,1,47.351111,1],"arts":[24.4,1,27.781667,0.833333]},"o":17.920375},{"e":"gevva-e2b","a":{"knowledge":[11.9435,1,13.306903,0.9],"language":[24.980426,1,19.688298,0.893617],"retrieval":[38.604848,1,39.107576,1],"tools":[52.53037,1,45.495926,1],"arts":[16.523333,1,11.583333,0.833333]},"o":14.109625},{"e":"metask-jev-4b","a":{"knowledge":[21.552833,1,19.96154,0.9],"language":[40.05234,1,33.580851,0.893617],"retrieval":[12.912727,1,14.973333,1],"tools":[37.592222,1,38.967037,1],"arts":[21.941667,1,19.123333,0.833333]},"o":30.07975},{"e":"semif","a":{"knowledge":[17.893333,1,14.52659,0.9],"language":[34.019574,1,32.639149,0.893617],"retrieval":[15.719394,1,18.646364,1],"tools":[39.095185,1,40.235926,1],"arts":[27.92,1,28.013333,0.833333]},"o":18.742875},{"e":"mini-jev","a":{"knowledge":[15.829,1,17.01331,0.9],"language":[29.785532,1,26.649787,0.893617],"retrieval":[8.659394,1,10.905758,1],"tools":[28.415926,1,30.304074,1],"arts":[25.29,1,27.175,0.833333]},"o":16.615},{"e":"sieve-2b","a":{"knowledge":[10.43,1,12.001327,0.9],"language":[20.784894,1,20.086383,0.893617],"retrieval":[28.46303,1,28.727273,1],"tools":[35.277778,1,32.412222,1],"arts":[19.456667,1,20.291667,0.833333]},"o":14.413125},{"e":"decider-2b-fp8-http","a":{"knowledge":[17.1805,1,14.650057,0.9],"language":[32.205106,1,25.103617,0.893617],"retrieval":[37.284848,1,37.31,1],"tools":[42.451481,1,36.294074,1],"arts":[17.568333,1,20.991667,0.833333]},"o":12.038},{"e":"decision-2.0-sol-2b","a":{"knowledge":[16.219167,1,12.480247,0.9],"language":[24.631915,1,22.13383,0.893617],"retrieval":[45.58,1,41.015455,1],"tools":[48.798889,1,47.117407,1],"arts":[14.411667,1,14.318333,0.833333]},"o":11.060375},{"e":"intern-decision-2b","a":{"knowledge":[12.01,1,9.38917,0.9],"language":[21.260851,1,26.346809,0.893617],"retrieval":[15.654848,1,18.547273,1],"tools":[33.334444,1,37.503704,1],"arts":[14.355,1,18.468333,0.833333]},"o":12.90625},{"e":"jpt-0.8b","a":{"knowledge":[8.157,1,6.838413,0.9],"language":[25.167872,1,17.935532,0.893617],"retrieval":[16.402727,1,15.46,1],"tools":[36.234074,1,34.671481,1],"arts":[8.443333,1,7.311667,0.833333]},"o":8.549},{"e":"this-that-1.2","a":{"knowledge":[16.583,1,13.634277,0.9],"language":[31.538936,1,21.562979,0.893617],"retrieval":[32.825152,1,38.488788,1],"tools":[44.965185,1,38.953333,1],"arts":[14.193333,1,16.583333,0.833333]},"o":15.649375},{"e":"bosun-v3.1-1.7b","a":{"knowledge":[4.748167,1,3.605523,0.9],"language":[13.522553,1,13.905106,0.893617],"retrieval":[42.820303,1,38.534242,1],"tools":[34.778519,1,32.578519,1],"arts":[8.961667,1,7.876667,0.833333]},"o":7.5375},{"e":"kodiak-v0.2-1b-accuracy","a":{"knowledge":[13.512167,1,9.262613,0.9],"language":[15.308085,1,10.283404,0.893617],"retrieval":[36.17697,1,30.61303,1],"tools":[15.133333,1,13.666667,1],"arts":[15.891667,1,15.11,0.833333]},"o":7.131625},{"e":"decision-2.0-kai-0.6b","a":{"knowledge":[6.436,1,6.287973,0.9],"language":[11.562766,1,12.478511,0.893617],"retrieval":[31.96697,1,27.498485,1],"tools":[26.726296,1,25.163704,1],"arts":[7.441667,1,7.631667,0.833333]},"o":11.634375},{"e":"decision-2.0-eos-0.8b","a":{"knowledge":[7.0335,1,7.672303,0.9],"language":[15.288298,1,18.298085,0.893617],"retrieval":[38.414848,1,33.725455,1],"tools":[34.024815,1,29.912963,1],"arts":[8.883333,1,8.798333,0.833333]},"o":10.15525},{"e":"tev1-0.8b","a":{"knowledge":[6.778167,1,4.88825,0.9],"language":[15.264043,1,13.634468,0.893617],"retrieval":[4.87697,1,4.412727,1],"tools":[31.965185,1,31.256667,1],"arts":[5.868333,1,5.551667,0.833333]},"o":8.6005},{"e":"intern-decision-0.8b","a":{"knowledge":[6.769167,1,6.757023,0.9],"language":[10.871702,1,8.371915,0.893617],"retrieval":[6.440303,1,9.43303,1],"tools":[28.382222,1,29.926667,1],"arts":[7.721667,1,7.583333,0.833333]},"o":8.9745},{"e":"kev-0.8b-r15","a":{"knowledge":[6.967833,1,7.508887,0.9],"language":[14.231064,1,11.865532,0.893617],"retrieval":[22.127273,1,17.580303,1],"tools":[35.522963,1,31.344074,1],"arts":[3.801667,1,4.728333,0.833333]},"o":12.66925},{"e":"von-1.3","a":{"knowledge":[9.277,1,6.419733,0.9],"language":[11.634043,1,13.944468,0.893617],"retrieval":[30.131212,1,21.662727,1],"tools":[11.742222,1,8.52037,1],"arts":[2.705,1,7.481667,0.833333]},"o":6.989625},{"e":"jevk5-lite","a":{"knowledge":[5.360333,1,5.141947,0.9],"language":[9.036383,1,13.121064,0.893617],"retrieval":[28.659394,1,21.158182,1],"tools":[3.095556,1,0.673333,1],"arts":[2.551667,1,5.613333,0.833333]},"o":4.17975},{"e":"gliner-decide","a":{"knowledge":[2.294,1,2.67586,0.9],"language":[8.26383,1,14.603404,0.893617],"retrieval":[25.320606,1,19.121515,1],"tools":[14.777778,1,13.180741,1],"arts":[9.788333,1,5.055,0.833333]},"o":4.85775},{"e":"bosun-v3.1-0.6b","a":{"knowledge":[2.632667,1,1.2791,0.9],"language":[4.31617,1,5.397234,0.893617],"retrieval":[34.152424,1,28.869091,1],"tools":[28.445556,1,27.204444,1],"arts":[6.68,1,6.288333,0.833333]},"o":4.176},{"e":"jev-style-0.8b-v3","a":{"knowledge":[5.358167,1,3.588227,0.9],"language":[11.891489,1,11.510851,0.893617],"retrieval":[26.213333,1,22.453636,1],"tools":[35.358148,1,24.551111,1],"arts":[8.301667,1,7.403333,0.833333]},"o":6.44175},{"e":"lavoir","a":{"knowledge":[2.774,1,3.215393,0.9],"language":[11.480213,1,9.088936,0.893617],"retrieval":[11.995758,1,17.913636,1],"tools":[13.021481,1,10.724815,1],"arts":[3.873333,1,4.81,0.833333]},"o":8.94075},{"e":"jeff-uncapped","a":{"knowledge":[2.666667,1,1.736483,0.9],"language":[11.636809,1,7.785319,0.893617],"retrieval":[21.310606,1,18.020303,1],"tools":[1.188148,1,0.775556,1],"arts":[0.923333,1,1.653333,0.833333]},"o":3.288},{"e":"lfm2600","a":{"knowledge":[6.5825,1,6.818717,0.9],"language":[5.905106,1,5.736383,0.893617],"retrieval":[10.534242,1,5.915152,1],"tools":[4.427037,1,9.046296,1],"arts":[6.296667,1,10.286667,0.833333]},"o":6.307125},{"e":"sifr-0.8b-v3.1","a":{"knowledge":[13.223,1,8.08648,0.9],"language":[23.095745,1,13.60617,0.893617],"retrieval":[45.509394,1,26.709394,1],"tools":[35.94963,1,31.967407,1],"arts":[5.77,1,8.015,0.833333]},"o":5.306625},{"e":"laya-1e28ac20","a":{"knowledge":[2.898,1,2.16079,0.9],"language":[10.051702,1,13.788511,0.893617],"retrieval":[7.149394,1,10.157273,1],"tools":[5.82,1,8.888519,1],"arts":[3.466667,1,6.606667,0.833333]},"o":6.63225},{"e":"clm-v0.1-8b","a":{"knowledge":[2.3855,1,1.013043,0.9],"language":[5.681277,1,5.974681,0.893617],"retrieval":[11.259394,1,5.590909,1],"tools":[13.98,1,13.171111,1],"arts":[5.156667,1,2.111667,0.833333]},"o":4.94175},{"e":"harsha","a":{"knowledge":[1.3995,1,2.330343,0.9],"language":[5.197872,1,7.762553,0.893617],"retrieval":[0.941818,1,2.029394,1],"tools":[6.997778,1,8.688148,1],"arts":[8.501667,1,11.713333,0.833333]},"o":7.397},{"e":"akash-gemma","a":{"knowledge":[1.273167,1,2.953,0.9],"language":[1.212766,1,2.414468,0.893617],"retrieval":[17.223636,1,9.730303,1],"tools":[3.308148,1,3.544444,1],"arts":[4.898333,1,6.9,0.833333]},"o":3.523125},{"e":"julia-1","a":{"knowledge":[1.429667,1,4.548397,0.9],"language":[1.391702,1,2.218936,0.893617],"retrieval":[9.911212,1,4.491515,1],"tools":[7.205556,1,7.033704,1],"arts":[2.085,1,2.846667,0.833333]},"o":3.184125},{"e":"verdict-8af2496e","a":{"knowledge":[0.322,1,2.818,0.9],"language":[1.639362,1,0.795745,0.893617],"retrieval":[0,1,0.532727,1],"tools":[6.576296,1,6.542593,1],"arts":[1.513333,1,0,0.833333]},"o":0.319875},{"e":"lfm350-sdpa","a":{"knowledge":[0.806833,1,0.595373,0.9],"language":[0.593617,1,2.694255,0.893617],"retrieval":[0.024242,1,0.107576,1],"tools":[3.84,1,4.464444,1],"arts":[3.57,1,4.436667,0.833333]},"o":8.18},{"e":"strands-decider-2b","a":{"knowledge":[11.188167,1,9.49053,0.9],"language":[25.651915,1,20.025532,0.893617],"retrieval":[34.036061,1,26.258788,1],"tools":[43.778889,1,34.544074,1],"arts":[10.653333,1,16.596667,0.833333]},"o":13.519875},{"e":"d1-3b","a":{"knowledge":[24.329167,1,17.986617,0.9],"language":[56.529362,1,37.917234,0.893617],"retrieval":[52.844848,1,49.905152,1],"tools":[74.49037,1,58.713704,1],"arts":[38.92,1,31.676667,0.833333]},"o":11.467875},{"e":"d1-omni-600m","a":{"knowledge":[6.546833,1,5.640643,0.9],"language":[19.29383,1,7.692553,0.893617],"retrieval":[34.966061,1,24.535758,1],"tools":[19.511481,1,14.847407,1],"arts":[6.515,1,8.868333,0.833333]},"o":7.12925},{"e":"clef-omni","a":{"knowledge":[53.766667,1,31.267153,0.9],"language":[49.76,1,36.90766,0.893617],"retrieval":[59.267879,1,56.948182,1],"tools":[72.621481,1,60.307778,1],"arts":[46.411667,1,32.035,0.833333]},"o":33.166625},{"e":"drex-v1.5","a":{"knowledge":[44.569,1,28.716683,0.9],"language":[60.439787,1,36.950426,0.893617],"retrieval":[61.69,1,59.559697,1],"tools":[74.968148,1,71.428148,1],"arts":[48.766667,1,42.15,0.833333]},"o":30.81675},{"e":"decisiontune-1.0","a":{"knowledge":[11.727667,1,10.362487,0.9],"language":[29.919787,1,13.973404,0.893617],"retrieval":[45.016061,1,34.452727,1],"tools":[46.492222,1,37.260741,1],"arts":[5.703333,1,5.665,0.833333]},"o":6.81925},{"e":"vera-spark","a":{"knowledge":[10.088667,1,10.641657,0.9],"language":[24.847872,1,10.684043,0.893617],"retrieval":[30.081212,1,19.497273,1],"tools":[18.066296,1,17.403333,1],"arts":[12.373333,1,14.736667,0.833333]},"o":2.81475},{"e":"nirnaya-0.4b","a":{"knowledge":[0.513,1,1.172193,0.9],"language":[0.494681,1,3.3,0.893617],"retrieval":[1.443939,1,0.466667,1],"tools":[5.451111,1,5.041111,1],"arts":[0.535,1,3.025,0.833333]},"o":2.978125},{"e":"saracura-ptbr-4b","a":{"knowledge":[25.253667,1,18.71745,0.9],"language":[41.900213,1,32.510638,0.893617],"retrieval":[46.244242,1,43.182121,1],"tools":[53.938889,1,49.538519,1],"arts":[14.855,1,15.763333,0.833333]},"o":25.8705},{"e":"aplomb-1","a":{"knowledge":[29.8505,1,28.716143,0.9],"language":[47.226809,1,39.308936,0.893617],"retrieval":[44.49697,1,43.503636,1],"tools":[61.577407,1,53.618148,1],"arts":[33.991667,1,32.333333,0.833333]},"o":29.67325},{"e":"rsi-jev-v6.1-vl-4b","a":{"knowledge":[31.638167,1,25.90403,0.9],"language":[55.308723,1,40.181702,0.893617],"retrieval":[57.072424,1,55.149394,1],"tools":[68.423333,1,67.42963,1],"arts":[45.708333,1,41.746667,0.833333]},"o":29.806},{"e":"sieve-4b","a":{"knowledge":[33.895333,1,25.014867,0.9],"language":[53.921489,1,38.335319,0.893617],"retrieval":[61.592727,1,60.870303,1],"tools":[66.75,1,57.391481,1],"arts":[28.301667,1,29.451667,0.833333]},"o":26.573},{"e":"jiwo-4b-v2","a":{"knowledge":[29.071167,1,26.5783,0.9],"language":[52.856383,1,46.237234,0.893617],"retrieval":[51.741818,1,51.536061,1],"tools":[63.141111,1,65.472222,1],"arts":[29.788333,1,27.953333,0.833333]},"o":30.404125},{"e":"cam1","a":{"knowledge":[19.212,1,16.01475,0.9],"language":[33.839787,1,29.105319,0.893617],"retrieval":[36.922121,1,43.092424,1],"tools":[46.325185,1,45.670741,1],"arts":[11.45,1,13.821667,0.833333]},"o":29.024125},{"e":"purplemist-flash-1.0","a":{"knowledge":[34.949833,1,26.204667,0.9],"language":[53.138298,1,45.850426,0.893617],"retrieval":[47.520606,1,58.62697,1],"tools":[65.141111,1,63.59,1],"arts":[25.465,1,28.728333,0.833333]},"o":36.10075},{"e":"sieve-9b-plus","a":{"knowledge":[38.248333,1,28.183463,0.9],"language":[55.942979,1,42.872766,0.893617],"retrieval":[64.170909,1,60.863333,1],"tools":[69.755556,1,66.457037,1],"arts":[33.993333,1,31.391667,0.833333]},"o":33.722625},{"e":"garas","a":{"knowledge":[43.2945,1,39.85914,0.9],"language":[58.70234,1,50.576596,0.893617],"retrieval":[61.848182,1,54.753636,1],"tools":[73.758889,1,76.258148,1],"arts":[43.565,1,43.888333,0.833333]},"o":48.894625},{"e":"darwin-27b-ztc","a":{"knowledge":[44.720167,1,39.238973,0.9],"language":[68.455957,1,46.10617,0.893617],"retrieval":[60.897576,1,65.51303,1],"tools":[86.686667,1,81.302222,1],"arts":[53.341667,1,45.615,0.833333]},"o":54.594125},{"e":"linne-fern-1.0","a":{"knowledge":[44.943833,1,35.818893,0.9],"language":[62.779787,1,49.895532,0.893617],"retrieval":[62.358788,1,62.671818,1],"tools":[78.957407,1,77.33963,1],"arts":[41.531667,1,37.245,0.833333]},"o":50.43875},{"e":"qwen-decision-27b-noncausal","a":{"knowledge":[42.766667,1,33.592703,0.9],"language":[62.339574,1,54.020851,0.893617],"retrieval":[57.812727,1,62.759697,1],"tools":[80.158519,1,79.716296,1],"arts":[43.065,1,38.835,0.833333]},"o":51.959},{"e":"reflex-27b-pmscr","a":{"knowledge":[37.28,1,32.225137,0.9],"language":[62.767021,1,52.734255,0.893617],"retrieval":[64.158182,1,64.62697,1],"tools":[77.205556,1,78.878148,1],"arts":[45.225,1,40.538333,0.833333]},"o":47.601375},{"e":"rsi-jev-v6.1-vl-27b-auto","a":{"knowledge":[51.7205,1,35.320223,0.9],"language":[69.09234,1,51.198936,0.893617],"retrieval":[69.09697,1,60.763636,1],"tools":[81.219259,1,79.647407,1],"arts":[57.303333,1,50.253333,0.833333]},"o":46.267375},{"e":"anyjev-l0-gemma-4-31b","a":{"knowledge":[47.819833,1,40.338053,0.9],"language":[61.670426,1,52.017872,0.893617],"retrieval":[63.509394,1,64.076667,1],"tools":[76.427407,1,76.624815,1],"arts":[48.033333,1,42.785,0.833333]},"o":55.33575},{"e":"decisio-0.6.0-gemma-4-31b","a":{"knowledge":[47.218167,1,40.738547,0.9],"language":[59.81766,1,51.528723,0.893617],"retrieval":[62.74697,1,64.067273,1],"tools":[75.037407,1,76.051111,1],"arts":[41.44,1,38.22,0.833333]},"o":54.147375},{"e":"decider-31b","a":{"knowledge":[50.324167,1,42.43482,0.9],"language":[66.690213,1,53.46383,0.893617],"retrieval":[65.464242,1,65.648485,1],"tools":[84.227778,1,80.602963,1],"arts":[55.186667,1,47.971667,0.833333]},"o":56.884625},{"e":"vllm-sr-d3-v310","a":{"knowledge":[55.258333,1,42.44085,0.9],"language":[68.036383,1,55.070851,0.893617],"retrieval":[66.855152,1,69.055152,1],"tools":[82.569259,1,79.900741,1],"arts":[51.025,1,47.938333,0.833333]},"o":57.864125},{"e":"vllm-sr-d3-flash","a":{"knowledge":[43.297167,1,31.660757,0.9],"language":[60.558085,1,48.818723,0.893617],"retrieval":[64.206061,1,62.956364,1],"tools":[76.008148,1,74.432963,1],"arts":[41.911667,1,37.873333,0.833333]},"o":41.870375},{"e":"vllm-sr-d3-mini","a":{"knowledge":[39.842667,1,30.407857,0.9],"language":[57.800851,1,44.782979,0.893617],"retrieval":[63.587879,1,61.840303,1],"tools":[71.994074,1,70.403704,1],"arts":[37.663333,1,35.803333,0.833333]},"o":36.523125},{"e":"vllm-sr-d3-nano","a":{"knowledge":[25.263,1,20.967773,0.9],"language":[42.168723,1,36.064468,0.893617],"retrieval":[58.829091,1,57.777576,1],"tools":[58.892963,1,58.59,1],"arts":[22.441667,1,24.775,0.833333]},"o":16.396875},{"e":"vllm-sr-d3-lite","a":{"knowledge":[18.567,1,12.873587,0.9],"language":[34.234681,1,29.727021,0.893617],"retrieval":[53.729394,1,56.710606,1],"tools":[56.337778,1,43.823333,1],"arts":[22.26,1,19.91,0.833333]},"o":9.329375},{"e":"vllm-sr-d3-edge","a":{"knowledge":[11.056833,1,7.168243,0.9],"language":[24.148085,1,13.554468,0.893617],"retrieval":[53.632424,1,47.650909,1],"tools":[42.672963,1,33.233704,1],"arts":[11.84,1,16.633333,0.833333]},"o":5.032125},{"e":"wald-26b-a4b","a":{"knowledge":[44.709833,1,37.585833,0.9],"language":[64.247872,1,48.888298,0.893617],"retrieval":[69.052121,1,60.601515,1],"tools":[83.279259,1,75.258889,1],"arts":[50.858333,1,41.908333,0.833333]},"o":41.911625},{"e":"synack-decide-26b-a4b-v7","a":{"knowledge":[43.656333,1,37.631303,0.9],"language":[63.543191,1,49.64234,0.893617],"retrieval":[62.41303,1,60.270303,1],"tools":[79.115556,1,73.166667,1],"arts":[50.131667,1,43.48,0.833333]},"o":43.788375},{"e":"ajev-gemma26-lora1","a":{"knowledge":[41.940167,1,35.692873,0.9],"language":[62.068298,1,50.84766,0.893617],"retrieval":[63.53697,1,60.457576,1],"tools":[73.22,1,66.927778,1],"arts":[46.796667,1,43.68,0.833333]},"o":43.585},{"e":"drex-v2.0","a":{"knowledge":[47.926833,1,32.98544,0.9],"language":[65.265957,1,47.471915,0.893617],"retrieval":[63.022424,1,58.822727,1],"tools":[76.474444,1,72.680741,1],"arts":[53.896667,1,46.365,0.833333]},"o":39.21775},{"e":"decision-4b-v1.6","a":{"knowledge":[26.389,1,25.918883,0.9],"language":[46.332128,1,40.142128,0.893617],"retrieval":[45.256667,1,48.387576,1],"tools":[59.83,1,54.483333,1],"arts":[38.995,1,33.535,0.833333]},"o":26.752},{"e":"sieve-0.8b","a":{"knowledge":[14.319667,1,8.192097,0.9],"language":[26.592979,1,18.769149,0.893617],"retrieval":[54.768182,1,43.231212,1],"tools":[51.22037,1,40.48037,1],"arts":[13.536667,1,11.688333,0.833333]},"o":9.329},{"e":"jiwo-0.8b-v2-r2","a":{"knowledge":[10.870333,1,12.26182,0.9],"language":[36.117872,1,32.131915,0.893617],"retrieval":[44.216667,1,45.25,1],"tools":[40.256296,1,37.433704,1],"arts":[12.226667,1,11.365,0.833333]},"o":13.7085},{"e":"ajev-lora7","a":{"knowledge":[38.270667,1,32.24037,0.9],"language":[62.590638,1,48.362128,0.893617],"retrieval":[62.981212,1,56.12697,1],"tools":[68.442963,1,61.679259,1],"arts":[39.793333,1,36.288333,0.833333]},"o":36.075625},{"e":"bobcat-1.3","a":{"knowledge":[47.994,1,42.264443,0.9],"language":[61.036809,1,52.044894,0.893617],"retrieval":[65.523939,1,65.90303,1],"tools":[76.115556,1,76.632222,1],"arts":[40.903333,1,41.28,0.833333]},"o":56.807375},{"e":"standard-one-8b-sh-v4","a":{"knowledge":[36.852667,1,28.21985,0.9],"language":[57.938723,1,43.019149,0.893617],"retrieval":[66.378485,1,56.996061,1],"tools":[84.011111,1,75.214074,1],"arts":[42.535,1,33.275,0.833333]},"o":24.066125},{"e":"jev-35b","a":{"knowledge":[42.563333,1,33.748597,0.9],"language":[63.163617,1,54.722979,0.893617],"retrieval":[70.958788,1,63.436061,1],"tools":[76.453333,1,71.756296,1],"arts":[42.188333,1,35.621667,0.833333]},"o":46.73425},{"e":"jev","a":{"knowledge":[57.120333,1,42.354977,0.9],"language":[61.181064,1,52.16617,0.893617],"retrieval":[63.530303,1,65.098485,1],"tools":[73.779259,1,74.621852,1],"arts":[39.156667,1,40.181667,0.833333]},"o":48.256875},{"e":"jev-1.13.0","a":{"knowledge":[53.865667,1,39.272577,0.9],"language":[59.224043,1,52.046809,0.893617],"retrieval":[55.424848,1,64.103333,1],"tools":[75.087778,1,73.37037,1],"arts":[39.08,1,41.22,0.833333]},"o":47.719375},{"e":"startlux-decision-v1.1-27b-instant","a":{"knowledge":[60.686,1,45.102897,0.9],"language":[75.560213,1,55.140213,0.893617],"retrieval":[74.797576,1,69.580303,1],"tools":[87.861481,1,84.171111,1],"arts":[66.205,1,50.496667,0.833333]},"o":56.984375},{"e":"pplx-decider-v1.2","a":{"knowledge":[55.293833,1,41.889587,0.9],"language":[71.878511,1,58.827447,0.893617],"retrieval":[68.076364,1,69.605455,1],"tools":[85.441852,1,82.540741,1],"arts":[56.235,1,47.346667,0.833333]},"o":57.759125},{"e":"pplx-decider-flash-v1.2","a":{"knowledge":[41.380333,1,29.440913,0.9],"language":[65.115957,1,55.881064,0.893617],"retrieval":[65.240303,1,62.493636,1],"tools":[81.002593,1,75.871852,1],"arts":[46.616667,1,37.936667,0.833333]},"o":40.0235},{"e":"amos-1-large","a":{"knowledge":[52.358167,1,40.636187,0.9],"language":[72.441064,1,59.163191,0.893617],"retrieval":[66.154242,1,67.227273,1],"tools":[79.233704,1,79.697778,1],"arts":[49.498333,1,44.785,0.833333]},"o":53.308},{"e":"lychee-27b","a":{"knowledge":[51.830667,1,40.457687,0.9],"language":[66.174681,1,56.103617,0.893617],"retrieval":[63.90303,1,70.15,1],"tools":[80.78963,1,81.47037,1],"arts":[46.98,1,42.451667,0.833333]},"o":58.655125},{"e":"span-02","a":{"knowledge":[51.97,1,40.962483,0.9],"language":[65.248936,1,51.623617,0.893617],"retrieval":[65.543636,1,63.49303,1],"tools":[83.910741,1,76.521481,1],"arts":[48.631667,1,45.313333,0.833333]},"o":45.838625},{"e":"drex-max-v1","a":{"knowledge":[57.137667,1,43.45167,0.9],"language":[72.218723,1,55.815957,0.893617],"retrieval":[68.834242,1,67.871515,1],"tools":[81.864074,1,80.501111,1],"arts":[53.401667,1,46.993333,0.833333]},"o":56.44975},{"e":"ajev-lm6q","a":{"knowledge":[53.8295,1,43.982147,0.9],"language":[68.598723,1,56.426596,0.893617],"retrieval":[67.267576,1,67.764545,1],"tools":[81.513333,1,79.051852,1],"arts":[54.423333,1,49.585,0.833333]},"o":58.054625},{"e":"mammoth-1","a":{"knowledge":[67.989,1,47.28983,0.9],"language":[76.114894,1,60.198936,0.893617],"retrieval":[77.370606,1,66.524545,1],"tools":[88.595556,1,83.521111,1],"arts":[64.083333,1,52.521667,0.833333]},"o":56.268625},{"e":"elephant-1-baselabs","a":{"knowledge":[60.6265,1,41.661957,0.9],"language":[73.761489,1,55.815319,0.893617],"retrieval":[76.227273,1,65.424848,1],"tools":[87.963333,1,82.171111,1],"arts":[61.635,1,47.555,0.833333]},"o":44.405625},{"e":"matilda-decision-v1","a":{"knowledge":[51.1405,1,40.986063,0.9],"language":[71.318936,1,52.582553,0.893617],"retrieval":[67.577879,1,64.263636,1],"tools":[84.594815,1,81.127778,1],"arts":[47.365,1,42.316667,0.833333]},"o":56.074875},{"e":"sieve-27b-v2","a":{"knowledge":[56.841667,1,37.63578,0.9],"language":[65.996809,1,54.771915,0.893617],"retrieval":[73.374545,1,59.808788,1],"tools":[85.823333,1,82.971852,1],"arts":[58.928333,1,48.003333,0.833333]},"o":46.6235},{"e":"cerno-27b","a":{"knowledge":[49.034167,1,36.077207,0.9],"language":[71.065532,1,52.082979,0.893617],"retrieval":[65.937879,1,63.239394,1],"tools":[86.731111,1,83.103333,1],"arts":[50.921667,1,43.126667,0.833333]},"o":51.387125},{"e":"jev-brain-gemma-4-31b","a":{"knowledge":[48.607167,1,44.137863,0.9],"language":[65.799149,1,54.840426,0.893617],"retrieval":[67.584545,1,68.429091,1],"tools":[76.655185,1,76.061852,1],"arts":[45.695,1,40.618333,0.833333]},"o":54.29425},{"e":"onejev1.6-27b","a":{"knowledge":[57.4575,1,41.283803,0.9],"language":[71.226383,1,55.347021,0.893617],"retrieval":[73.673939,1,66.791515,1],"tools":[85.455556,1,81.738148,1],"arts":[59.146667,1,48.113333,0.833333]},"o":46.785375},{"e":"onejev1.6-9b","a":{"knowledge":[50.141667,1,34.438873,0.9],"language":[66.565106,1,50.974255,0.893617],"retrieval":[71.739091,1,60.362424,1],"tools":[82.998889,1,77.085926,1],"arts":[55.95,1,46.195,0.833333]},"o":36.66425},{"e":"onejev1.6-4b","a":{"knowledge":[45.0005,1,30.280903,0.9],"language":[64.399787,1,50.754681,0.893617],"retrieval":[68.916364,1,64.310606,1],"tools":[81.499259,1,75.682963,1],"arts":[49.746667,1,42.185,0.833333]},"o":29.55525},{"e":"onejev1.6-0.8b","a":{"knowledge":[26.0895,1,16.573997,0.9],"language":[48.141702,1,33.10766,0.893617],"retrieval":[63.385758,1,52.261818,1],"tools":[72.858148,1,62.976667,1],"arts":[35.638333,1,28.333333,0.833333]},"o":9.279875},{"e":"garum","a":{"knowledge":[44.807833,1,39.67017,0.9],"language":[61.785319,1,54.59234,0.893617],"retrieval":[63.286061,1,60.58,1],"tools":[75.037407,1,76.346667,1],"arts":[44.158333,1,41.933333,0.833333]},"o":48.940375},{"e":"jebadiah-27b-v2.1-t1e","a":{"knowledge":[48.234833,1,39.27346,0.9],"language":[65.464894,1,55.194681,0.893617],"retrieval":[59.943939,1,61.239697,1],"tools":[79.26,1,79.142222,1],"arts":[45.796667,1,40.458333,0.833333]},"o":56.574},{"e":"jebadiah-9b-v2.1-t1e","a":{"knowledge":[35.273333,1,32.433683,0.9],"language":[56.583404,1,44.701915,0.893617],"retrieval":[55.49697,1,56.713636,1],"tools":[70.369259,1,69.716296,1],"arts":[38.101667,1,34.246667,0.833333]},"o":39.468625},{"e":"xor-26b-a4b-v1.2","a":{"knowledge":[45.495,1,41.289763,0.9],"language":[63.218723,1,53.688511,0.893617],"retrieval":[64.840303,1,62.083636,1],"tools":[72.167778,1,72.159259,1],"arts":[51.861667,1,44.56,0.833333]},"o":47.299},{"e":"startlux-decision-v1.1-9b-instant","a":{"knowledge":[53.372167,1,40.33193,0.9],"language":[73.065957,1,54.23617,0.893617],"retrieval":[74.26303,1,64.480909,1],"tools":[88.17,1,80.75,1],"arts":[63.086667,1,48.886667,0.833333]},"o":41.381},{"e":"startlux-decision-v1.1-4b-instant","a":{"knowledge":[50.234167,1,36.91777,0.9],"language":[71.192766,1,50.477447,0.893617],"retrieval":[73.00697,1,64.996364,1],"tools":[86.421852,1,77.116667,1],"arts":[60.025,1,47.961667,0.833333]},"o":38.072625},{"e":"startlux-decision-v1.1-2b-instant","a":{"knowledge":[36.1615,1,24.940747,0.9],"language":[62.387234,1,46.328936,0.893617],"retrieval":[69.714848,1,59.830909,1],"tools":[78.953333,1,71.874444,1],"arts":[53.673333,1,43.77,0.833333]},"o":17.035875},{"e":"startlux-decision-v1.1-0.8b-instant","a":{"knowledge":[28.525333,1,18.171237,0.9],"language":[55.533617,1,32.612979,0.893617],"retrieval":[66.506364,1,48.725152,1],"tools":[74.583704,1,62.032963,1],"arts":[51.103333,1,41.743333,0.833333]},"o":6.864},{"e":"coco-decision-2.4-4b","a":{"knowledge":[35.6315,1,29.918063,0.9],"language":[62.614681,1,50.255745,0.893617],"retrieval":[58.739697,1,51.564848,1],"tools":[77.267778,1,74.172963,1],"arts":[39.108333,1,34.416667,0.833333]},"o":27.80925},{"e":"jegue-0.1","a":{"knowledge":[33.465,1,25.069097,0.9],"language":[62.544043,1,48.377872,0.893617],"retrieval":[65.496364,1,54.930909,1],"tools":[72.308519,1,67.169259,1],"arts":[29.801667,1,31.011667,0.833333]},"o":28.597625},{"e":"nari-decision-4b-v3.2","a":{"knowledge":[28.753167,1,23.018693,0.9],"language":[44.467447,1,35.565957,0.893617],"retrieval":[59.823333,1,56.257576,1],"tools":[62.115926,1,60.922963,1],"arts":[25.391667,1,28.023333,0.833333]},"o":28.576},{"e":"latentmesh-blitz-2b-v1","a":{"knowledge":[22.284833,1,24.54961,0.9],"language":[54.027021,1,34.028936,0.893617],"retrieval":[60.377273,1,51.087879,1],"tools":[66.516296,1,63.167407,1],"arts":[41.768333,1,42.115,0.833333]},"o":14.51825},{"e":"sieve-2b-v3","a":{"knowledge":[30.845833,1,22.297153,0.9],"language":[49.315957,1,40.788085,0.893617],"retrieval":[66.469394,1,50.766061,1],"tools":[80.502963,1,66.957037,1],"arts":[47.638333,1,38.788333,0.833333]},"o":12.614},{"e":"vera-core-2","a":{"knowledge":[21.3645,1,18.083817,0.9],"language":[43.874255,1,33.097234,0.893617],"retrieval":[53.820303,1,53.458788,1],"tools":[61.507407,1,56.074815,1],"arts":[23.935,1,25.973333,0.833333]},"o":12.486625},{"e":"aurelis-2b","a":{"knowledge":[32.242667,1,25.240673,0.9],"language":[54.112979,1,38.633617,0.893617],"retrieval":[54.169697,1,47.209697,1],"tools":[79.472593,1,70.065556,1],"arts":[35.948333,1,28.09,0.833333]},"o":10.096375},{"e":"saina-helm-2-v2","a":{"knowledge":[16.616333,1,14.57118,0.9],"language":[44.046383,1,27.880426,0.893617],"retrieval":[50.537273,1,42.288788,1],"tools":[61.324815,1,55.935185,1],"arts":[22.543333,1,25.611667,0.833333]},"o":7.748125},{"e":"saina-helm-2-omni","a":{"knowledge":[15.307,1,14.368073,0.9],"language":[39.261277,1,32.515106,0.893617],"retrieval":[50.248182,1,43.72697,1],"tools":[59.659259,1,53.539259,1],"arts":[23.711667,1,20.886667,0.833333]},"o":9.59475},{"e":"wyat-1-800m","a":{"knowledge":[11.309,1,9.569773,0.9],"language":[20.430851,1,20.75766,0.893617],"retrieval":[42.972727,1,40.903939,1],"tools":[41.858889,1,33.058148,1],"arts":[13.173333,1,13.693333,0.833333]},"o":11.67925},{"e":"amber-0.4b","a":{"knowledge":[18.908833,1,12.54427,0.9],"language":[48.673191,1,28.284043,0.893617],"retrieval":[64.350909,1,39.868788,1],"tools":[78.932222,1,56.778889,1],"arts":[49.551667,1,43.163333,0.833333]},"o":7.85775},{"e":"crivo-1b","a":{"knowledge":[18.460333,1,15.18272,0.9],"language":[56.259362,1,32.539574,0.893617],"retrieval":[55.822121,1,49.273939,1],"tools":[78.744074,1,61.11037,1],"arts":[28.878333,1,26.645,0.833333]},"o":10.747875},{"e":"crivo-150m-v2","a":{"knowledge":[8.291167,1,5.30581,0.9],"language":[34.526809,1,11.310213,0.893617],"retrieval":[48.234545,1,38.637576,1],"tools":[62.775556,1,49.358148,1],"arts":[21.748333,1,21.288333,0.833333]},"o":5.476},{"e":"crivo-68m","a":{"knowledge":[4.964833,1,3.164357,0.9],"language":[30.832553,1,12.740213,0.893617],"retrieval":[48.800909,1,40.112121,1],"tools":[64.030741,1,50.712222,1],"arts":[20.908333,1,20.87,0.833333]},"o":7.592875},{"e":"crivo-32m-v2","a":{"knowledge":[2.7385,1,1.435167,0.9],"language":[25.01617,1,9.293404,0.893617],"retrieval":[45.567879,1,30.426364,1],"tools":[49.963333,1,34.506296,1],"arts":[19.043333,1,19.788333,0.833333]},"o":5.28825},{"e":"crivo-17m","a":{"knowledge":[1.237833,1,2.093267,0.9],"language":[21.016596,1,9.435106,0.893617],"retrieval":[40.100606,1,25.337273,1],"tools":[36.493704,1,28.74037,1],"arts":[17.011667,1,14.636667,0.833333]},"o":3.207875},{"e":"mola-150m","a":{"knowledge":[11.965,1,7.54381,0.9],"language":[30.218085,1,12.56,0.893617],"retrieval":[51.427879,1,41.052424,1],"tools":[50.088148,1,41.388148,1],"arts":[9.858333,1,9.165,0.833333]},"o":6.32325},{"e":"mola-68m","a":{"knowledge":[7.915167,1,4.43997,0.9],"language":[23.294468,1,11.724894,0.893617],"retrieval":[49.355455,1,44.18303,1],"tools":[47.904815,1,37.639259,1],"arts":[5.666667,1,5.488333,0.833333]},"o":7.736},{"e":"jev-t5-342m-public-v2","a":{"knowledge":[12.3805,1,7.237453,0.9],"language":[29.70234,1,21.825319,0.893617],"retrieval":[35.506061,1,37.073939,1],"tools":[17.592963,1,13.505185,1],"arts":[8.133333,1,9.79,0.833333]},"o":9.895},{"e":"prumo","a":{"knowledge":[16.990167,1,6.96191,0.9],"language":[3.11383,1,4.061489,0.893617],"retrieval":[26.595152,1,21.149697,1],"tools":[45.297407,1,36.553333,1],"arts":[13.408333,1,9.081667,0.833333]},"o":8.076375},{"e":"dinah-1","a":{"knowledge":[23.102333,1,10.282167,0.9],"language":[23.054255,1,13.650213,0.893617],"retrieval":[50.722424,1,27.980606,1],"tools":[62.208519,1,42.551852,1],"arts":[8.595,1,11.663333,0.833333]},"o":4.854625},{"e":"crivo-400m","a":{"knowledge":[11.468667,1,8.977467,0.9],"language":[44.312128,1,19.726596,0.893617],"retrieval":[51.025758,1,41.606364,1],"tools":[69.043704,1,56.388519,1],"arts":[25.365,1,24.936667,0.833333]},"o":9.475125},{"e":"jev-ettin-400m","a":{"knowledge":[8.665667,1,7.842893,0.9],"language":[12.847021,1,10.677447,0.893617],"retrieval":[26.295758,1,28.595758,1],"tools":[14.791852,1,12.405926,1],"arts":[5.798333,1,10.03,0.833333]},"o":11.77375},{"e":"mogan-decision-31b-it","a":{"knowledge":[51.634333,1,40.11563,0.9],"language":[70.044043,1,47.421702,0.893617],"retrieval":[69.594848,1,63.938485,1],"tools":[79.649259,1,73.666296,1],"arts":[56.066667,1,47.331667,0.833333]},"o":49.388},{"e":"jeffan-31b-1710","a":{"knowledge":[63.191667,1,50.696023,0.9],"language":[75.998298,1,54.673404,0.893617],"retrieval":[72.181818,1,69.873636,1],"tools":[89.367037,1,81.298148,1],"arts":[63.6,1,54.323333,0.833333]},"o":59.0555},{"e":"autotrust-jev-27b-v11","a":{"knowledge":[55.195,1,43.742497,0.9],"language":[73.402128,1,55.152128,0.893617],"retrieval":[66.965152,1,72.421212,1],"tools":[85.187407,1,80.254074,1],"arts":[50.986667,1,48.636667,0.833333]},"o":57.957375},{"e":"flexdm-gemma-4-31b","a":{"knowledge":[51.095167,1,45.767977,0.9],"language":[66.38617,1,55.274043,0.893617],"retrieval":[69.398788,1,69.973333,1],"tools":[79.665926,1,76.636667,1],"arts":[48.043333,1,44.33,0.833333]},"o":58.521875},{"e":"visionjev-1.0.0","a":{"knowledge":[55.776833,1,45.705247,0.9],"language":[71.357447,1,52.98766,0.893617],"retrieval":[67.95,1,66.144242,1],"tools":[82.704444,1,82.552963,1],"arts":[48.721667,1,45.59,0.833333]},"o":57.3725},{"e":"nimble-v4-max","a":{"knowledge":[55.358667,1,42.026533,0.9],"language":[70.414681,1,57.344681,0.893617],"retrieval":[71.557576,1,61.86303,1],"tools":[88.590741,1,83.987778,1],"arts":[54.256667,1,47.591667,0.833333]},"o":57.005125},{"e":"aleria-judge-1.1.1","a":{"knowledge":[50.981,1,44.799957,0.9],"language":[67.910851,1,56.148723,0.893617],"retrieval":[66.213333,1,66.607879,1],"tools":[78.962963,1,76.064815,1],"arts":[50.355,1,46.855,0.833333]},"o":57.440375},{"e":"anyjev-tacitus-31b","a":{"knowledge":[52.7475,1,43.667153,0.9],"language":[70.317447,1,57.251064,0.893617],"retrieval":[67.548485,1,66.221515,1],"tools":[81.309259,1,79.138889,1],"arts":[53.205,1,42.918333,0.833333]},"o":56.92925},{"e":"pick-150m-v5","a":{"knowledge":[8.041,1,5.9772,0.9],"language":[35.477447,1,11.301277,0.893617],"retrieval":[45.20697,1,31.652424,1],"tools":[59.885926,1,38.94037,1],"arts":[8.398333,1,4.551667,0.833333]},"o":6.484125},{"e":"vllm-sr-d3-turbo-v11","a":{"knowledge":[58.918,1,45.00039,0.9],"language":[74.475957,1,54.917021,0.893617],"retrieval":[75.817879,1,68.862727,1],"tools":[86.853333,1,83.952593,1],"arts":[60.871667,1,48.275,0.833333]},"o":56.410875},{"e":"cerno-9b-v11","a":{"knowledge":[39.255833,1,29.907433,0.9],"language":[64.320426,1,48.602979,0.893617],"retrieval":[65.101515,1,57.952121,1],"tools":[79.244074,1,70.845556,1],"arts":[42.988333,1,38.461667,0.833333]},"o":30.634875},{"e":"cerno-4b-v11","a":{"knowledge":[34.4565,1,27.157643,0.9],"language":[63.027447,1,41.503191,0.893617],"retrieval":[65.878182,1,51.956667,1],"tools":[82.867778,1,68.446667,1],"arts":[40.623333,1,34.236667,0.833333]},"o":24.1975},{"e":"shisa-de-2.1-repeat","a":{"knowledge":[47.266167,1,40.34583,0.9],"language":[61.932553,1,49.211915,0.893617],"retrieval":[64.791212,1,63.263333,1],"tools":[76.272963,1,71.004815,1],"arts":[40.858333,1,38.621667,0.833333]},"o":48.67375},{"e":"trio-spark-di","a":{"knowledge":[39.192,1,31.09672,0.9],"language":[59.349574,1,41.950851,0.893617],"retrieval":[58.800909,1,58.279091,1],"tools":[76.565926,1,68.951111,1],"arts":[42.806667,1,37.84,0.833333]},"o":29.4185},{"e":"tern-35b-a3b","a":{"knowledge":[43.166167,1,36.989427,0.9],"language":[64.870851,1,55.038298,0.893617],"retrieval":[63.493333,1,59.063636,1],"tools":[76.955185,1,70.708148,1],"arts":[43.038333,1,35.39,0.833333]},"o":44.94775},{"e":"jdi-ptr14-0.8b","a":{"knowledge":[13.5185,1,9.644237,0.9],"language":[40.269149,1,19.870851,0.893617],"retrieval":[52.548788,1,50.993636,1],"tools":[53.021111,1,52.314444,1],"arts":[16.763333,1,16.833333,0.833333]},"o":8.4365},{"e":"glide-v1.2","a":{"knowledge":[60.590167,1,51.890657,0.9],"language":[75.49234,1,58.915106,0.893617],"retrieval":[72.243636,1,73.274242,1],"tools":[88.035926,1,84.983333,1],"arts":[62.593333,1,49.203333,0.833333]},"o":59.6585},{"e":"allec","a":{"knowledge":[49.8185,1,44.669813,0.9],"language":[64.770213,1,55.705957,0.893617],"retrieval":[64.854242,1,65.209394,1],"tools":[77.444074,1,75.122963,1],"arts":[48.415,1,45.773333,0.833333]},"o":57.043875},{"e":"lumma-fev-27b","a":{"knowledge":[42.525667,1,35.76973,0.9],"language":[60.785957,1,51.01766,0.893617],"retrieval":[57.695758,1,59.172121,1],"tools":[78.844444,1,78.962222,1],"arts":[44.778333,1,39.163333,0.833333]},"o":51.01},{"e":"muria","a":{"knowledge":[31.4085,1,33.29608,0.9],"language":[56.62383,1,47.961277,0.893617],"retrieval":[58.903636,1,59.906364,1],"tools":[70.81963,1,71.280741,1],"arts":[39.058333,1,37.273333,0.833333]},"o":39.784},{"e":"mica-v0.5-4b","a":{"knowledge":[38.7845,1,32.099107,0.9],"language":[63.099149,1,47.002979,0.893617],"retrieval":[61.729697,1,57.883333,1],"tools":[80.918148,1,69.421111,1],"arts":[40.753333,1,38.695,0.833333]},"o":31.501375},{"e":"jeffan-4b-2010","a":{"knowledge":[49.540333,1,41.527767,0.9],"language":[69.784043,1,45.718298,0.893617],"retrieval":[62.756364,1,61.350909,1],"tools":[87.403333,1,79.942593,1],"arts":[54.623333,1,45.213333,0.833333]},"o":33.516375},{"e":"tasksource-decider-nano-v5","a":{"knowledge":[9.709167,1,5.587067,0.9],"language":[38.645319,1,19.3,0.893617],"retrieval":[50.12,1,43.020303,1],"tools":[50.95,1,39.176667,1],"arts":[16.638333,1,20.686667,0.833333]},"o":7.24925},{"e":"senova-4b","a":{"knowledge":[38.913,1,26.90909,0.9],"language":[64.97617,1,43.820638,0.893617],"retrieval":[67.038788,1,56.503333,1],"tools":[84.56963,1,70.210741,1],"arts":[58.631667,1,39.698333,0.833333]},"o":33.636125},{"e":"standard-one-3b-sh-v4","a":{"knowledge":[36.262667,1,27.26948,0.9],"language":[57.534255,1,42.069149,0.893617],"retrieval":[66.976667,1,55.027879,1],"tools":[85.743333,1,75.343704,1],"arts":[47.47,1,35.861667,0.833333]},"o":18.51575},{"e":"knowline-35b-a3b-gen1.5","a":{"knowledge":[55.044167,1,42.468267,0.9],"language":[72.915532,1,60.112766,0.893617],"retrieval":[75.387576,1,70.734242,1],"tools":[88.247407,1,79.791481,1],"arts":[64.191667,1,56.365,0.833333]},"o":51.82475},{"e":"knowline-26b-a4b-gen2.5","a":{"knowledge":[55.082,1,43.162743,0.9],"language":[71.187234,1,60.067872,0.893617],"retrieval":[70.912121,1,64.666061,1],"tools":[83.296296,1,76.228519,1],"arts":[61.43,1,48.563333,0.833333]},"o":52.6725},{"e":"knowline-4b-gen5.5","a":{"knowledge":[47.353167,1,38.310213,0.9],"language":[68.17766,1,56.676809,0.893617],"retrieval":[74.042727,1,66.427273,1],"tools":[87.364815,1,77.276296,1],"arts":[60.968333,1,49.328333,0.833333]},"o":37.282875},{"e":"gliclass-v4-edge","a":{"knowledge":[21.4285,1,17.565533,0.9],"language":[54.617447,1,40.412979,0.893617],"retrieval":[66.667879,1,58.118182,1],"tools":[58.118148,1,50.229259,1],"arts":[38.588333,1,33.996667,0.833333]},"o":10.526625},{"e":"vllm-sr-d3-ultra","a":{"knowledge":[62.32,1,48.78309,0.9],"language":[72.348511,1,56.206596,0.893617],"retrieval":[71.763939,1,73.413636,1],"tools":[87.762222,1,84.543704,1],"arts":[63.496667,1,48,0.833333]},"o":59.16325},{"e":"standard-one-35b-sh-v1","a":{"knowledge":[52.150833,1,37.532737,0.9],"language":[66.028298,1,57.470213,0.893617],"retrieval":[72.492424,1,70.034242,1],"tools":[86.091852,1,79.166296,1],"arts":[54.606667,1,43.941667,0.833333]},"o":44.98275},{"e":"crystal-0.8b","a":{"knowledge":[19.066,1,16.078867,0.9],"language":[44.759149,1,36.043617,0.893617],"retrieval":[55.398788,1,48.814848,1],"tools":[64.902222,1,56.407037,1],"arts":[33.368333,1,30.036667,0.833333]},"o":10.285},{"e":"jev-ettin-150m","a":{"knowledge":[14.451,1,9.05906,0.9],"language":[43.181702,1,14.89383,0.893617],"retrieval":[52.142727,1,43.064848,1],"tools":[60.404444,1,42.037778,1],"arts":[9.351667,1,11.696667,0.833333]},"o":5.364875},{"e":"valen-0.8b-r2","a":{"knowledge":[10.1645,1,10.28391,0.9],"language":[24.779362,1,18.485319,0.893617],"retrieval":[20.605758,1,23.389091,1],"tools":[35.638889,1,27.845926,1],"arts":[11.171667,1,10.153333,0.833333]},"o":8.569875},{"e":"valen-2b-r2","a":{"knowledge":[14.84,1,14.735077,0.9],"language":[30.379574,1,25.272553,0.893617],"retrieval":[26.165455,1,19.796061,1],"tools":[45.118148,1,36.122593,1],"arts":[17.748333,1,16.766667,0.833333]},"o":14.234875},{"e":"valen-4b-r2","a":{"knowledge":[27.428833,1,23.309493,0.9],"language":[46.265745,1,40.108298,0.893617],"retrieval":[41.860909,1,43.003939,1],"tools":[52.051852,1,52.995185,1],"arts":[17.376667,1,16.98,0.833333]},"o":27.7345},{"e":"vela3-fast","a":{"knowledge":[21.155833,1,15.202263,0.9],"language":[53.060851,1,31.568936,0.893617],"retrieval":[66.915455,1,52.521818,1],"tools":[73.690741,1,59.507037,1],"arts":[43.076667,1,39.208333,0.833333]},"o":8.963},{"e":"vela3-fast-vision-text","a":{"knowledge":[13.244333,1,10.07521,0.9],"language":[36.585745,1,21.391277,0.893617],"retrieval":[54.925152,1,40.955455,1],"tools":[51.81037,1,45.895926,1],"arts":[41.536667,1,36.166667,0.833333]},"o":6.454},{"e":"sieve-31b-v3","a":{"knowledge":[61.192,1,49.13325,0.9],"language":[71.639362,1,61.692979,0.893617],"retrieval":[76.065455,1,72.903333,1],"tools":[86.92,1,83.863333,1],"arts":[62.251667,1,51.601667,0.833333]},"o":58.85575},{"e":"ainvest-decision-27b-v1.0","a":{"knowledge":[57.907667,1,42.302443,0.9],"language":[74.690426,1,55.098298,0.893617],"retrieval":[75.510303,1,73.853939,1],"tools":[87.061852,1,82.519259,1],"arts":[65.683333,1,50.105,0.833333]},"o":57.911375},{"e":"oak-1.1","a":{"knowledge":[54.417333,1,38.199733,0.9],"language":[70.943617,1,55.057872,0.893617],"retrieval":[71.754242,1,71.809394,1],"tools":[85.72,1,80.630741,1],"arts":[57.473333,1,49.44,0.833333]},"o":59.173125},{"e":"wemm-jev-27b","a":{"knowledge":[58.946333,1,46.329213,0.9],"language":[74.053404,1,56.269787,0.893617],"retrieval":[70.051515,1,69.300606,1],"tools":[86.032222,1,84.878519,1],"arts":[52.42,1,48.97,0.833333]},"o":57.4505},{"e":"knowline-31b-gen1","a":{"knowledge":[59.4265,1,50.004857,0.9],"language":[71.990213,1,60.304894,0.893617],"retrieval":[73.025758,1,68.877273,1],"tools":[84.74037,1,82.355556,1],"arts":[60.493333,1,51.545,0.833333]},"o":62.018375},{"e":"h2o-lightning-31b-v1.1","a":{"knowledge":[56.494,1,44.90021,0.9],"language":[69.638298,1,60.036383,0.893617],"retrieval":[66.908182,1,69.874545,1],"tools":[86.297778,1,83.785185,1],"arts":[59.745,1,45.736667,0.833333]},"o":57.485375},{"e":"oneshot-27b","a":{"knowledge":[52.902833,1,39.931393,0.9],"language":[70.865957,1,57.560426,0.893617],"retrieval":[65.098788,1,65.970303,1],"tools":[81.864815,1,81.24037,1],"arts":[54.158333,1,44.435,0.833333]},"o":55.980875},{"e":"turion-jev-31b","a":{"knowledge":[55.418833,1,47.61741,0.9],"language":[69.913191,1,58.580638,0.893617],"retrieval":[71.172727,1,71.803939,1],"tools":[78.680741,1,77.88,1],"arts":[51.26,1,46.998333,0.833333]},"o":61.502125},{"e":"verdin-4b-v1.1","a":{"knowledge":[43.692833,1,34.178107,0.9],"language":[63.304043,1,44.895106,0.893617],"retrieval":[66.855152,1,54.906061,1],"tools":[80.972593,1,76.347037,1],"arts":[51.131667,1,49.491667,0.833333]},"o":37.8805},{"e":"vera-large","a":{"knowledge":[51.232833,1,39.79395,0.9],"language":[75.708085,1,51.981702,0.893617],"retrieval":[69.531818,1,69.688485,1],"tools":[82.975926,1,80.608148,1],"arts":[58.488333,1,48.345,0.833333]},"o":55.6105},{"e":"amber-hybrid-4b","a":{"knowledge":[36.685,1,26.557037,0.9],"language":[60.126809,1,42.319574,0.893617],"retrieval":[68.892727,1,58.739394,1],"tools":[84.884815,1,69.623704,1],"arts":[55.315,1,45.951667,0.833333]},"o":23.713875},{"e":"jev-style-2b-v4","a":{"knowledge":[23.623333,1,21.800847,0.9],"language":[42.464255,1,35.59617,0.893617],"retrieval":[65.394242,1,52.031515,1],"tools":[61.761481,1,58.774074,1],"arts":[29.013333,1,29.366667,0.833333]},"o":17.594},{"e":"knowline-9b-gen1","a":{"knowledge":[47.611667,1,39.413847,0.9],"language":[69.101064,1,57.234043,0.893617],"retrieval":[71.924545,1,63.178485,1],"tools":[82.719259,1,77.18037,1],"arts":[59.756667,1,48.256667,0.833333]},"o":42.702},{"e":"seb-9b-v2","a":{"knowledge":[39.861167,1,31.001757,0.9],"language":[62.152979,1,48.97,0.893617],"retrieval":[63.455455,1,51.805455,1],"tools":[83.135556,1,75.357037,1],"arts":[48.918333,1,43.361667,0.833333]},"o":27.826625},{"e":"jpt-35b-a3b","a":{"knowledge":[36.4655,1,32.809807,0.9],"language":[61.207447,1,51.498723,0.893617],"retrieval":[55.572727,1,55.502424,1],"tools":[69.212222,1,66.664444,1],"arts":[36.551667,1,34.97,0.833333]},"o":44.430375},{"e":"zev-latest","a":{"knowledge":[0.6715,1,0.432387,0.9],"language":[0.178298,1,2.525106,0.893617],"retrieval":[0.92,1,0.391212,1],"tools":[3.006667,1,3.544444,1],"arts":[5.99,1,1.78,0.833333]},"o":3.761375},{"e":"exaone-4.0-1.2b-jev-v0.3","a":{"knowledge":[14.995333,1,14.44853,0.9],"language":[26.580851,1,20.535957,0.893617],"retrieval":[40.947576,1,31.847576,1],"tools":[56.595185,1,44.812222,1],"arts":[13.86,1,14.218333,0.833333]},"o":7.769125},{"e":"rsi-jev-v5.0-vl-3b","a":{"knowledge":[24.069167,1,20.33039,0.9],"language":[45.029574,1,38.307234,0.893617],"retrieval":[42.978182,1,48.832424,1],"tools":[52.808148,1,49.981852,1],"arts":[19.861667,1,19.055,0.833333]},"o":29.851},{"e":"ezjev-4b-s2","a":{"knowledge":[31.0035,1,26.327733,0.9],"language":[60.490426,1,42.499574,0.893617],"retrieval":[56.318485,1,52.414545,1],"tools":[69.92037,1,62.88963,1],"arts":[31.155,1,31.988333,0.833333]},"o":31.214375},{"e":"bekko-v0-17m","a":{"knowledge":[2.528,1,0.724943,0.9],"language":[2.704255,1,1.495957,0.893617],"retrieval":[21.264848,1,15.355152,1],"tools":[5.908519,1,7.75963,1],"arts":[7.476667,1,4.103333,0.833333]},"o":7.39575},{"e":"bekko-v0-68m","a":{"knowledge":[3.988167,1,1.730647,0.9],"language":[6.365532,1,6.65383,0.893617],"retrieval":[32.188788,1,29.259394,1],"tools":[14.102963,1,12.244074,1],"arts":[13.143333,1,14.751667,0.833333]},"o":8.731625},{"e":"bekko-v0-400m","a":{"knowledge":[11.642667,1,7.931023,0.9],"language":[23.984894,1,18.265319,0.893617],"retrieval":[42.180909,1,38.349697,1],"tools":[18.601481,1,19.061481,1],"arts":[13.903333,1,21.108333,0.833333]},"o":9.05425},{"e":"torchcast-decision-27b","a":{"knowledge":[46.656325,1,35.566497,0.9],"language":[72.039362,1,51.514894,0.893617],"retrieval":[67.542424,1,63.542727,1],"tools":[84.581481,1,79.001111,1],"arts":[54.358333,1,43.876667,0.833333]},"o":52.191},{"e":"kas-4b","a":{"knowledge":[32.216333,1,24.25542,0.9],"language":[35.073404,1,34.191702,0.893617],"retrieval":[40.397576,1,41.340606,1],"tools":[48.862593,1,54.215556,1],"arts":[39.398333,1,26.871667,0.833333]},"o":20.430625},{"e":"lev1-3.7b","a":{"knowledge":[23.066,1,18.81837,0.9],"language":[44.333617,1,37.889149,0.893617],"retrieval":[47.109394,1,44.549394,1],"tools":[68.841481,1,64.01037,1],"arts":[32.411667,1,32.73,0.833333]},"o":19.888875},{"e":"decisio-0.4.0-qwen3.6-35b-a3b","a":{"knowledge":[33.417167,1,28.69028,0.9],"language":[50.081489,1,45.316596,0.893617],"retrieval":[53.557273,1,54.412424,1],"tools":[68.431481,1,67.211852,1],"arts":[36.133333,1,33.995,0.833333]},"o":41.6275},{"e":"decisio-0.4.0-gemma-4-12b","a":{"knowledge":[30.7565,1,27.591583,0.9],"language":[52.381915,1,39.775532,0.893617],"retrieval":[57.999697,1,56.876667,1],"tools":[71.61963,1,70.938519,1],"arts":[35.298333,1,32.755,0.833333]},"o":38.307125},{"e":"hopper-12b-trained","a":{"knowledge":[31.696,1,25.701097,0.9],"language":[53.784468,1,42.527021,0.893617],"retrieval":[53.171212,1,53.337273,1],"tools":[66.258519,1,66.774074,1],"arts":[34.165,1,31.601667,0.833333]},"o":34.052875},{"e":"dm-jepa-1.1","a":{"knowledge":[17.273667,1,1.282447,0.9],"language":[21.693617,1,3.245106,0.893617],"retrieval":[33.99697,1,5.271515,1],"tools":[7.893333,1,6.925185,1],"arts":[8.636667,1,3.875,0.833333]},"o":4.042375},{"e":"jevstral-8b","a":{"knowledge":[24.669333,1,20.9897,0.9],"language":[33.635957,1,33.844255,0.893617],"retrieval":[39.244545,1,37.222121,1],"tools":[52.927778,1,57.989259,1],"arts":[14.088333,1,15.408333,0.833333]},"o":19.77475},{"e":"deck31b","a":{"knowledge":[46.66525,1,40.76595,0.9],"language":[60.018085,1,51.655745,0.893617],"retrieval":[62.759394,1,65.048788,1],"tools":[75.57963,1,75.741481,1],"arts":[42.285,1,37.393333,0.833333]},"o":54.771625},{"e":"quyet-1.0-large","a":{"knowledge":[49.696975,1,42.766543,0.9],"language":[66.961702,1,52.252128,0.893617],"retrieval":[65.836061,1,63.52697,1],"tools":[72.552593,1,74.600741,1],"arts":[41.866667,1,37.981667,0.833333]},"o":49.86225}]} \ No newline at end of file diff --git a/data/v03.json b/data/v03.json deleted file mode 100644 index 93e4332aeee4b239ddcb4609bf928416de3fda2f..0000000000000000000000000000000000000000 --- a/data/v03.json +++ /dev/null @@ -1 +0,0 @@ -{"generated_utc":"2026-10-28T19:32:17Z","edition":"0.4","label":"Decision Index 0.4.1","weights":{"public":0,"same_skills":0.6,"new_domains":0.4},"public_index_note":"GSM8K rebuilt, ForecastBench retired","vision":{"entrants":61},"models":[{"engine":"decider-chat-gemma4-31b","pub":57.79,"same_skills":56.88,"new_domains":54.01,"v":61.63,"lab":false,"jev":false,"rank_v":41,"rank_pub":81,"name":"Decider chat · Gemma-4-31B","eq":{"S":56.48,"O":69.35},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":33.89,"new_domains":27.74}},{"engine":"rune-26b-a4b-v3","pub":58.28,"same_skills":57.09,"new_domains":43.2,"v":56.42,"lab":false,"jev":false,"rank_v":70,"rank_pub":73,"name":"Surogate Rune 26B-A4B v3","eq":{"S":56.7,"O":56},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":34.02,"new_domains":22.4}},{"engine":"decision-2.0-vega-27b","pub":56.97,"same_skills":55.71,"new_domains":48.12,"v":58.01,"lab":false,"jev":false,"rank_v":65,"rank_pub":91,"name":"vLLM-SR Decision 2.0 Vega 27B","eq":{"S":55.3,"O":62.07},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":33.18,"new_domains":24.83}},{"engine":"kev-27b","pub":56.69,"same_skills":55.51,"new_domains":51.51,"v":59.56,"lab":false,"jev":false,"rank_v":54,"rank_pub":92,"name":"Kev 27B","eq":{"S":55.1,"O":66.26},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":33.06,"new_domains":26.5}},{"engine":"blink-v0.3-26b-a4b-nvfp4","pub":58.21,"same_skills":57.21,"new_domains":42.05,"v":55.92,"lab":false,"jev":false,"rank_v":73,"rank_pub":74,"name":"Blink v0.3 26B-A4B (NVFP4)","eq":{"S":56.81,"O":54.57},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":34.09,"new_domains":21.83}},{"engine":"simple-jev-qwen3.8-27b","pub":56.05,"same_skills":55.46,"new_domains":46.36,"v":56.99,"lab":false,"jev":false,"rank_v":68,"rank_pub":96,"name":"simple-jev · Qwen3.8-27B (featherless)","eq":{"S":55.05,"O":59.9},"rank_tie":14,"tied":true,"parts":{"public":0,"same_skills":33.03,"new_domains":23.96}},{"engine":"jev-27b","pub":53.64,"same_skills":54.22,"new_domains":48.61,"v":57.35,"lab":false,"jev":false,"rank_v":67,"rank_pub":103,"name":"JEV-27B","eq":{"S":53.79,"O":62.67},"rank_tie":14,"tied":true,"parts":{"public":0,"same_skills":32.28,"new_domains":25.07}},{"engine":"eikos-27b-fp8","pub":52.85,"same_skills":55.92,"new_domains":49.43,"v":58.79,"lab":false,"jev":false,"rank_v":60,"rank_pub":107,"name":"Eikos-27B-FP8 Qwen3.8-27B LoRA","eq":{"S":55.52,"O":63.69},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":33.31,"new_domains":25.48}},{"engine":"jade","pub":53.42,"same_skills":54.29,"new_domains":44.92,"v":55.57,"lab":false,"jev":false,"rank_v":74,"rank_pub":105,"name":"JADE","eq":{"S":53.86,"O":58.12},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":32.32,"new_domains":23.25}},{"engine":"jev-gemma4-26b-a4b","pub":58.03,"same_skills":54.99,"new_domains":39.86,"v":53.5,"lab":false,"jev":false,"rank_v":84,"rank_pub":79,"name":"GEV-26B-Decide","eq":{"S":54.58,"O":51.88},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":32.75,"new_domains":20.75}},{"engine":"xor-1.2","pub":52.28,"same_skills":53.6,"new_domains":41.82,"v":53.62,"lab":false,"jev":false,"rank_v":83,"rank_pub":109,"name":"Xor 1.2","eq":{"S":53.17,"O":54.29},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":31.9,"new_domains":21.71}},{"engine":"decider-chat-qwen3.6-27b","pub":51.57,"same_skills":52.15,"new_domains":45.72,"v":54.67,"lab":false,"jev":false,"rank_v":79,"rank_pub":110,"name":"Decider chat · Qwen3.6-27B","eq":{"S":51.7,"O":59.11},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":31.02,"new_domains":23.65}},{"engine":"clef","pub":61.71,"same_skills":53.11,"new_domains":45.06,"v":54.92,"lab":false,"jev":false,"rank_v":77,"rank_pub":50,"name":"Cloudflare clef","eq":{"S":52.67,"O":58.3},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":31.6,"new_domains":23.32}},{"engine":"joshua-diffusion-full","pub":50.14,"same_skills":50.02,"new_domains":37.94,"v":49.53,"lab":false,"jev":false,"rank_v":94,"rank_pub":118,"name":"JoshuaSP diffgemma","eq":{"S":49.54,"O":49.5},"rank_tie":21,"tied":true,"parts":{"public":0,"same_skills":29.72,"new_domains":19.8}},{"engine":"winnow-12b-q8","pub":50.66,"same_skills":48.14,"new_domains":34.26,"v":46.57,"lab":false,"jev":false,"rank_v":104,"rank_pub":116,"name":"Winnow-12B","eq":{"S":47.64,"O":44.95},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":28.59,"new_domains":17.98}},{"engine":"jevfire-uncapped","pub":49.8,"same_skills":46.86,"new_domains":31.43,"v":44.39,"lab":false,"jev":false,"rank_v":114,"rank_pub":120,"name":"Jevfire Qwen3.8-27B","eq":{"S":46.35,"O":41.46},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":27.81,"new_domains":16.58}},{"engine":"decision-2.0-lux-9b","pub":46.76,"same_skills":47.35,"new_domains":34.02,"v":45.97,"lab":false,"jev":false,"rank_v":106,"rank_pub":133,"name":"vLLM-SR Decision 2.0 Lux 9B","eq":{"S":46.85,"O":44.66},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":28.11,"new_domains":17.87}},{"engine":"nimble-v3","pub":57.19,"same_skills":52.34,"new_domains":38.2,"v":51.06,"lab":false,"jev":false,"rank_v":90,"rank_pub":89,"name":"Bespoke Nimble 9B v3","eq":{"S":51.89,"O":49.82},"rank_tie":20,"tied":true,"parts":{"public":0,"same_skills":31.13,"new_domains":19.93}},{"engine":"jevk5-9b-v0.3.3","pub":47.62,"same_skills":48.19,"new_domains":30.79,"v":44.89,"lab":false,"jev":false,"rank_v":112,"rank_pub":130,"name":"JevK5 9B v0.3.3","eq":{"S":47.69,"O":40.67},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":28.62,"new_domains":16.27}},{"engine":"jpt-9b","pub":46.96,"same_skills":43.45,"new_domains":34.98,"v":44.08,"lab":false,"jev":false,"rank_v":118,"rank_pub":132,"name":"JPT-9B Qwen3.5-9B LoRA","eq":{"S":42.9,"O":45.85},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":25.74,"new_domains":18.34}},{"engine":"jev-9b","pub":43.28,"same_skills":42.83,"new_domains":31.85,"v":42.15,"lab":false,"jev":false,"rank_v":124,"rank_pub":140,"name":"JEV-9B","eq":{"S":42.27,"O":41.98},"rank_tie":28,"tied":true,"parts":{"public":0,"same_skills":25.36,"new_domains":16.79}},{"engine":"sieve-9b","pub":41.88,"same_skills":42.85,"new_domains":30.96,"v":41.73,"lab":false,"jev":false,"rank_v":126,"rank_pub":146,"name":"Sieve 9B","eq":{"S":42.3,"O":40.87},"rank_tie":28,"tied":true,"parts":{"public":0,"same_skills":25.38,"new_domains":16.35}},{"engine":"kev-9b-v2","pub":41.54,"same_skills":42.37,"new_domains":29.9,"v":40.91,"lab":false,"jev":false,"rank_v":129,"rank_pub":147,"name":"Kev 9B v2","eq":{"S":41.81,"O":39.57},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.08,"new_domains":15.83}},{"engine":"jev-omni","pub":41.04,"same_skills":42.18,"new_domains":36.04,"v":43.83,"lab":false,"jev":false,"rank_v":120,"rank_pub":152,"name":"Jev-Omni","eq":{"S":41.62,"O":47.16},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":24.97,"new_domains":18.86}},{"engine":"decision-2.0-nox-4b","pub":44.21,"same_skills":43.72,"new_domains":27.91,"v":40.75,"lab":false,"jev":false,"rank_v":132,"rank_pub":138,"name":"vLLM-SR Decision 2.0 Nox 4B","eq":{"S":43.17,"O":37.12},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.9,"new_domains":14.85}},{"engine":"decider-35b-nvfp4","pub":47.7,"same_skills":45.12,"new_domains":38.12,"v":46.64,"lab":false,"jev":false,"rank_v":103,"rank_pub":129,"name":"Decider 35B-A3B","eq":{"S":44.59,"O":49.72},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":26.76,"new_domains":19.89}},{"engine":"djev","pub":40.89,"same_skills":41.63,"new_domains":25.22,"v":38.15,"lab":false,"jev":false,"rank_v":140,"rank_pub":153,"name":"djev diffgemma","eq":{"S":41.06,"O":33.79},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":24.64,"new_domains":13.52}},{"engine":"jpt-4b","pub":42.82,"same_skills":40.11,"new_domains":29.93,"v":39.56,"lab":false,"jev":false,"rank_v":137,"rank_pub":141,"name":"JPT-4B Qwen3.5-4B LoRA","eq":{"S":39.52,"O":39.61},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":23.71,"new_domains":15.84}},{"engine":"nimble-v2","pub":39.9,"same_skills":39.51,"new_domains":35.74,"v":42.06,"lab":false,"jev":false,"rank_v":125,"rank_pub":155,"name":"Bespoke Nimble 9B v2","eq":{"S":38.91,"O":46.78},"rank_tie":28,"tied":true,"parts":{"public":0,"same_skills":23.35,"new_domains":18.71}},{"engine":"hopper-g-1.2","pub":41.15,"same_skills":42.37,"new_domains":28.39,"v":40.17,"lab":false,"jev":false,"rank_v":133,"rank_pub":150,"name":"Hopper (G) 1.2 Qwen3.5-4B LoRA","eq":{"S":41.8,"O":37.71},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":25.08,"new_domains":15.08}},{"engine":"razorback-one-read-3e296f08","pub":37.75,"same_skills":39.74,"new_domains":26.43,"v":37.6,"lab":false,"jev":false,"rank_v":142,"rank_pub":166,"name":"razorback16 diffgemma","eq":{"S":39.15,"O":35.28},"rank_tie":32,"tied":true,"parts":{"public":0,"same_skills":23.49,"new_domains":14.11}},{"engine":"kev-4b-r10","pub":37.95,"same_skills":36.17,"new_domains":25.4,"v":34.93,"lab":false,"jev":false,"rank_v":148,"rank_pub":165,"name":"Kev 4B r10","eq":{"S":35.54,"O":34.01},"rank_tie":34,"tied":true,"parts":{"public":0,"same_skills":21.32,"new_domains":13.6}},{"engine":"lev","pub":39.2,"same_skills":37.2,"new_domains":25.23,"v":35.47,"lab":false,"jev":false,"rank_v":146,"rank_pub":158,"name":"Interfaze lev Qwen3.5-4B LoRA","eq":{"S":36.58,"O":33.8},"rank_tie":34,"tied":true,"parts":{"public":0,"same_skills":21.95,"new_domains":13.52}},{"engine":"solomon-v11-bf16-encoding","pub":38.07,"same_skills":36.95,"new_domains":41.25,"v":43.23,"lab":false,"jev":false,"rank_v":121,"rank_pub":162,"name":"Solomon v1.1","eq":{"S":36.32,"O":53.59},"rank_tie":27,"tied":true,"parts":{"public":0,"same_skills":21.79,"new_domains":21.44}},{"engine":"jevk5-v0.3","pub":41.14,"same_skills":41.75,"new_domains":21.91,"v":36.59,"lab":false,"jev":false,"rank_v":144,"rank_pub":151,"name":"JevK5 v0.3","eq":{"S":41.18,"O":29.71},"rank_tie":33,"tied":true,"parts":{"public":0,"same_skills":24.71,"new_domains":11.88}},{"engine":"neohorse-jev-4b","pub":37,"same_skills":36.9,"new_domains":29.23,"v":37.26,"lab":false,"jev":false,"rank_v":143,"rank_pub":171,"name":"TokenRhythm NeoHorse-Jev-4B","eq":{"S":36.27,"O":38.74},"rank_tie":32,"tied":true,"parts":{"public":0,"same_skills":21.76,"new_domains":15.5}},{"engine":"ryotide-qwen-v0.3.1","pub":37.28,"same_skills":36.67,"new_domains":18.83,"v":31.99,"lab":false,"jev":false,"rank_v":162,"rank_pub":170,"name":"Ryotide Qwen v0.3.1","eq":{"S":36.04,"O":25.9},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":21.62,"new_domains":10.36}},{"engine":"intern-decision-4b","pub":38,"same_skills":39.1,"new_domains":29.41,"v":38.69,"lab":false,"jev":false,"rank_v":138,"rank_pub":163,"name":"InternLM Intern-Decision Qwen3.5-4B","eq":{"S":38.5,"O":38.97},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":23.1,"new_domains":15.59}},{"engine":"winnow-e4b-q8","pub":40.86,"same_skills":36.79,"new_domains":20.16,"v":32.72,"lab":false,"jev":false,"rank_v":158,"rank_pub":154,"name":"Winnow-E4B","eq":{"S":36.17,"O":27.54},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":21.7,"new_domains":11.02}},{"engine":"clef-flash","pub":56.15,"same_skills":44.01,"new_domains":35,"v":44.43,"lab":false,"jev":false,"rank_v":113,"rank_pub":95,"name":"Cloudflare clef-flash","eq":{"S":43.46,"O":45.87},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":26.08,"new_domains":18.35}},{"engine":"decider-4b","pub":41.19,"same_skills":37.19,"new_domains":26.93,"v":36.3,"lab":false,"jev":false,"rank_v":145,"rank_pub":149,"name":"Decider 4B","eq":{"S":36.57,"O":35.91},"rank_tie":33,"tied":true,"parts":{"public":0,"same_skills":21.94,"new_domains":14.36}},{"engine":"vllm-pr57250","pub":32.69,"same_skills":33.44,"new_domains":27.38,"v":34.25,"lab":false,"jev":false,"rank_v":155,"rank_pub":180,"name":"mmastrac diffgemma","eq":{"S":32.78,"O":36.47},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":19.67,"new_domains":14.59}},{"engine":"intelif-qwen3-4b","pub":32.47,"same_skills":35.92,"new_domains":18.43,"v":31.33,"lab":false,"jev":false,"rank_v":163,"rank_pub":181,"name":"Intelif Qwen3-4B","eq":{"S":35.28,"O":25.41},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":21.17,"new_domains":10.16}},{"engine":"jobe","pub":32.93,"same_skills":34.34,"new_domains":18.39,"v":30.35,"lab":false,"jev":false,"rank_v":168,"rank_pub":178,"name":"Jobe Qwen3.5-4B","eq":{"S":33.68,"O":25.36},"rank_tie":38,"tied":true,"parts":{"public":0,"same_skills":20.21,"new_domains":10.14}},{"engine":"gevva-e4b","pub":32.44,"same_skills":31.03,"new_domains":20.15,"v":29.21,"lab":false,"jev":false,"rank_v":173,"rank_pub":182,"name":"Gevva E4B","eq":{"S":30.34,"O":27.53},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":18.2,"new_domains":11.01}},{"engine":"pngwn-space-uncapped-full","pub":30.49,"same_skills":30.03,"new_domains":25.19,"v":31.1,"lab":false,"jev":false,"rank_v":166,"rank_pub":188,"name":"open-jev (pngwn)","eq":{"S":29.33,"O":33.75},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":17.6,"new_domains":13.5}},{"engine":"tev1-4b","pub":29.51,"same_skills":31.78,"new_domains":28.37,"v":33.73,"lab":false,"jev":false,"rank_v":157,"rank_pub":192,"name":"Together Tev1 Qwen3.5-4B","eq":{"S":31.09,"O":37.68},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":18.66,"new_domains":15.07}},{"engine":"openvons","pub":28.66,"same_skills":32.13,"new_domains":17.92,"v":28.78,"lab":false,"jev":false,"rank_v":175,"rank_pub":195,"name":"openvons Qwen3-4B","eq":{"S":31.45,"O":24.78},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":18.87,"new_domains":9.91}},{"engine":"gevva-e2b","pub":28.53,"same_skills":27.78,"new_domains":14.11,"v":24.26,"lab":false,"jev":false,"rank_v":183,"rank_pub":197,"name":"Gevva E2B","eq":{"S":27.05,"O":20.07},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":16.23,"new_domains":8.03}},{"engine":"metask-jev-4b","pub":27.58,"same_skills":27.82,"new_domains":30.08,"v":32.17,"lab":false,"jev":false,"rank_v":161,"rank_pub":198,"name":"Metask-Jev-4B","eq":{"S":27.09,"O":39.79},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":16.25,"new_domains":15.92}},{"engine":"semif","pub":26.5,"same_skills":28.04,"new_domains":18.74,"v":26.71,"lab":false,"jev":false,"rank_v":176,"rank_pub":202,"name":"SemIf Qwen3.5-4B Base","eq":{"S":27.32,"O":25.79},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":16.39,"new_domains":10.32}},{"engine":"mini-jev","pub":21.25,"same_skills":23.36,"new_domains":16.61,"v":22.81,"lab":false,"jev":false,"rank_v":188,"rank_pub":210,"name":"mini-jev Qwen3-4B","eq":{"S":22.58,"O":23.16},"rank_tie":43,"tied":true,"parts":{"public":0,"same_skills":13.55,"new_domains":9.27}},{"engine":"sieve-2b","pub":22.16,"same_skills":23.66,"new_domains":14.41,"v":21.9,"lab":false,"jev":false,"rank_v":191,"rank_pub":207,"name":"Sieve 2B","eq":{"S":22.88,"O":20.44},"rank_tie":44,"tied":true,"parts":{"public":0,"same_skills":13.73,"new_domains":8.18}},{"engine":"decider-2b-fp8-http","pub":29.75,"same_skills":28.47,"new_domains":12.04,"v":23.65,"lab":false,"jev":false,"rank_v":185,"rank_pub":190,"name":"Decider 2B","eq":{"S":27.75,"O":17.51},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":16.65,"new_domains":7}},{"engine":"decision-2.0-sol-2b","pub":30.05,"same_skills":29.25,"new_domains":11.06,"v":23.64,"lab":false,"jev":false,"rank_v":186,"rank_pub":189,"name":"vLLM-SR Decision 2.0 Sol 2B","eq":{"S":28.54,"O":16.3},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":17.12,"new_domains":6.52}},{"engine":"intern-decision-2b","pub":19.26,"same_skills":23.28,"new_domains":12.91,"v":20.93,"lab":false,"jev":false,"rank_v":193,"rank_pub":217,"name":"InternLM Intern-Decision Qwen3.5-2B","eq":{"S":22.5,"O":18.58},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":13.5,"new_domains":7.43}},{"engine":"jpt-0.8b","pub":19.37,"same_skills":17.82,"new_domains":8.55,"v":15.46,"lab":false,"jev":false,"rank_v":214,"rank_pub":216,"name":"JPT-0.8B Qwen3.5-0.8B LoRA","eq":{"S":16.97,"O":13.2},"rank_tie":50,"tied":true,"parts":{"public":0,"same_skills":10.18,"new_domains":5.28}},{"engine":"this-that-1.2","pub":28.65,"same_skills":27.51,"new_domains":15.65,"v":24.85,"lab":false,"jev":false,"rank_v":182,"rank_pub":196,"name":"FLock this-that 1.2","eq":{"S":26.78,"O":21.97},"rank_tie":41,"tied":true,"parts":{"public":0,"same_skills":16.07,"new_domains":8.79}},{"engine":"bosun-v3.1-1.7b","pub":20.55,"same_skills":20.41,"new_domains":7.54,"v":16.54,"lab":false,"jev":false,"rank_v":209,"rank_pub":213,"name":"Bosun v3.1 1.7B LoRA","eq":{"S":19.6,"O":11.95},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":11.76,"new_domains":4.78}},{"engine":"kodiak-v0.2-1b-accuracy","pub":19.05,"same_skills":16.33,"new_domains":7.13,"v":13.86,"lab":false,"jev":false,"rank_v":219,"rank_pub":219,"name":"Kodiak v0.2 1B (accuracy)","eq":{"S":15.47,"O":11.45},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":9.28,"new_domains":4.58}},{"engine":"decision-2.0-kai-0.6b","pub":16.68,"same_skills":16.9,"new_domains":11.63,"v":16.43,"lab":false,"jev":false,"rank_v":210,"rank_pub":223,"name":"vLLM-SR Decision 2.0 Kai 0.6B","eq":{"S":16.05,"O":17.01},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":9.63,"new_domains":6.81}},{"engine":"decision-2.0-eos-0.8b","pub":20.57,"same_skills":21.31,"new_domains":10.16,"v":18.37,"lab":false,"jev":false,"rank_v":200,"rank_pub":212,"name":"vLLM-SR Decision 2.0 Eos 0.8B","eq":{"S":20.5,"O":15.19},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":12.3,"new_domains":6.07}},{"engine":"tev1-0.8b","pub":13.1,"same_skills":12.84,"new_domains":8.6,"v":12.47,"lab":false,"jev":false,"rank_v":222,"rank_pub":227,"name":"Together Tev1 Qwen3.5-0.8B","eq":{"S":11.94,"O":13.27},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":7.16,"new_domains":5.31}},{"engine":"intern-decision-0.8b","pub":11.81,"same_skills":12.93,"new_domains":8.97,"v":12.71,"lab":false,"jev":false,"rank_v":220,"rank_pub":229,"name":"InternLM Intern-Decision Qwen3.5-0.8B","eq":{"S":12.03,"O":13.73},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":7.22,"new_domains":5.49}},{"engine":"kev-0.8b-r15","pub":16.78,"same_skills":15.84,"new_domains":12.67,"v":16.3,"lab":false,"jev":false,"rank_v":212,"rank_pub":222,"name":"Kev 0.8B r15","eq":{"S":14.97,"O":18.29},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":8.98,"new_domains":7.32}},{"engine":"von-1.3","pub":13.86,"same_skills":12.8,"new_domains":6.99,"v":11.65,"lab":false,"jev":false,"rank_v":226,"rank_pub":226,"name":"Von 1.3","eq":{"S":11.9,"O":11.28},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":7.14,"new_domains":4.51}},{"engine":"jevk5-lite","pub":10.28,"same_skills":10.37,"new_domains":4.18,"v":8.78,"lab":false,"jev":false,"rank_v":232,"rank_pub":231,"name":"JevK5 Lite","eq":{"S":9.44,"O":7.81},"rank_tie":55,"tied":true,"parts":{"public":0,"same_skills":5.66,"new_domains":3.12}},{"engine":"gliner-decide","pub":11.48,"same_skills":12.05,"new_domains":4.86,"v":10.14,"lab":false,"jev":false,"rank_v":230,"rank_pub":230,"name":"GLiNER2.5-Decide","eq":{"S":11.14,"O":8.64},"rank_tie":54,"tied":false,"parts":{"public":0,"same_skills":6.69,"new_domains":3.46}},{"engine":"bosun-v3.1-0.6b","pub":14.5,"same_skills":14.09,"new_domains":4.18,"v":11.04,"lab":false,"jev":false,"rank_v":227,"rank_pub":224,"name":"Bosun v3.1 0.6B LoRA","eq":{"S":13.21,"O":7.8},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":7.92,"new_domains":3.12}},{"engine":"jev-style-0.8b-v3","pub":17,"same_skills":14.65,"new_domains":6.44,"v":12.5,"lab":false,"jev":false,"rank_v":221,"rank_pub":221,"name":"Jev-Style 0.8B v3","eq":{"S":13.77,"O":10.6},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":8.26,"new_domains":4.24}},{"engine":"lavoir","pub":8.85,"same_skills":9.9,"new_domains":8.94,"v":10.85,"lab":false,"jev":false,"rank_v":229,"rank_pub":232,"name":"MoganAI Lavoir ModernBERT-large","eq":{"S":8.96,"O":13.69},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":5.38,"new_domains":5.47}},{"engine":"jeff-uncapped","pub":8.27,"same_skills":6.86,"new_domains":3.29,"v":6.21,"lab":false,"jev":false,"rank_v":237,"rank_pub":233,"name":"jeff GLiFormer-large","eq":{"S":5.88,"O":6.7},"rank_tie":57,"tied":true,"parts":{"public":0,"same_skills":3.53,"new_domains":2.68}},{"engine":"lfm2600","pub":6.78,"same_skills":7.65,"new_domains":6.31,"v":8.18,"lab":false,"jev":false,"rank_v":233,"rank_pub":236,"name":"LFM2.5-2.6B-RLCD","eq":{"S":6.68,"O":10.43},"rank_tie":56,"tied":true,"parts":{"public":0,"same_skills":4.01,"new_domains":4.17}},{"engine":"sifr-0.8b-v3.1","pub":25.65,"same_skills":18.93,"new_domains":5.31,"v":14.53,"lab":false,"jev":false,"rank_v":216,"rank_pub":204,"name":"Sifr 0.8B v3.1","eq":{"S":18.09,"O":9.2},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":10.86,"new_domains":3.68}},{"engine":"laya-1e28ac20","pub":6.19,"same_skills":9.08,"new_domains":6.63,"v":9.21,"lab":false,"jev":false,"rank_v":231,"rank_pub":237,"name":"Laya","eq":{"S":8.13,"O":10.84},"rank_tie":55,"tied":true,"parts":{"public":0,"same_skills":4.88,"new_domains":4.33}},{"engine":"clm-v0.1-8b","pub":7.41,"same_skills":5.96,"new_domains":4.94,"v":6.49,"lab":false,"jev":false,"rank_v":236,"rank_pub":235,"name":"CLM-v0.1-8B Qwen3-8B heads","eq":{"S":4.98,"O":8.75},"rank_tie":57,"tied":true,"parts":{"public":0,"same_skills":2.99,"new_domains":3.5}},{"engine":"harsha","pub":4.02,"same_skills":6.21,"new_domains":7.4,"v":7.85,"lab":false,"jev":false,"rank_v":234,"rank_pub":240,"name":"Qwen-2.5-1B-RLCD","eq":{"S":5.23,"O":11.78},"rank_tie":56,"tied":true,"parts":{"public":0,"same_skills":3.14,"new_domains":4.71}},{"engine":"akash-gemma","pub":5.19,"same_skills":5.03,"new_domains":3.52,"v":5.22,"lab":false,"jev":false,"rank_v":239,"rank_pub":238,"name":"system-one-gemma","eq":{"S":4.03,"O":7},"rank_tie":58,"tied":true,"parts":{"public":0,"same_skills":2.42,"new_domains":2.8}},{"engine":"julia-1","pub":4.24,"same_skills":4.54,"new_domains":3.18,"v":4.75,"lab":false,"jev":false,"rank_v":241,"rank_pub":239,"name":"Supersonic Labs Julia 1","eq":{"S":3.54,"O":6.58},"rank_tie":58,"tied":true,"parts":{"public":0,"same_skills":2.12,"new_domains":2.63}},{"engine":"verdict-8af2496e","pub":1.86,"same_skills":2.41,"new_domains":0.32,"v":2.04,"lab":false,"jev":false,"rank_v":244,"rank_pub":241,"name":"Verdict","eq":{"S":1.38,"O":3.04},"rank_tie":60,"tied":false,"parts":{"public":0,"same_skills":0.83,"new_domains":1.22}},{"engine":"lfm350-sdpa","pub":1.43,"same_skills":2.29,"new_domains":8.18,"v":5.86,"lab":false,"jev":false,"rank_v":238,"rank_pub":244,"name":"LFM2.5-350M-RLCD","eq":{"S":1.27,"O":12.75},"rank_tie":57,"tied":true,"parts":{"public":0,"same_skills":0.76,"new_domains":5.1}},{"engine":"strands-decider-2b","pub":25.41,"same_skills":22.43,"new_domains":13.52,"v":20.72,"lab":false,"jev":false,"rank_v":195,"rank_pub":205,"name":"Strands Decider 2B","eq":{"S":21.64,"O":19.34},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":12.98,"new_domains":7.74}},{"engine":"d1-3b","pub":48.99,"same_skills":41.23,"new_domains":11.47,"v":31.12,"lab":false,"jev":false,"rank_v":165,"rank_pub":123,"name":"LiquidAI d1-3B","eq":{"S":40.65,"O":16.81},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":24.39,"new_domains":6.72}},{"engine":"d1-omni-600m","pub":17.9,"same_skills":12.86,"new_domains":7.13,"v":11.75,"lab":false,"jev":false,"rank_v":224,"rank_pub":220,"name":"LiquidAI d1-omni-600M","eq":{"S":11.96,"O":11.45},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":7.17,"new_domains":4.58}},{"engine":"clef-omni","pub":56.54,"same_skills":46.51,"new_domains":33.17,"v":45.04,"lab":false,"jev":false,"rank_v":111,"rank_pub":94,"name":"Cloudflare clef-omni (30B-A3B)","eq":{"S":45.99,"O":43.61},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":27.6,"new_domains":17.44}},{"engine":"drex-v1.5","pub":58.08,"same_skills":49.65,"new_domains":30.82,"v":45.78,"lab":false,"jev":false,"rank_v":107,"rank_pub":77,"name":"Nace.AI Drex v1.5","eq":{"S":49.17,"O":40.7},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":29.5,"new_domains":16.28}},{"engine":"decisiontune-1.0","pub":28.85,"same_skills":22.11,"new_domains":6.82,"v":17.22,"lab":false,"jev":false,"rank_v":207,"rank_pub":193,"name":"DecisionTune 1.0","eq":{"S":21.32,"O":11.07},"rank_tie":48,"tied":true,"parts":{"public":0,"same_skills":12.79,"new_domains":4.43}},{"engine":"vera-spark","pub":19.59,"same_skills":15.13,"new_domains":2.81,"v":11,"lab":false,"jev":false,"rank_v":228,"rank_pub":215,"name":"vera-spark","eq":{"S":14.25,"O":6.12},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":8.55,"new_domains":2.45}},{"engine":"nirnaya-0.4b","pub":1.6,"same_skills":2.66,"new_domains":2.98,"v":3.51,"lab":false,"jev":false,"rank_v":242,"rank_pub":242,"name":"Nirnaya 0.4B","eq":{"S":1.64,"O":6.32},"rank_tie":59,"tied":true,"parts":{"public":0,"same_skills":0.98,"new_domains":2.53}},{"engine":"saracura-ptbr-4b","pub":37.96,"same_skills":34.97,"new_domains":25.87,"v":34.43,"lab":false,"jev":false,"rank_v":152,"rank_pub":164,"name":"Saracura PT-BR 4B","eq":{"S":34.32,"O":34.6},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":20.59,"new_domains":13.84}},{"engine":"aplomb-1","pub":43.49,"same_skills":42.29,"new_domains":29.67,"v":40.75,"lab":false,"jev":false,"rank_v":131,"rank_pub":139,"name":"Aplomb 1","eq":{"S":41.73,"O":39.29},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.04,"new_domains":15.72}},{"engine":"rsi-jev-v6.1-vl-4b","pub":50.98,"same_skills":47.98,"new_domains":29.81,"v":44.27,"lab":false,"jev":false,"rank_v":115,"rank_pub":114,"name":"RSI-Jev v6.1 VL 4B","eq":{"S":47.49,"O":39.46},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":28.49,"new_domains":15.78}},{"engine":"sieve-4b","pub":50.06,"same_skills":45.16,"new_domains":26.57,"v":40.96,"lab":false,"jev":false,"rank_v":128,"rank_pub":119,"name":"Sieve 4B","eq":{"S":44.63,"O":35.46},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":26.78,"new_domains":14.19}},{"engine":"jiwo-4b-v2","pub":46.06,"same_skills":47.21,"new_domains":30.4,"v":44.1,"lab":false,"jev":false,"rank_v":117,"rank_pub":135,"name":"jiwo 4B v2","eq":{"S":46.7,"O":40.19},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":28.02,"new_domains":16.08}},{"engine":"cam1","pub":30.72,"same_skills":32.28,"new_domains":29.02,"v":34.36,"lab":false,"jev":false,"rank_v":153,"rank_pub":186,"name":"Cam1","eq":{"S":31.6,"O":38.49},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":18.96,"new_domains":15.4}},{"engine":"purplemist-flash-1.0","pub":46.74,"same_skills":48.24,"new_domains":36.1,"v":47.54,"lab":false,"jev":false,"rank_v":99,"rank_pub":134,"name":"PurpleMIST-Flash-1.0","eq":{"S":47.74,"O":47.23},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":28.65,"new_domains":18.89}},{"engine":"sieve-9b-plus","pub":53.35,"same_skills":49.29,"new_domains":33.72,"v":47,"lab":false,"jev":false,"rank_v":102,"rank_pub":106,"name":"Sieve 9B v2","eq":{"S":48.81,"O":44.29},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":29.29,"new_domains":17.72}},{"engine":"garas","pub":56.59,"same_skills":56.63,"new_domains":48.89,"v":58.95,"lab":false,"jev":false,"rank_v":58,"rank_pub":93,"name":"garas (Gemma 4 31B)","eq":{"S":56.24,"O":63.03},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":33.74,"new_domains":25.21}},{"engine":"darwin-27b-ztc","pub":62.63,"same_skills":58.71,"new_domains":54.59,"v":63.03,"lab":false,"jev":false,"rank_v":38,"rank_pub":47,"name":"Darwin-27B-ZTC","eq":{"S":58.34,"O":70.07},"rank_tie":8,"tied":true,"parts":{"public":0,"same_skills":35,"new_domains":28.03}},{"engine":"linne-fern-1.0","pub":58.92,"same_skills":56.52,"new_domains":50.44,"v":59.65,"lab":false,"jev":false,"rank_v":53,"rank_pub":70,"name":"linne-fern 1.0 (27B)","eq":{"S":56.12,"O":64.94},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":33.67,"new_domains":25.98}},{"engine":"qwen-decision-27b-noncausal","pub":57.7,"same_skills":57.71,"new_domains":51.96,"v":61.12,"lab":false,"jev":false,"rank_v":44,"rank_pub":83,"name":"Qwen Decision 27B noncausal LoRA","eq":{"S":57.32,"O":66.82},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":34.39,"new_domains":26.73}},{"engine":"reflex-27b-pmscr","pub":57.34,"same_skills":57.39,"new_domains":47.6,"v":58.77,"lab":false,"jev":false,"rank_v":61,"rank_pub":88,"name":"reflex Qwen3.8-27B-FP8 (pmscr)","eq":{"S":57,"O":61.43},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":34.2,"new_domains":24.57}},{"engine":"rsi-jev-v6.1-vl-27b-auto","pub":65.64,"same_skills":58.19,"new_domains":46.27,"v":58.6,"lab":false,"jev":false,"rank_v":62,"rank_pub":30,"name":"RSI-Jev v6.1 VL 27B (effort auto)","eq":{"S":57.81,"O":59.79},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":34.69,"new_domains":23.91}},{"engine":"anyjev-l0-gemma-4-31b","pub":59.79,"same_skills":59.13,"new_domains":55.34,"v":63.65,"lab":false,"jev":false,"rank_v":36,"rank_pub":61,"name":"AnyJev L0 · Gemma-4-31B","eq":{"S":58.76,"O":70.99},"rank_tie":8,"tied":true,"parts":{"public":0,"same_skills":35.26,"new_domains":28.39}},{"engine":"decisio-0.6.0-gemma-4-31b","pub":58.09,"same_skills":58.5,"new_domains":54.15,"v":62.68,"lab":false,"jev":false,"rank_v":39,"rank_pub":76,"name":"decisio 0.6.0 (Gemma 4 31B)","eq":{"S":58.12,"O":69.52},"rank_tie":9,"tied":false,"parts":{"public":0,"same_skills":34.87,"new_domains":27.81}},{"engine":"decider-31b","pub":64.27,"same_skills":61.79,"new_domains":56.88,"v":66.03,"lab":false,"jev":false,"rank_v":25,"rank_pub":42,"name":"Decider 31B (NVFP4)","eq":{"S":61.45,"O":72.9},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":36.87,"new_domains":29.16}},{"engine":"vllm-sr-d3-v310","pub":65.45,"same_skills":62.83,"new_domains":57.86,"v":67.15,"lab":false,"jev":false,"rank_v":19,"rank_pub":34,"name":"vLLM-SR d3 (27B)","eq":{"S":62.5,"O":74.11},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.5,"new_domains":29.64}},{"engine":"vllm-sr-d3-flash","pub":57.79,"same_skills":54.63,"new_domains":41.87,"v":54.27,"lab":false,"jev":false,"rank_v":81,"rank_pub":82,"name":"vLLM-SR d3-flash (9B)","eq":{"S":54.21,"O":54.36},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":32.52,"new_domains":21.74}},{"engine":"vllm-sr-d3-mini","pub":54.9,"same_skills":51.9,"new_domains":36.52,"v":49.97,"lab":false,"jev":false,"rank_v":93,"rank_pub":100,"name":"vLLM-SR d3-mini (4B)","eq":{"S":51.45,"O":47.75},"rank_tie":21,"tied":true,"parts":{"public":0,"same_skills":30.87,"new_domains":19.1}},{"engine":"vllm-sr-d3-nano","pub":42.22,"same_skills":42.47,"new_domains":16.4,"v":34.3,"lab":false,"jev":false,"rank_v":154,"rank_pub":145,"name":"vLLM-SR d3-nano (2B)","eq":{"S":41.91,"O":22.89},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":25.15,"new_domains":9.16}},{"engine":"vllm-sr-d3-lite","pub":36.93,"same_skills":34.81,"new_domains":9.33,"v":26.16,"lab":false,"jev":false,"rank_v":178,"rank_pub":172,"name":"vLLM-SR d3-lite (0.8B)","eq":{"S":34.16,"O":14.17},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":20.49,"new_domains":5.67}},{"engine":"vllm-sr-d3-edge","pub":28.82,"same_skills":24.34,"new_domains":5.03,"v":17.69,"lab":false,"jev":false,"rank_v":204,"rank_pub":194,"name":"vLLM-SR d3-edge (0.6B)","eq":{"S":23.57,"O":8.86},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":14.14,"new_domains":3.54}},{"engine":"wald-26b-a4b","pub":62.3,"same_skills":56.38,"new_domains":41.91,"v":55.35,"lab":false,"jev":false,"rank_v":75,"rank_pub":48,"name":"Wald-26B-A4B","eq":{"S":55.98,"O":54.41},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":33.59,"new_domains":21.76}},{"engine":"synack-decide-26b-a4b-v7","pub":59.68,"same_skills":56.29,"new_domains":43.79,"v":56.22,"lab":false,"jev":false,"rank_v":71,"rank_pub":63,"name":"SynACK Decide 26B-A4B v7","eq":{"S":55.89,"O":56.72},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":33.53,"new_domains":22.69}},{"engine":"ajev-gemma26-lora1","pub":57.67,"same_skills":54.92,"new_domains":43.58,"v":55.29,"lab":false,"jev":false,"rank_v":76,"rank_pub":84,"name":"AJev lora1 (Gemma 4 26B-A4B)","eq":{"S":54.5,"O":56.47},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":32.7,"new_domains":22.59}},{"engine":"drex-v2.0","pub":61.25,"same_skills":54.3,"new_domains":39.22,"v":52.76,"lab":false,"jev":false,"rank_v":86,"rank_pub":54,"name":"Nace.AI Drex v2.0 (35B-A3B)","eq":{"S":53.87,"O":51.08},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":32.32,"new_domains":20.43}},{"engine":"decision-4b-v1.6","pub":42.7,"same_skills":43.09,"new_domains":26.75,"v":39.8,"lab":false,"jev":false,"rank_v":135,"rank_pub":143,"name":"FlyMy.AI Decision 4B v1.6","eq":{"S":42.54,"O":35.68},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":25.52,"new_domains":14.27}},{"engine":"sieve-0.8b","pub":32.26,"same_skills":26.01,"new_domains":9.33,"v":20.82,"lab":false,"jev":false,"rank_v":194,"rank_pub":183,"name":"Sieve 0.8B","eq":{"S":25.26,"O":14.17},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":15.16,"new_domains":5.67}},{"engine":"jiwo-0.8b-v2-r2","pub":29.58,"same_skills":30.66,"new_domains":13.71,"v":25.81,"lab":false,"jev":false,"rank_v":180,"rank_pub":191,"name":"jiwo 0.8B v2","eq":{"S":29.96,"O":19.57},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":17.98,"new_domains":7.83}},{"engine":"ajev-lora7","pub":55.17,"same_skills":50.51,"new_domains":36.08,"v":48.91,"lab":false,"jev":false,"rank_v":95,"rank_pub":99,"name":"AJev lora7 (Gemma 4 12B)","eq":{"S":50.04,"O":47.2},"rank_tie":22,"tied":true,"parts":{"public":0,"same_skills":30.03,"new_domains":18.88}},{"engine":"bobcat-1.3","pub":59.31,"same_skills":59.9,"new_domains":56.81,"v":64.85,"lab":false,"jev":false,"rank_v":30,"rank_pub":65,"name":"Bobcat 1.3 (31B)","eq":{"S":59.54,"O":72.8},"rank_tie":6,"tied":true,"parts":{"public":0,"same_skills":35.73,"new_domains":29.12}},{"engine":"standard-one-8b-sh-v4","pub":57.4,"same_skills":50.43,"new_domains":24.07,"v":42.93,"lab":false,"jev":false,"rank_v":122,"rank_pub":87,"name":"Standard One 8B SH (v4)","eq":{"S":49.96,"O":32.37},"rank_tie":27,"tied":true,"parts":{"public":0,"same_skills":29.98,"new_domains":12.95}},{"engine":"jev-35b","pub":59.73,"same_skills":56.18,"new_domains":46.73,"v":57.61,"lab":false,"jev":false,"rank_v":66,"rank_pub":62,"name":"JEV-35B","eq":{"S":55.78,"O":60.36},"rank_tie":14,"tied":true,"parts":{"public":0,"same_skills":33.47,"new_domains":24.15}},{"engine":"jev","pub":60.7,"same_skills":59.28,"new_domains":48.26,"v":60.24,"lab":false,"jev":true,"rank_v":48,"rank_pub":57,"name":"Jev 1.14","eq":{"S":58.91,"O":62.24},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.34,"new_domains":24.9}},{"engine":"jev-1.13.0","pub":57.96,"same_skills":58.04,"new_domains":47.72,"v":59.22,"lab":false,"jev":false,"rank_v":56,"rank_pub":80,"name":"Jev 1.13","eq":{"S":57.66,"O":61.58},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":34.59,"new_domains":24.63}},{"engine":"startlux-decision-v1.1-27b-instant","pub":72.88,"same_skills":64.82,"new_domains":56.98,"v":67.92,"lab":false,"jev":false,"rank_v":13,"rank_pub":3,"name":"StartLux-Decision v1.1 27B","eq":{"S":64.51,"O":73.02},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.71,"new_domains":29.21}},{"engine":"pplx-decider-v1.2","pub":67.74,"same_skills":64.3,"new_domains":57.76,"v":67.98,"lab":false,"jev":false,"rank_v":11,"rank_pub":22,"name":"Perplexity Decider v1.2 (27B)","eq":{"S":63.99,"O":73.98},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.39,"new_domains":29.59}},{"engine":"pplx-decider-flash-v1.2","pub":60.06,"same_skills":56.16,"new_domains":40.02,"v":54.29,"lab":false,"jev":false,"rank_v":80,"rank_pub":59,"name":"Perplexity Decider Flash v1.2 (4B)","eq":{"S":55.76,"O":52.07},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":33.46,"new_domains":20.83}},{"engine":"amos-1-large","pub":64.94,"same_skills":62.69,"new_domains":53.31,"v":64.81,"lab":false,"jev":false,"rank_v":31,"rank_pub":40,"name":"Amos 1 Large (27B)","eq":{"S":62.37,"O":68.48},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":37.42,"new_domains":27.39}},{"engine":"lychee-27b","pub":62.76,"same_skills":62.52,"new_domains":58.66,"v":67.35,"lab":false,"jev":false,"rank_v":17,"rank_pub":46,"name":"Lychee-27B","eq":{"S":62.19,"O":75.09},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.31,"new_domains":30.03}},{"engine":"span-02","pub":63.62,"same_skills":59.32,"new_domains":45.84,"v":59.07,"lab":false,"jev":false,"rank_v":57,"rank_pub":44,"name":"span-02 (Gemma 4 26B-A4B)","eq":{"S":58.95,"O":59.26},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":35.37,"new_domains":23.7}},{"engine":"drex-max-v1","pub":67.52,"same_skills":63.08,"new_domains":56.45,"v":66.6,"lab":false,"jev":false,"rank_v":22,"rank_pub":23,"name":"Nace.AI Drex Max v1 (27B)","eq":{"S":62.76,"O":72.36},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.65,"new_domains":28.94},"new":true},{"engine":"ajev-lm6q","pub":65.46,"same_skills":63.37,"new_domains":58.05,"v":67.57,"lab":false,"jev":false,"rank_v":15,"rank_pub":33,"name":"AJev lm6q (Gemma 4 31B)","eq":{"S":63.05,"O":74.34},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.83,"new_domains":29.74},"new":true},{"engine":"mammoth-1","pub":75.34,"same_skills":66.26,"new_domains":56.27,"v":68.44,"lab":false,"jev":false,"rank_v":9,"rank_pub":1,"name":"baselabs mammoth-1 (27B)","eq":{"S":65.98,"O":72.14},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":39.59,"new_domains":28.86},"new":true},{"engine":"elephant-1-baselabs","pub":72.24,"same_skills":62.44,"new_domains":44.41,"v":60.26,"lab":false,"jev":false,"rank_v":47,"rank_pub":4,"name":"baselabs elephant-1 (9B)","eq":{"S":62.11,"O":57.49},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":37.27,"new_domains":22.99},"new":true},{"engine":"matilda-decision-v1","pub":65.39,"same_skills":60.34,"new_domains":56.07,"v":64.75,"lab":false,"jev":false,"rank_v":32,"rank_pub":36,"name":"Matilda Decision v1","eq":{"S":59.98,"O":71.9},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":35.99,"new_domains":28.76},"new":true},{"engine":"sieve-27b-v2","pub":68.02,"same_skills":60.03,"new_domains":46.62,"v":59.89,"lab":false,"jev":false,"rank_v":51,"rank_pub":18,"name":"Sieve 27B v2","eq":{"S":59.67,"O":60.23},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.8,"new_domains":24.09},"new":true},{"engine":"cerno-27b","pub":65.19,"same_skills":59.09,"new_domains":51.39,"v":61.68,"lab":false,"jev":false,"rank_v":40,"rank_pub":38,"name":"Cerno 27B","eq":{"S":58.72,"O":66.11},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":35.23,"new_domains":26.44},"new":true},{"engine":"jev-brain-gemma-4-31b","pub":61.69,"same_skills":61.56,"new_domains":54.29,"v":64.61,"lab":false,"jev":false,"rank_v":33,"rank_pub":51,"name":"jev-brain · Gemma-4-31B","eq":{"S":61.22,"O":69.7},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":36.73,"new_domains":27.88},"new":true},{"engine":"onejev1.6-27b","pub":69.55,"same_skills":62.48,"new_domains":46.79,"v":61.46,"lab":false,"jev":false,"rank_v":43,"rank_pub":13,"name":"OneJev1.6-27B","eq":{"S":62.15,"O":60.43},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":37.29,"new_domains":24.17},"new":true},{"engine":"onejev1.6-9b","pub":65.3,"same_skills":56.86,"new_domains":36.66,"v":53.05,"lab":false,"jev":false,"rank_v":85,"rank_pub":37,"name":"OneJev1.6-9B","eq":{"S":56.46,"O":47.93},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":33.88,"new_domains":19.17},"new":true},{"engine":"onejev1.6-4b","pub":61.95,"same_skills":55.78,"new_domains":29.56,"v":48.88,"lab":false,"jev":false,"rank_v":96,"rank_pub":49,"name":"OneJev1.6-4B","eq":{"S":55.37,"O":39.15},"rank_tie":22,"tied":true,"parts":{"public":0,"same_skills":33.22,"new_domains":15.66},"new":true},{"engine":"onejev1.6-0.8b","pub":48.76,"same_skills":40.49,"new_domains":9.28,"v":29.58,"lab":false,"jev":false,"rank_v":169,"rank_pub":124,"name":"OneJev1.6-0.8B","eq":{"S":39.9,"O":14.11},"rank_tie":38,"tied":true,"parts":{"public":0,"same_skills":23.94,"new_domains":5.64},"new":true},{"engine":"garum","pub":58.36,"same_skills":58.76,"new_domains":48.94,"v":60.27,"lab":false,"jev":false,"rank_v":46,"rank_pub":72,"name":"garum (Gemma 4 31B)","eq":{"S":58.38,"O":63.09},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.03,"new_domains":25.23},"new":true},{"engine":"jebadiah-27b-v2.1-t1e","pub":60.46,"same_skills":59.35,"new_domains":56.57,"v":64.4,"lab":false,"jev":false,"rank_v":35,"rank_pub":58,"name":"frontier-infra Jebadiah 27B v2.1 (t1e)","eq":{"S":58.98,"O":72.52},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":35.39,"new_domains":29.01},"new":true},{"engine":"jebadiah-9b-v2.1-t1e","pub":51.53,"same_skills":51.04,"new_domains":39.47,"v":50.9,"lab":false,"jev":false,"rank_v":91,"rank_pub":111,"name":"frontier-infra Jebadiah 9B v2.1 (t1e)","eq":{"S":50.57,"O":51.39},"rank_tie":20,"tied":true,"parts":{"public":0,"same_skills":30.34,"new_domains":20.56},"new":true},{"engine":"xor-26b-a4b-v1.2","pub":59.46,"same_skills":58.74,"new_domains":47.3,"v":59.44,"lab":false,"jev":false,"rank_v":55,"rank_pub":64,"name":"Xor 26B-A4B v1.2","eq":{"S":58.37,"O":61.06},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":35.02,"new_domains":24.42},"new":true},{"engine":"startlux-decision-v1.1-9b-instant","pub":69.98,"same_skills":61.3,"new_domains":41.38,"v":58.07,"lab":false,"jev":false,"rank_v":64,"rank_pub":12,"name":"StartLux-Decision v1.1 9B","eq":{"S":60.95,"O":53.75},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":36.57,"new_domains":21.5},"new":true},{"engine":"startlux-decision-v1.1-4b-instant","pub":67.81,"same_skills":58.6,"new_domains":38.07,"v":54.8,"lab":false,"jev":false,"rank_v":78,"rank_pub":19,"name":"StartLux-Decision v1.1 4B","eq":{"S":58.22,"O":49.67},"rank_tie":16,"tied":true,"parts":{"public":0,"same_skills":34.93,"new_domains":19.87},"new":true},{"engine":"startlux-decision-v1.1-2b-instant","pub":59.23,"same_skills":51.52,"new_domains":17.04,"v":40.11,"lab":false,"jev":false,"rank_v":134,"rank_pub":66,"name":"StartLux-Decision v1.1 2B","eq":{"S":51.07,"O":23.68},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":30.64,"new_domains":9.47},"new":true},{"engine":"startlux-decision-v1.1-0.8b-instant","pub":53.79,"same_skills":41.29,"new_domains":6.86,"v":28.88,"lab":false,"jev":false,"rank_v":174,"rank_pub":102,"name":"StartLux-Decision v1.1 0.8B","eq":{"S":40.71,"O":11.12},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":24.43,"new_domains":4.45},"new":true},{"engine":"coco-decision-2.4-4b","pub":55.19,"same_skills":51.67,"new_domains":27.81,"v":45.52,"lab":false,"jev":false,"rank_v":110,"rank_pub":98,"name":"CoCo-Decision-4B 2.4","eq":{"S":51.21,"O":36.99},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":30.73,"new_domains":14.8},"new":true},{"engine":"jegue-0.1","pub":54.13,"same_skills":48.78,"new_domains":28.6,"v":44.16,"lab":false,"jev":false,"rank_v":116,"rank_pub":101,"name":"Jegue 0.1 (9B)","eq":{"S":48.29,"O":37.96},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":28.97,"new_domains":15.19},"new":true},{"engine":"nari-decision-4b-v3.2","pub":44.8,"same_skills":43.38,"new_domains":28.58,"v":40.87,"lab":false,"jev":false,"rank_v":130,"rank_pub":136,"name":"Nari Decision 4B v3.2","eq":{"S":42.83,"O":37.94},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":25.7,"new_domains":15.17},"new":true},{"engine":"latentmesh-blitz-2b-v1","pub":48.15,"same_skills":44.23,"new_domains":14.52,"v":34.44,"lab":false,"jev":false,"rank_v":151,"rank_pub":127,"name":"LatentMesh Blitz 2B v1","eq":{"S":43.68,"O":20.57},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":26.21,"new_domains":8.23},"new":true},{"engine":"sieve-2b-v3","pub":53.51,"same_skills":45.8,"new_domains":12.61,"v":34.45,"lab":false,"jev":false,"rank_v":150,"rank_pub":104,"name":"Sieve 2B v3","eq":{"S":45.27,"O":18.22},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":27.16,"new_domains":7.29},"new":true},{"engine":"vera-core-2","pub":41.28,"same_skills":39.55,"new_domains":12.49,"v":30.6,"lab":false,"jev":false,"rank_v":167,"rank_pub":148,"name":"vera-core-2","eq":{"S":38.95,"O":18.07},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":23.37,"new_domains":7.23},"new":true},{"engine":"aurelis-2b","pub":51.29,"same_skills":44.71,"new_domains":10.1,"v":32.55,"lab":false,"jev":false,"rank_v":159,"rank_pub":113,"name":"Aurelis-2B","eq":{"S":44.17,"O":15.11},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":26.5,"new_domains":6.05},"new":true},{"engine":"saina-helm-2-v2","pub":39.26,"same_skills":34.65,"new_domains":7.75,"v":25.29,"lab":false,"jev":false,"rank_v":181,"rank_pub":157,"name":"Saina Helm 2.0 v2","eq":{"S":34,"O":12.21},"rank_tie":41,"tied":true,"parts":{"public":0,"same_skills":20.4,"new_domains":4.89},"new":true},{"engine":"saina-helm-2-omni","pub":37.44,"same_skills":35.21,"new_domains":9.59,"v":26.54,"lab":false,"jev":false,"rank_v":177,"rank_pub":168,"name":"Saina Helm 2 Omni (0.8B)","eq":{"S":34.57,"O":14.49},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":20.74,"new_domains":5.8},"new":true},{"engine":"wyat-1-800m","pub":25.78,"same_skills":25.21,"new_domains":11.68,"v":21.49,"lab":false,"jev":false,"rank_v":192,"rank_pub":203,"name":"wyat-1-800m","eq":{"S":24.45,"O":17.07},"rank_tie":44,"tied":true,"parts":{"public":0,"same_skills":14.67,"new_domains":6.83},"new":true},{"engine":"amber-0.4b","pub":49.74,"same_skills":35.73,"new_domains":7.86,"v":26,"lab":false,"jev":false,"rank_v":179,"rank_pub":121,"name":"Amber 0.4B","eq":{"S":35.09,"O":12.35},"rank_tie":40,"tied":true,"parts":{"public":0,"same_skills":21.06,"new_domains":4.94},"new":true},{"engine":"crivo-1b","pub":47.77,"same_skills":38.75,"new_domains":10.75,"v":29.25,"lab":false,"jev":false,"rank_v":172,"rank_pub":128,"name":"Crivo-1B","eq":{"S":38.15,"O":15.92},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":22.89,"new_domains":6.37},"new":true},{"engine":"crivo-150m-v2","pub":34.38,"same_skills":24.93,"new_domains":5.48,"v":18.26,"lab":false,"jev":false,"rank_v":201,"rank_pub":174,"name":"Crivo-150M v2","eq":{"S":24.16,"O":9.41},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":14.5,"new_domains":3.76},"new":true},{"engine":"crivo-68m","pub":32.82,"same_skills":25.27,"new_domains":7.59,"v":19.51,"lab":false,"jev":false,"rank_v":198,"rank_pub":179,"name":"Crivo-68M","eq":{"S":24.51,"O":12.02},"rank_tie":46,"tied":true,"parts":{"public":0,"same_skills":14.71,"new_domains":4.81},"new":true},{"engine":"crivo-32m-v2","pub":27.34,"same_skills":18.44,"new_domains":5.29,"v":14.23,"lab":false,"jev":false,"rank_v":217,"rank_pub":199,"name":"Crivo-32M v2","eq":{"S":17.6,"O":9.18},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":10.56,"new_domains":3.67},"new":true},{"engine":"crivo-17m","pub":22.15,"same_skills":15.88,"new_domains":3.21,"v":11.65,"lab":false,"jev":false,"rank_v":225,"rank_pub":208,"name":"Crivo-17M","eq":{"S":15.01,"O":6.61},"rank_tie":53,"tied":true,"parts":{"public":0,"same_skills":9.01,"new_domains":2.64},"new":true},{"engine":"mola-150m","pub":31.34,"same_skills":23.55,"new_domains":6.32,"v":17.84,"lab":false,"jev":false,"rank_v":203,"rank_pub":185,"name":"Mola-150M","eq":{"S":22.77,"O":10.45},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":13.66,"new_domains":4.18},"new":true},{"engine":"mola-68m","pub":27.27,"same_skills":21.99,"new_domains":7.74,"v":17.6,"lab":false,"jev":false,"rank_v":205,"rank_pub":200,"name":"Mola-68M","eq":{"S":21.2,"O":12.2},"rank_tie":48,"tied":true,"parts":{"public":0,"same_skills":12.72,"new_domains":4.88},"new":true},{"engine":"jev-t5-342m-public-v2","pub":22.02,"same_skills":19.77,"new_domains":9.9,"v":17.31,"lab":false,"jev":false,"rank_v":206,"rank_pub":209,"name":"JEV FLAN-T5 Large public v2 (342M)","eq":{"S":18.94,"O":14.86},"rank_tie":48,"tied":true,"parts":{"public":0,"same_skills":11.37,"new_domains":5.95},"new":true},{"engine":"prumo","pub":20.14,"same_skills":15.78,"new_domains":8.08,"v":13.99,"lab":false,"jev":false,"rank_v":218,"rank_pub":214,"name":"Prumo","eq":{"S":14.91,"O":12.62},"rank_tie":51,"tied":true,"parts":{"public":0,"same_skills":8.95,"new_domains":5.05},"new":true},{"engine":"dinah-1","pub":34.32,"same_skills":22.29,"new_domains":4.85,"v":16.36,"lab":false,"jev":false,"rank_v":211,"rank_pub":175,"name":"Dinah-1","eq":{"S":21.5,"O":8.64},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":12.9,"new_domains":3.46},"new":true},{"engine":"crivo-400m","pub":39.79,"same_skills":30.7,"new_domains":9.48,"v":23.74,"lab":false,"jev":false,"rank_v":184,"rank_pub":156,"name":"Crivo-400M","eq":{"S":30,"O":14.35},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":18,"new_domains":5.74},"new":true},{"engine":"jev-ettin-400m","pub":14.11,"same_skills":14.82,"new_domains":11.77,"v":15.24,"lab":false,"jev":false,"rank_v":215,"rank_pub":225,"name":"JEV Ettin 400M","eq":{"S":13.94,"O":17.19},"rank_tie":50,"tied":true,"parts":{"public":0,"same_skills":8.36,"new_domains":6.87},"new":true},{"engine":"mogan-decision-31b-it","pub":65.55,"same_skills":57.67,"new_domains":49.39,"v":59.82,"lab":false,"jev":false,"rank_v":52,"rank_pub":31,"name":"mogan-decision-31B-it","eq":{"S":57.28,"O":63.64},"rank_tie":12,"tied":true,"parts":{"public":0,"same_skills":34.37,"new_domains":25.46},"new":true},{"engine":"jeffan-31b-1710","pub":73.13,"same_skills":66.15,"new_domains":59.06,"v":69.75,"lab":false,"jev":false,"rank_v":6,"rank_pub":2,"name":"Jeffan-31b-1710","eq":{"S":65.86,"O":75.58},"rank_tie":2,"tied":true,"parts":{"public":0,"same_skills":39.52,"new_domains":30.23},"new":true},{"engine":"autotrust-jev-27b-v11","pub":67.32,"same_skills":64.08,"new_domains":57.96,"v":67.95,"lab":false,"jev":false,"rank_v":12,"rank_pub":24,"name":"AutoTrust-JEV-27B v1.1","eq":{"S":63.77,"O":74.22},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.26,"new_domains":29.69},"new":true},{"engine":"flexdm-gemma-4-31b","pub":63.63,"same_skills":62.98,"new_domains":58.52,"v":67.56,"lab":false,"jev":false,"rank_v":16,"rank_pub":43,"name":"FlexDM · Gemma-4-31B","eq":{"S":62.66,"O":74.92},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.59,"new_domains":29.97},"new":true},{"engine":"visionjev-1.0.0","pub":66.46,"same_skills":62.8,"new_domains":57.37,"v":66.88,"lab":false,"jev":false,"rank_v":20,"rank_pub":26,"name":"VisionJEV 1.0.0 (27B)","eq":{"S":62.47,"O":73.5},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.48,"new_domains":29.4},"new":true},{"engine":"nimble-v4-max","pub":68.46,"same_skills":62.56,"new_domains":57.01,"v":66.56,"lab":false,"jev":false,"rank_v":23,"rank_pub":15,"name":"Bespoke Nimble v4 Max (27B)","eq":{"S":62.24,"O":73.05},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.34,"new_domains":29.22},"new":true},{"engine":"aleria-judge-1.1.1","pub":63.46,"same_skills":62.39,"new_domains":57.44,"v":66.67,"lab":false,"jev":false,"rank_v":21,"rank_pub":45,"name":"Aleria Judge 1.1.1 (Gemma 4 31B)","eq":{"S":62.06,"O":73.59},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.23,"new_domains":29.43},"new":true},{"engine":"anyjev-tacitus-31b","pub":65.52,"same_skills":62.48,"new_domains":56.93,"v":66.47,"lab":false,"jev":false,"rank_v":24,"rank_pub":32,"name":"AnyJev Tacitus 31B","eq":{"S":62.15,"O":72.95},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":37.29,"new_domains":29.18},"new":true},{"engine":"pick-150m-v5","pub":32.09,"same_skills":19.76,"new_domains":6.48,"v":15.62,"lab":false,"jev":false,"rank_v":213,"rank_pub":184,"name":"PICK-150M v5","eq":{"S":18.94,"O":10.65},"rank_tie":50,"tied":true,"parts":{"public":0,"same_skills":11.36,"new_domains":4.26},"new":true},{"engine":"vllm-sr-d3-turbo-v11","pub":71.62,"same_skills":64.29,"new_domains":56.41,"v":67.31,"lab":false,"jev":false,"rank_v":18,"rank_pub":8,"name":"vLLM-SR d3-turbo v1.1 (27B)","eq":{"S":63.98,"O":72.31},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":38.39,"new_domains":28.93},"new":true},{"engine":"cerno-9b-v11","pub":58.59,"same_skills":52.36,"new_domains":30.63,"v":47.34,"lab":false,"jev":false,"rank_v":100,"rank_pub":71,"name":"Cerno 9B v1.1","eq":{"S":51.91,"O":40.48},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":31.15,"new_domains":16.19},"new":true},{"engine":"cerno-4b-v11","pub":57.6,"same_skills":47.41,"new_domains":24.2,"v":41.15,"lab":false,"jev":false,"rank_v":127,"rank_pub":85,"name":"Cerno 4B v1.1","eq":{"S":46.9,"O":32.53},"rank_tie":29,"tied":true,"parts":{"public":0,"same_skills":28.14,"new_domains":13.01},"new":true},{"engine":"shisa-de-2.1-repeat","pub":59.23,"same_skills":56.62,"new_domains":48.67,"v":58.84,"lab":false,"jev":false,"rank_v":59,"rank_pub":67,"name":"Shisa DE-2.1 (repeat)","eq":{"S":56.22,"O":62.76},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":33.73,"new_domains":25.1},"new":true},{"engine":"trio-spark-di","pub":55.52,"same_skills":50.47,"new_domains":29.42,"v":45.59,"lab":false,"jev":false,"rank_v":109,"rank_pub":97,"name":"Trio-Spark DI (9B)","eq":{"S":50,"O":38.98},"rank_tie":25,"tied":true,"parts":{"public":0,"same_skills":30,"new_domains":15.59},"new":true},{"engine":"tern-35b-a3b","pub":59.01,"same_skills":56,"new_domains":44.95,"v":56.62,"lab":false,"jev":false,"rank_v":69,"rank_pub":69,"name":"Tern-35B-A3B","eq":{"S":55.59,"O":58.16},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":33.36,"new_domains":23.26},"new":true},{"engine":"jdi-ptr14-0.8b","pub":35.79,"same_skills":31.28,"new_domains":8.44,"v":23.58,"lab":false,"jev":false,"rank_v":187,"rank_pub":173,"name":"jdi-ptr14-0.8b","eq":{"S":30.58,"O":13.06},"rank_tie":42,"tied":true,"parts":{"public":0,"same_skills":18.35,"new_domains":5.23},"new":true},{"engine":"glide-v1.2","pub":71.99,"same_skills":68.57,"new_domains":59.66,"v":71.52,"lab":false,"jev":false,"rank_v":2,"rank_pub":5,"name":"Fastino GLiDE 1.2","eq":{"S":68.31,"O":76.32},"rank_tie":1,"tied":true,"parts":{"public":0,"same_skills":40.99,"new_domains":30.53},"new":true},{"engine":"allec","pub":61.6,"same_skills":61.63,"new_domains":57.04,"v":66.01,"lab":false,"jev":false,"rank_v":26,"rank_pub":52,"name":"allec (Gemma 4 31B)","eq":{"S":61.29,"O":73.1},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":36.77,"new_domains":29.24},"new":true},{"engine":"lumma-fev-27b","pub":57.15,"same_skills":56.59,"new_domains":51.01,"v":59.97,"lab":false,"jev":false,"rank_v":50,"rank_pub":90,"name":"Lumma-fev-27B","eq":{"S":56.2,"O":65.64},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":33.72,"new_domains":26.26},"new":true},{"engine":"muria","pub":51.4,"same_skills":53.5,"new_domains":39.78,"v":52.55,"lab":false,"jev":false,"rank_v":87,"rank_pub":112,"name":"muria (Gemma 4 12B)","eq":{"S":53.07,"O":51.78},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":31.84,"new_domains":20.71},"new":true},{"engine":"mica-v0.5-4b","pub":57.56,"same_skills":52.25,"new_domains":31.5,"v":47.7,"lab":false,"jev":false,"rank_v":98,"rank_pub":86,"name":"Mica v0.5 4B","eq":{"S":51.81,"O":41.55},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":31.08,"new_domains":16.62},"new":true},{"engine":"jeffan-4b-2010","pub":64.85,"same_skills":58.03,"new_domains":33.52,"v":52.21,"lab":false,"jev":false,"rank_v":88,"rank_pub":41,"name":"Jeffan-4B-2010","eq":{"S":57.65,"O":44.04},"rank_tie":18,"tied":true,"parts":{"public":0,"same_skills":34.59,"new_domains":17.62},"new":true},{"engine":"tasksource-decider-nano-v5","pub":33.51,"same_skills":26.1,"new_domains":7.25,"v":19.85,"lab":false,"jev":false,"rank_v":197,"rank_pub":177,"name":"tasksource Decider nano v5 (150M)","eq":{"S":25.35,"O":11.6},"rank_tie":46,"tied":true,"parts":{"public":0,"same_skills":15.21,"new_domains":4.64},"new":true},{"engine":"senova-4b","pub":61.6,"same_skills":49.89,"new_domains":33.64,"v":47.32,"lab":false,"jev":false,"rank_v":101,"rank_pub":53,"name":"Senova-4B","eq":{"S":49.42,"O":44.19},"rank_tie":23,"tied":true,"parts":{"public":0,"same_skills":29.65,"new_domains":17.67},"new":true},{"engine":"standard-one-3b-sh-v4","pub":58.08,"same_skills":49.79,"new_domains":18.52,"v":39.79,"lab":false,"jev":false,"rank_v":136,"rank_pub":78,"name":"Standard One 3B SH (v4)","eq":{"S":49.31,"O":25.51},"rank_tie":30,"tied":true,"parts":{"public":0,"same_skills":29.58,"new_domains":10.2},"new":true},{"engine":"knowline-35b-a3b-gen1.5","pub":70.72,"same_skills":65.49,"new_domains":51.82,"v":65.77,"lab":false,"jev":false,"rank_v":27,"rank_pub":10,"name":"KnowLine 35B-A3B Gen1.5","eq":{"S":65.19,"O":66.65},"rank_tie":5,"tied":true,"parts":{"public":0,"same_skills":39.11,"new_domains":26.66},"new":true},{"engine":"knowline-26b-a4b-gen2.5","pub":68.21,"same_skills":62.82,"new_domains":52.67,"v":64.57,"lab":false,"jev":false,"rank_v":34,"rank_pub":16,"name":"KnowLine 26B-A4B Gen2.5","eq":{"S":62.49,"O":67.7},"rank_tie":7,"tied":true,"parts":{"public":0,"same_skills":37.5,"new_domains":27.08},"new":true},{"engine":"knowline-4b-gen5.5","pub":66.76,"same_skills":61.2,"new_domains":37.28,"v":55.99,"lab":false,"jev":false,"rank_v":72,"rank_pub":25,"name":"KnowLine 4B Gen5.5","eq":{"S":60.85,"O":48.69},"rank_tie":15,"tied":true,"parts":{"public":0,"same_skills":36.51,"new_domains":19.48},"new":true},{"engine":"gliclass-v4-edge","pub":47.49,"same_skills":42.16,"new_domains":10.53,"v":31.21,"lab":false,"jev":false,"rank_v":164,"rank_pub":131,"name":"GLiClass v4 edge (0.4B)","eq":{"S":41.59,"O":15.64},"rank_tie":37,"tied":true,"parts":{"public":0,"same_skills":24.95,"new_domains":6.26},"new":true},{"engine":"vllm-sr-d3-ultra","pub":71.57,"same_skills":66.77,"new_domains":59.16,"v":70.18,"lab":false,"jev":false,"rank_v":5,"rank_pub":9,"name":"vLLM-SR d3-ultra (Decision 3.0, 27B)","eq":{"S":66.49,"O":75.71},"rank_tie":2,"tied":true,"parts":{"public":0,"same_skills":39.89,"new_domains":30.29},"new":true},{"engine":"standard-one-35b-sh-v1","pub":66.26,"same_skills":61.77,"new_domains":44.98,"v":60.14,"lab":false,"jev":false,"rank_v":49,"rank_pub":27,"name":"Standard One 35B SH (v1)","eq":{"S":61.43,"O":58.2},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":36.86,"new_domains":23.28},"new":true},{"engine":"crystal-0.8b","pub":42.79,"same_skills":39.31,"new_domains":10.29,"v":29.37,"lab":false,"jev":false,"rank_v":170,"rank_pub":142,"name":"crystal-0.8b","eq":{"S":38.72,"O":15.35},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":23.23,"new_domains":6.14},"new":true},{"engine":"jev-ettin-150m","pub":37.31,"same_skills":25.45,"new_domains":5.36,"v":18.52,"lab":false,"jev":false,"rank_v":199,"rank_pub":169,"name":"JEV Ettin 150M","eq":{"S":24.69,"O":9.27},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":14.82,"new_domains":3.71},"new":true},{"engine":"valen-0.8b-r2","pub":20.79,"same_skills":19.6,"new_domains":8.57,"v":16.55,"lab":false,"jev":false,"rank_v":208,"rank_pub":211,"name":"Valen-0.8B","eq":{"S":18.77,"O":13.23},"rank_tie":49,"tied":true,"parts":{"public":0,"same_skills":11.26,"new_domains":5.29},"new":true},{"engine":"valen-2b-r2","pub":26.95,"same_skills":24.29,"new_domains":14.23,"v":22.2,"lab":false,"jev":false,"rank_v":189,"rank_pub":201,"name":"Valen-2B","eq":{"S":23.51,"O":20.22},"rank_tie":43,"tied":true,"parts":{"public":0,"same_skills":14.11,"new_domains":8.09},"new":true},{"engine":"valen-4b-r2","pub":38.68,"same_skills":39.13,"new_domains":27.73,"v":37.88,"lab":false,"jev":false,"rank_v":141,"rank_pub":159,"name":"Valen-4B","eq":{"S":38.53,"O":36.9},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":23.12,"new_domains":14.76},"new":true},{"engine":"vela3-fast","pub":50.36,"same_skills":40.22,"new_domains":8.96,"v":29.27,"lab":false,"jev":false,"rank_v":171,"rank_pub":117,"name":"vLLM-SR vela3-fast (Vela 3.0, 0.4B)","eq":{"S":39.63,"O":13.71},"rank_tie":39,"tied":true,"parts":{"public":0,"same_skills":23.78,"new_domains":5.49},"new":true},{"engine":"vela3-fast-vision-text","pub":37.5,"same_skills":30.47,"new_domains":6.45,"v":22.11,"lab":false,"jev":false,"rank_v":190,"rank_pub":167,"name":"vLLM-SR vela3-fast-vision (Vela 3.0, 0.25B)","eq":{"S":29.77,"O":10.62},"rank_tie":43,"tied":true,"parts":{"public":0,"same_skills":17.86,"new_domains":4.25},"new":true},{"engine":"sieve-31b-v3","pub":71.68,"same_skills":68.53,"new_domains":58.86,"v":71.1,"lab":false,"jev":false,"rank_v":3,"rank_pub":7,"name":"Sieve 31B v3","eq":{"S":68.27,"O":75.33},"rank_tie":1,"tied":true,"parts":{"public":0,"same_skills":40.96,"new_domains":30.13},"new":true},{"engine":"ainvest-decision-27b-v1.0","pub":71.88,"same_skills":64.58,"new_domains":57.91,"v":68.23,"lab":false,"jev":false,"rank_v":10,"rank_pub":6,"name":"Ainvest-Decision-27B v1.0","eq":{"S":64.27,"O":74.17},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":38.56,"new_domains":29.67},"new":true},{"engine":"oak-1.1","pub":68.19,"same_skills":62.55,"new_domains":59.17,"v":67.62,"lab":false,"jev":false,"rank_v":14,"rank_pub":17,"name":"MISA AI 2.0 Decision Plus · oak 1.1 (31B)","eq":{"S":62.22,"O":75.73},"rank_tie":4,"tied":true,"parts":{"public":0,"same_skills":37.33,"new_domains":30.29},"new":true},{"engine":"wemm-jev-27b","pub":69.37,"same_skills":65.39,"new_domains":57.45,"v":68.49,"lab":false,"jev":false,"rank_v":8,"rank_pub":14,"name":"WeMM-JEV-27B","eq":{"S":65.09,"O":73.6},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":39.05,"new_domains":29.44},"new":true},{"engine":"knowline-31b-gen1","pub":70.13,"same_skills":67.22,"new_domains":62.02,"v":71.86,"lab":false,"jev":false,"rank_v":1,"rank_pub":11,"name":"KnowLine 31B Gen1","eq":{"S":66.94,"O":79.24},"rank_tie":1,"tied":true,"parts":{"public":0,"same_skills":40.17,"new_domains":31.7},"new":true},{"engine":"h2o-lightning-31b-v1.1","pub":67.75,"same_skills":65.6,"new_domains":57.49,"v":68.64,"lab":false,"jev":false,"rank_v":7,"rank_pub":20,"name":"H2O Lightning 31B v1.1","eq":{"S":65.3,"O":73.64},"rank_tie":3,"tied":true,"parts":{"public":0,"same_skills":39.18,"new_domains":29.46},"new":true},{"engine":"oneshot-27b","pub":65.41,"same_skills":62.05,"new_domains":55.98,"v":65.74,"lab":false,"jev":false,"rank_v":28,"rank_pub":35,"name":"OneShot 27B","eq":{"S":61.71,"O":71.78},"rank_tie":6,"tied":true,"parts":{"public":0,"same_skills":37.03,"new_domains":28.71},"new":true},{"engine":"turion-jev-31b","pub":66.16,"same_skills":65.34,"new_domains":61.5,"v":70.47,"lab":false,"jev":false,"rank_v":4,"rank_pub":28,"name":"Turion Jev 31B","eq":{"S":65.04,"O":78.6},"rank_tie":2,"tied":true,"parts":{"public":0,"same_skills":39.02,"new_domains":31.44},"new":true},{"engine":"verdin-4b-v1.1","pub":60.96,"same_skills":54.13,"new_domains":37.88,"v":51.99,"lab":false,"jev":false,"rank_v":89,"rank_pub":55,"name":"Verdin-4B v1.1","eq":{"S":53.7,"O":49.43},"rank_tie":19,"tied":false,"parts":{"public":0,"same_skills":32.22,"new_domains":19.77},"new":true},{"engine":"vera-large","pub":67.75,"same_skills":61.56,"new_domains":55.61,"v":65.26,"lab":false,"jev":false,"rank_v":29,"rank_pub":21,"name":"vera-large","eq":{"S":61.21,"O":71.33},"rank_tie":6,"tied":true,"parts":{"public":0,"same_skills":36.73,"new_domains":28.53},"new":true},{"engine":"amber-hybrid-4b","pub":59.87,"same_skills":50.42,"new_domains":23.71,"v":42.74,"lab":false,"jev":false,"rank_v":123,"rank_pub":60,"name":"Amber Hybrid 4B","eq":{"S":49.95,"O":31.93},"rank_tie":27,"tied":true,"parts":{"public":0,"same_skills":29.97,"new_domains":12.77},"new":true},{"engine":"jev-style-2b-v4","pub":44.37,"same_skills":41.87,"new_domains":17.59,"v":34.53,"lab":false,"jev":false,"rank_v":149,"rank_pub":137,"name":"Jev-Style 2B v4","eq":{"S":41.3,"O":24.37},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":24.78,"new_domains":9.75},"new":true},{"engine":"knowline-9b-gen1","pub":65.67,"same_skills":60.82,"new_domains":42.7,"v":58.44,"lab":false,"jev":false,"rank_v":63,"rank_pub":29,"name":"KnowLine 9B Gen1","eq":{"S":60.48,"O":55.38},"rank_tie":13,"tied":true,"parts":{"public":0,"same_skills":36.29,"new_domains":22.15},"new":true},{"engine":"seb-9b-v2","pub":59.16,"same_skills":52.86,"new_domains":27.83,"v":46.25,"lab":false,"jev":false,"rank_v":105,"rank_pub":68,"name":"Seb-9B v2","eq":{"S":52.41,"O":37.01},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":31.45,"new_domains":14.8},"new":true},{"engine":"jpt-35b-a3b","pub":52.68,"same_skills":52.25,"new_domains":44.43,"v":54.09,"lab":false,"jev":false,"rank_v":82,"rank_pub":108,"name":"JPT-35B-A3B","eq":{"S":51.8,"O":57.52},"rank_tie":17,"tied":true,"parts":{"public":0,"same_skills":31.08,"new_domains":23.01}},{"engine":"zev-latest","pub":1.55,"same_skills":1.79,"new_domains":3.76,"v":3.37,"lab":false,"jev":false,"rank_v":243,"rank_pub":243,"name":"Zev","eq":{"S":0.76,"O":7.29},"rank_tie":59,"tied":true,"parts":{"public":0,"same_skills":0.46,"new_domains":2.92}},{"engine":"exaone-4.0-1.2b-jev-v0.3","pub":30.68,"same_skills":26.92,"new_domains":7.77,"v":20.6,"lab":false,"jev":false,"rank_v":196,"rank_pub":187,"name":"EXAONE-4.0-1.2B-JEV v0.3","eq":{"S":26.18,"O":12.24},"rank_tie":45,"tied":true,"parts":{"public":0,"same_skills":15.71,"new_domains":4.9}},{"engine":"rsi-jev-v5.0-vl-3b","pub":38.11,"same_skills":38.69,"new_domains":29.85,"v":38.65,"lab":false,"jev":false,"rank_v":139,"rank_pub":161,"name":"RSI-Jev v5.0 VL 3B","eq":{"S":38.08,"O":39.51},"rank_tie":31,"tied":true,"parts":{"public":0,"same_skills":22.85,"new_domains":15.8}},{"engine":"ezjev-4b-s2","pub":50.82,"same_skills":46.22,"new_domains":31.21,"v":43.9,"lab":false,"jev":false,"rank_v":119,"rank_pub":115,"name":"ezjev 4B s2","eq":{"S":45.7,"O":41.19},"rank_tie":26,"tied":true,"parts":{"public":0,"same_skills":27.42,"new_domains":16.48}},{"engine":"bekko-v0-17m","pub":7.44,"same_skills":5.89,"new_domains":7.4,"v":7.66,"lab":false,"jev":false,"rank_v":235,"rank_pub":234,"name":"Bekko System One v0 17M","eq":{"S":4.91,"O":11.78},"rank_tie":56,"tied":true,"parts":{"public":0,"same_skills":2.94,"new_domains":4.71}},{"engine":"bekko-v0-68m","pub":13.01,"same_skills":12.62,"new_domains":8.73,"v":12.4,"lab":false,"jev":false,"rank_v":223,"rank_pub":228,"name":"Bekko System One v0 68M","eq":{"S":11.72,"O":13.43},"rank_tie":52,"tied":true,"parts":{"public":0,"same_skills":7.03,"new_domains":5.37}},{"engine":"bekko-v0-400m","pub":22.45,"same_skills":21.55,"new_domains":9.05,"v":17.98,"lab":false,"jev":false,"rank_v":202,"rank_pub":206,"name":"Bekko System One v0 400M","eq":{"S":20.75,"O":13.83},"rank_tie":47,"tied":true,"parts":{"public":0,"same_skills":12.45,"new_domains":5.53}},{"engine":"torchcast-decision-27b","pub":65.1,"same_skills":58.13,"new_domains":52.19,"v":61.49,"lab":false,"jev":false,"rank_v":42,"rank_pub":39,"name":"Torchcast Decision 27B","eq":{"S":57.75,"O":67.1},"rank_tie":10,"tied":true,"parts":{"public":0,"same_skills":34.65,"new_domains":26.84}},{"engine":"kas-4b","pub":38.35,"same_skills":38.69,"new_domains":20.43,"v":34,"lab":false,"jev":false,"rank_v":156,"rank_pub":160,"name":"Kas 4B","eq":{"S":38.09,"O":27.88},"rank_tie":35,"tied":true,"parts":{"public":0,"same_skills":22.85,"new_domains":11.15}},{"engine":"lev1-3.7b","pub":42.68,"same_skills":41.45,"new_domains":19.89,"v":35.41,"lab":false,"jev":false,"rank_v":147,"rank_pub":144,"name":"Lev1 3.7B","eq":{"S":40.88,"O":27.21},"rank_tie":34,"tied":true,"parts":{"public":0,"same_skills":24.53,"new_domains":10.88}},{"engine":"decisio-0.4.0-qwen3.6-35b-a3b","pub":48.43,"same_skills":49.15,"new_domains":41.63,"v":50.82,"lab":false,"jev":false,"rank_v":92,"rank_pub":125,"name":"decisio 0.4.0 (Qwen3.6-35B-A3B)","eq":{"S":48.67,"O":54.06},"rank_tie":20,"tied":true,"parts":{"public":0,"same_skills":29.2,"new_domains":21.62}},{"engine":"decisio-0.4.0-gemma-4-12b","pub":49.72,"same_skills":48.44,"new_domains":38.31,"v":48.75,"lab":false,"jev":false,"rank_v":97,"rank_pub":122,"name":"decisio 0.4.0 (Gemma 4 12B)","eq":{"S":47.94,"O":49.95},"rank_tie":22,"tied":true,"parts":{"public":0,"same_skills":28.77,"new_domains":19.98}},{"engine":"hopper-12b-trained","pub":48.27,"same_skills":46.97,"new_domains":34.05,"v":45.76,"lab":false,"jev":false,"rank_v":108,"rank_pub":126,"name":"Hopper 12B (trained)","eq":{"S":46.46,"O":44.7},"rank_tie":24,"tied":true,"parts":{"public":0,"same_skills":27.88,"new_domains":17.88}},{"engine":"dm-jepa-1.1","pub":19.19,"same_skills":4.17,"new_domains":4.04,"v":4.96,"lab":false,"jev":false,"rank_v":240,"rank_pub":218,"name":"DM-JEPA 1.1","eq":{"S":3.17,"O":7.64},"rank_tie":58,"tied":true,"parts":{"public":0,"same_skills":1.9,"new_domains":3.05}},{"engine":"jevstral-8b","pub":34.01,"same_skills":36.31,"new_domains":19.77,"v":32.23,"lab":false,"jev":false,"rank_v":160,"rank_pub":176,"name":"Jevstral 8B","eq":{"S":35.68,"O":27.07},"rank_tie":36,"tied":true,"parts":{"public":0,"same_skills":21.41,"new_domains":10.83}},{"engine":"deck31b","pub":58.19,"same_skills":58.6,"new_domains":54.77,"v":63.05,"lab":false,"jev":false,"rank_v":37,"rank_pub":75,"name":"deck31b (Gemma 4 31B)","eq":{"S":58.23,"O":70.29},"rank_tie":8,"tied":true,"parts":{"public":0,"same_skills":34.94,"new_domains":28.12}},{"engine":"quyet-1.0-large","pub":60.79,"same_skills":58.83,"new_domains":49.86,"v":60.77,"lab":false,"jev":false,"rank_v":45,"rank_pub":56,"name":"Quyet-1.0-Large","eq":{"S":58.46,"O":64.23},"rank_tie":11,"tied":true,"parts":{"public":0,"same_skills":35.08,"new_domains":25.69}}],"formula":{"id":"v0.4","decided_utc":"2026-10-19","equated":true,"tie_band":0.9,"equating":{"bases":21,"public":{"mean":23.5,"sd":16.54},"same_skills":{"mean":24.27,"sd":16.35},"new_domains":{"mean":16.88,"sd":13.39}},"gap_band":{"bases":21,"sd":0.91}}} \ No newline at end of file diff --git a/data/vision.json b/data/vision.json deleted file mode 100644 index 12747bbb0ff9f396999119baa9cb74d9077b1b2f..0000000000000000000000000000000000000000 --- a/data/vision.json +++ /dev/null @@ -1 +0,0 @@ -{"generated_utc":"2026-10-28T19:32:21Z","edition":"0.4","image_cap_px":1638400,"weights":{"public":0.5,"private":0.5},"bench_weights":{"CV-Bench":1.0,"BLINK":1.0,"RealWorldQA":1.0,"CharXiv":1.0,"InfographicVQA":1.0,"Mind2Web":1.0,"Winoground":1.0,"KIE (CORD+FUNSD)":1.0,"Moderation (Hateful Memes)":1.0,"R-Bench-M":0.5,"MMMU-Pro vision":0.25},"private_sets":["CV-Bench","BLINK","RealWorldQA","CharXiv","InfographicVQA","Mind2Web","Winoground","KIE (CORD+FUNSD)","Moderation (Hateful Memes)"],"entrants":[{"engine":"solomon-v11-bf16-encoding","name":"Solomon v1.1 (27B)","url":"https://huggingface.co/DoccyHealth/Solomon","kind":"head / adapter","technique":"autoregressive","org":{"id":"DoccyHealth","name":"Doccy Health","logo":"data/orgs/DoccyHealth.png"},"pub":69.88,"priv":63.79,"full":66.83,"base":null,"caveat":"Answers up to 8 options: MMMU-Pro rows with more options count as failures.","bench":{"CV-Bench":{"pub":74.74,"private":42.71},"BLINK":{"pub":59.24,"private":59.33},"RealWorldQA":{"pub":76.14,"private":45.33},"CharXiv":{"pub":74.94,"private":58.52},"InfographicVQA":{"pub":94.54,"private":87.33},"Mind2Web":{"pub":79.72,"private":58.67},"Winoground":{"pub":84.25,"private":91.0},"KIE (CORD+FUNSD)":{"pub":92.25,"private":84.16},"Moderation (Hateful Memes)":{"pub":36.36,"private":47.0},"R-Bench-M":{"pub":18.24,"private":null},"MMMU-Pro vision":{"pub":-3.03,"private":null}},"params":27781427952,"active":null,"params_estimated":false,"cal":{"acc":0.7711,"conf":0.8458,"ece":0.0748,"over95":0.0286,"n":12255,"benchmarks":11},"ms_row":561.6,"gpu_h":3.006,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":69.8,"priv":65.52,"full":67.66,"budget":"16.7 MP"},"rank_full":21,"rank_pub":25},{"engine":"xor-1.2","name":"Xor 1.2 (35B-A3B)","url":"https://huggingface.co/juspay/xor","kind":"LoRA","technique":"autoregressive","org":{"id":"juspay","name":"Juspay Technologies","logo":"data/orgs/juspay.png"},"pub":70.21,"priv":58.15,"full":64.18,"base":{"name":"Qwen/Qwen3.6-35B-A3B","pub":69.34,"priv":56.25},"caveat":"Environment rebuilt from the reference image (no lane recipe existed); same model code.","bench":{"CV-Bench":{"pub":76.44,"private":44.05},"BLINK":{"pub":56.99,"private":47.33},"RealWorldQA":{"pub":66.43,"private":45.33},"CharXiv":{"pub":71.18,"private":51.92},"InfographicVQA":{"pub":94.54,"private":75.33},"Mind2Web":{"pub":75.61,"private":55.33},"Winoground":{"pub":84.25,"private":93.0},"KIE (CORD+FUNSD)":{"pub":92.85,"private":87.01},"Moderation (Hateful Memes)":{"pub":38.5,"private":24.0},"R-Bench-M":{"pub":35.39,"private":null},"MMMU-Pro vision":{"pub":40.13,"private":null}},"params":35950000000.0,"active":"A3B","params_estimated":false,"cal":{"acc":0.7625,"conf":0.7644,"ece":0.0104,"over95":0.0079,"n":13695,"benchmarks":11},"ms_row":242.2,"gpu_h":1.297,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":70.32,"priv":58.27,"full":64.3,"budget":"16.7 MP"},"rank_full":25,"rank_pub":24},{"engine":"rune-26b-a4b-v3","name":"Surogate Rune 26B-A4B v3","url":"https://huggingface.co/surogate/rune-26b-a4b-GGUF","kind":"full fine-tune","technique":"autoregressive","org":{"id":"surogate","name":"Surogate","logo":"data/orgs/surogate.png"},"pub":65.27,"priv":51.99,"full":58.63,"base":{"name":"google/gemma-4-26B-A4B-it","pub":66.66,"priv":53.97},"caveat":null,"bench":{"CV-Bench":{"pub":70.21,"private":42.04},"BLINK":{"pub":53.12,"private":4.67},"RealWorldQA":{"pub":54.22,"private":28.0},"CharXiv":{"pub":73.43,"private":52.87},"InfographicVQA":{"pub":89.62,"private":94.0},"Mind2Web":{"pub":63.82,"private":52.0},"Winoground":{"pub":78.5,"private":81.0},"KIE (CORD+FUNSD)":{"pub":90.98,"private":79.29},"Moderation (Hateful Memes)":{"pub":38.17,"private":34.0},"R-Bench-M":{"pub":33.22,"private":null},"MMMU-Pro vision":{"pub":30.72,"private":null}},"params":25805936206,"active":"A4B","params_estimated":false,"cal":{"acc":0.7265,"conf":0.8042,"ece":0.0797,"over95":0.0298,"n":13693,"benchmarks":11},"ms_row":143.4,"gpu_h":0.768,"effective_max_pixels":645120,"effective_budget":"280 soft tokens (~0.65 MP)","effective_budget_source":"fixed by its engine (surogate exposes no image budget setting)","native":{"pub":65.27,"priv":51.99,"full":58.63,"budget":"280 soft tokens (~0.65 MP)"},"rank_full":35,"rank_pub":32},{"engine":"jpt-9b","name":"JPT-9B","url":"https://huggingface.co/kirp/jpt-9b","kind":"LoRA","technique":"autoregressive","org":null,"pub":65.39,"priv":58.04,"full":61.71,"base":{"name":"Qwen/Qwen3.5-9B-Base","pub":63.39,"priv":53.95},"caveat":null,"bench":{"CV-Bench":{"pub":76.04,"private":44.72},"BLINK":{"pub":52.96,"private":38.0},"RealWorldQA":{"pub":65.04,"private":36.67},"CharXiv":{"pub":69.17,"private":50.04},"InfographicVQA":{"pub":93.44,"private":94.67},"Mind2Web":{"pub":69.03,"private":52.67},"Winoground":{"pub":79.25,"private":90.0},"KIE (CORD+FUNSD)":{"pub":91.78,"private":86.6},"Moderation (Hateful Memes)":{"pub":23.17,"private":29.0},"R-Bench-M":{"pub":18.42,"private":null},"MMMU-Pro vision":{"pub":33.77,"private":null}},"params":9653104368,"active":null,"params_estimated":false,"cal":{"acc":0.726,"conf":0.8001,"ece":0.0741,"over95":0.0224,"n":13695,"benchmarks":11},"ms_row":300.8,"gpu_h":1.61,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":65.5,"priv":58.24,"full":61.87,"budget":"16.7 MP"},"rank_full":31,"rank_pub":31},{"engine":"jpt-4b","name":"JPT-4B","url":"https://huggingface.co/kirp/jpt-4b","kind":"LoRA","technique":"autoregressive","org":null,"pub":64.62,"priv":53.44,"full":59.03,"base":{"name":"Qwen/Qwen3.5-4B-Base","pub":59.47,"priv":45.4},"caveat":null,"bench":{"CV-Bench":{"pub":75.88,"private":43.38},"BLINK":{"pub":53.77,"private":43.33},"RealWorldQA":{"pub":62.54,"private":26.67},"CharXiv":{"pub":69.67,"private":46.27},"InfographicVQA":{"pub":90.71,"private":84.67},"Mind2Web":{"pub":66.7,"private":41.33},"Winoground":{"pub":77.0,"private":88.0},"KIE (CORD+FUNSD)":{"pub":90.45,"private":83.35},"Moderation (Hateful Memes)":{"pub":26.3,"private":24.0},"R-Bench-M":{"pub":19.51,"private":null},"MMMU-Pro vision":{"pub":29.0,"private":null}},"params":4659865088,"active":null,"params_estimated":false,"cal":{"acc":0.7177,"conf":0.7749,"ece":0.0572,"over95":0.0183,"n":13695,"benchmarks":11},"ms_row":177.9,"gpu_h":0.952,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":64.83,"priv":54.47,"full":59.65,"budget":"16.7 MP"},"rank_full":33,"rank_pub":35},{"engine":"winnow-12b-q8","name":"Winnow-12B","url":"https://huggingface.co/EldanRing/Winnow-12B","kind":"LoRA","technique":"autoregressive","org":null,"pub":63.86,"priv":53.64,"full":58.75,"base":{"name":"google/gemma-4-12B","pub":17.87,"priv":12.09},"caveat":"Its base answers at chance through the converted projector, so the base comparison is not meaningful.","bench":{"CV-Bench":{"pub":67.54,"private":31.32},"BLINK":{"pub":51.19,"private":22.0},"RealWorldQA":{"pub":52.83,"private":18.67},"CharXiv":{"pub":67.42,"private":59.47},"InfographicVQA":{"pub":85.25,"private":88.67},"Mind2Web":{"pub":76.16,"private":60.0},"Winoground":{"pub":80.75,"private":80.0},"KIE (CORD+FUNSD)":{"pub":90.58,"private":77.66},"Moderation (Hateful Memes)":{"pub":31.57,"private":45.0},"R-Bench-M":{"pub":24.92,"private":null},"MMMU-Pro vision":{"pub":27.67,"private":null}},"params":11959730224,"active":null,"params_estimated":false,"cal":{"acc":0.7143,"conf":0.7909,"ece":0.0823,"over95":0.0308,"n":13695,"benchmarks":11},"ms_row":567.5,"gpu_h":3.038,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"Gemma-4 budget nearest below the 1.6 MP cap","native":{"pub":64.38,"priv":55.29,"full":59.84,"budget":"~1,120 image tokens"},"rank_full":34,"rank_pub":36},{"engine":"neohorse-jev-4b","name":"NeoHorse-Jev-4B","url":"https://huggingface.co/TokenRhythm/NeoHorse-Jev-4B","kind":"head / adapter","technique":"autoregressive","org":{"id":"TokenRhythm","name":"TokenRhythm","logo":"data/orgs/TokenRhythm.png"},"pub":57.14,"priv":44.99,"full":51.06,"base":null,"caveat":"Takes a single image only: multi-image rows (BLINK, Winoground) count as failures.","bench":{"CV-Bench":{"pub":73.77,"private":35.34},"BLINK":{"pub":-2.13,"private":-33.33},"RealWorldQA":{"pub":57.83,"private":27.33},"CharXiv":{"pub":64.16,"private":37.78},"InfographicVQA":{"pub":88.52,"private":76.0},"Mind2Web":{"pub":73.14,"private":56.67},"Winoground":{"pub":77.5,"private":84.0},"KIE (CORD+FUNSD)":{"pub":88.91,"private":83.76},"Moderation (Hateful Memes)":{"pub":20.03,"private":4.0},"R-Bench-M":{"pub":13.55,"private":null},"MMMU-Pro vision":{"pub":25.68,"private":null}},"params":4659865088,"active":null,"params_estimated":false,"cal":{"acc":0.7204,"conf":0.7322,"ece":0.0219,"over95":0.0126,"n":12321,"benchmarks":11},"ms_row":103.6,"gpu_h":0.608,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":57.22,"priv":45.88,"full":51.55,"budget":"16.7 MP"},"rank_full":47,"rank_pub":47},{"engine":"jev-omni","name":"Jev-Omni (12B)","url":"https://huggingface.co/akhilaaa3/Jev-Omni","kind":"LoRA + head","technique":"autoregressive","org":null,"pub":60.1,"priv":52.05,"full":56.08,"base":null,"caveat":"Takes a single image only: multi-image rows (BLINK, Winoground) count as failures.","bench":{"CV-Bench":{"pub":73.53,"private":37.69},"BLINK":{"pub":-4.55,"private":-33.33},"RealWorldQA":{"pub":55.88,"private":20.67},"CharXiv":{"pub":72.18,"private":51.92},"InfographicVQA":{"pub":85.25,"private":76.0},"Mind2Web":{"pub":72.05,"private":59.33},"Winoground":{"pub":79.75,"private":84.0},"KIE (CORD+FUNSD)":{"pub":91.85,"private":76.85},"Moderation (Hateful Memes)":{"pub":42.62,"private":62.0},"R-Bench-M":{"pub":11.02,"private":null},"MMMU-Pro vision":{"pub":29.59,"private":null}},"params":11959730224,"active":null,"params_estimated":false,"cal":{"acc":0.7324,"conf":0.8175,"ece":0.0853,"over95":0.0288,"n":12321,"benchmarks":11},"ms_row":269.2,"gpu_h":0.551,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"Gemma-4 budget nearest below the 1.6 MP cap","native":{"pub":60.47,"priv":53.07,"full":56.77,"budget":"1,120 soft tokens (~2.6 MP)"},"rank_full":43,"rank_pub":44},{"engine":"winnow-e4b-q8","name":"Winnow-E4B","url":"https://huggingface.co/EldanRing/Winnow-E4B","kind":"LoRA","technique":"autoregressive","org":null,"pub":48.06,"priv":36.06,"full":42.06,"base":{"name":"google/gemma-4-E4B","pub":44.97,"priv":38.46},"caveat":null,"bench":{"CV-Bench":{"pub":49.49,"private":17.25},"BLINK":{"pub":34.76,"private":10.0},"RealWorldQA":{"pub":35.08,"private":10.0},"CharXiv":{"pub":43.36,"private":18.93},"InfographicVQA":{"pub":83.06,"private":61.33},"Mind2Web":{"pub":53.27,"private":46.0},"Winoground":{"pub":63.25,"private":68.0},"KIE (CORD+FUNSD)":{"pub":80.03,"private":74.01},"Moderation (Hateful Memes)":{"pub":15.91,"private":19.0},"R-Bench-M":{"pub":15.18,"private":null},"MMMU-Pro vision":{"pub":11.16,"private":null}},"params":7996156490,"active":null,"params_estimated":false,"cal":{"acc":0.6104,"conf":0.6287,"ece":0.0495,"over95":0.0185,"n":13695,"benchmarks":11},"ms_row":152.6,"gpu_h":0.817,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"Gemma-4 budget nearest below the 1.6 MP cap","native":{"pub":48.46,"priv":38.37,"full":43.41,"budget":"~1,120 image tokens"},"rank_full":58,"rank_pub":55},{"engine":"jpt-0.8b","name":"JPT-0.8B","url":"https://huggingface.co/kirp/jpt-0.8b","kind":"LoRA","technique":"autoregressive","org":null,"pub":44.55,"priv":39.65,"full":42.1,"base":{"name":"Qwen/Qwen3.5-0.8B-Base","pub":34.35,"priv":30.37},"caveat":null,"bench":{"CV-Bench":{"pub":51.19,"private":26.3},"BLINK":{"pub":33.63,"private":38.67},"RealWorldQA":{"pub":40.9,"private":22.67},"CharXiv":{"pub":52.13,"private":33.07},"InfographicVQA":{"pub":74.86,"private":54.67},"Mind2Web":{"pub":37.92,"private":33.33},"Winoground":{"pub":48.0,"private":62.0},"KIE (CORD+FUNSD)":{"pub":73.89,"private":69.14},"Moderation (Hateful Memes)":{"pub":12.78,"private":17.0},"R-Bench-M":{"pub":13.73,"private":null},"MMMU-Pro vision":{"pub":8.64,"private":null}},"params":873438784,"active":null,"params_estimated":false,"cal":{"acc":0.5892,"conf":0.6544,"ece":0.0652,"over95":0.0075,"n":13695,"benchmarks":11},"ms_row":115.7,"gpu_h":0.619,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":44.72,"priv":40.98,"full":42.85,"budget":"16.7 MP"},"rank_full":57,"rank_pub":59},{"engine":"djev","name":"djev (26B-A4B)","url":"https://github.com/Davipar/djev-dev","kind":"inference technique","technique":"diffusion","org":null,"pub":55.2,"priv":40.35,"full":47.78,"base":null,"caveat":"Ran with its published canvas 128 and 32K context, since the long-context engine takes no images.","bench":{"CV-Bench":{"pub":62.6,"private":26.97},"BLINK":{"pub":41.52,"private":3.33},"RealWorldQA":{"pub":50.06,"private":24.0},"CharXiv":{"pub":39.6,"private":25.53},"InfographicVQA":{"pub":73.77,"private":67.33},"Mind2Web":{"pub":64.51,"private":51.33},"Winoground":{"pub":54.5,"private":66.0},"KIE (CORD+FUNSD)":{"pub":89.78,"private":77.66},"Moderation (Hateful Memes)":{"pub":41.3,"private":21.0},"R-Bench-M":{"pub":24.92,"private":null},"MMMU-Pro vision":{"pub":32.51,"private":null}},"params":25823778864,"active":"A4B","params_estimated":false,"cal":{"acc":0.671,"conf":0.8147,"ece":0.1439,"over95":0.0586,"n":13695,"benchmarks":11},"ms_row":154.1,"gpu_h":0.412,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"largest budget its vision server accepts (sliding-window limit 1,024)","native":{"pub":55.2,"priv":40.35,"full":47.78,"budget":"560 soft tokens (~1.3 MP)"},"rank_full":50,"rank_pub":49},{"engine":"razorback-one-read-3e296f08","name":"razorback16 diffgemma (26B-A4B)","url":"https://github.com/vllm-project/vllm/pull/57250","kind":"inference technique","technique":"diffusion","org":null,"pub":54.45,"priv":41.02,"full":47.74,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":61.87,"private":28.98},"BLINK":{"pub":53.77,"private":5.33},"RealWorldQA":{"pub":50.61,"private":26.0},"CharXiv":{"pub":33.33,"private":21.76},"InfographicVQA":{"pub":72.68,"private":67.33},"Mind2Web":{"pub":66.02,"private":61.33},"Winoground":{"pub":50.75,"private":67.0},"KIE (CORD+FUNSD)":{"pub":89.65,"private":78.48},"Moderation (Hateful Memes)":{"pub":33.39,"private":13.0},"R-Bench-M":{"pub":23.3,"private":null},"MMMU-Pro vision":{"pub":28.73,"private":null}},"params":25823778864,"active":"A4B","params_estimated":false,"cal":{"acc":0.6655,"conf":0.8398,"ece":0.1743,"over95":0.0699,"n":13695,"benchmarks":11},"ms_row":255.2,"gpu_h":0.209,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"Gemma-4 budget nearest below the 1.6 MP cap","native":{"pub":54.7,"priv":40.5,"full":47.6,"budget":"1,120 soft tokens (~2.6 MP)"},"rank_full":51,"rank_pub":50},{"engine":"vllm-pr57250","name":"mmastrac diffgemma (26B-A4B)","url":"https://github.com/vllm-project/vllm/pull/57250","kind":"inference technique","technique":"diffusion","org":null,"pub":53.45,"priv":39.23,"full":46.34,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":64.06,"private":28.64},"BLINK":{"pub":53.44,"private":11.33},"RealWorldQA":{"pub":46.45,"private":15.33},"CharXiv":{"pub":32.83,"private":13.28},"InfographicVQA":{"pub":71.04,"private":69.33},"Mind2Web":{"pub":68.48,"private":58.0},"Winoground":{"pub":41.75,"private":73.0},"KIE (CORD+FUNSD)":{"pub":88.91,"private":69.14},"Moderation (Hateful Memes)":{"pub":36.19,"private":15.0},"R-Bench-M":{"pub":23.3,"private":null},"MMMU-Pro vision":{"pub":25.22,"private":null}},"params":25823778864,"active":"A4B","params_estimated":false,"cal":{"acc":0.6605,"conf":0.796,"ece":0.1355,"over95":0.0436,"n":13695,"benchmarks":11},"ms_row":456.9,"gpu_h":0.454,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"Gemma-4 budget nearest below the 1.6 MP cap","native":{"pub":55.75,"priv":42.82,"full":49.28,"budget":"1,120 soft tokens (~2.6 MP)"},"rank_full":52,"rank_pub":51},{"engine":"intern-decision-4b","name":"Intern-Decision-4B","url":"https://huggingface.co/internlm/Intern-Decision-4B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"},"pub":61.69,"priv":51.29,"full":56.49,"base":{"name":"Qwen/Qwen3.5-4B-Base","pub":56.88,"priv":40.07},"caveat":"The vision patch embed was swapped for the identical linear map to run at normal speed; outputs agree to bf16 precision.","bench":{"CV-Bench":{"pub":74.1,"private":36.68},"BLINK":{"pub":53.77,"private":36.0},"RealWorldQA":{"pub":58.94,"private":30.0},"CharXiv":{"pub":57.14,"private":41.56},"InfographicVQA":{"pub":83.06,"private":67.33},"Mind2Web":{"pub":65.88,"private":43.33},"Winoground":{"pub":77.0,"private":87.0},"KIE (CORD+FUNSD)":{"pub":87.98,"private":77.66},"Moderation (Hateful Memes)":{"pub":26.63,"private":42.0},"R-Bench-M":{"pub":19.15,"private":null},"MMMU-Pro vision":{"pub":29.53,"private":null}},"params":4659865088,"active":null,"params_estimated":false,"cal":{"acc":0.6981,"conf":0.6876,"ece":0.0137,"over95":0.0019,"n":13695,"benchmarks":11},"ms_row":142.4,"gpu_h":0.762,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":61.83,"priv":51.95,"full":56.89,"budget":"16.7 MP"},"rank_full":42,"rank_pub":43},{"engine":"intern-decision-2b","name":"Intern-Decision-2B","url":"https://huggingface.co/internlm/Intern-Decision-2B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"},"pub":52.92,"priv":42.71,"full":47.82,"base":{"name":"Qwen/Qwen3.5-2B-Base","pub":43.52,"priv":34.1},"caveat":"The vision patch embed was swapped for the identical linear map to run at normal speed; outputs agree to bf16 precision.","bench":{"CV-Bench":{"pub":65.03,"private":36.35},"BLINK":{"pub":42.97,"private":26.0},"RealWorldQA":{"pub":50.34,"private":34.67},"CharXiv":{"pub":58.15,"private":28.36},"InfographicVQA":{"pub":78.69,"private":58.0},"Mind2Web":{"pub":49.98,"private":42.0},"Winoground":{"pub":63.25,"private":77.0},"KIE (CORD+FUNSD)":{"pub":84.44,"private":74.01},"Moderation (Hateful Memes)":{"pub":13.93,"private":8.0},"R-Bench-M":{"pub":10.48,"private":null},"MMMU-Pro vision":{"pub":15.87,"private":null}},"params":2274069824,"active":null,"params_estimated":false,"cal":{"acc":0.6372,"conf":0.6167,"ece":0.0324,"over95":0.0018,"n":13695,"benchmarks":11},"ms_row":106.8,"gpu_h":0.572,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":53.29,"priv":43.42,"full":48.36,"budget":"16.7 MP"},"rank_full":49,"rank_pub":53},{"engine":"intern-decision-0.8b","name":"Intern-Decision-0.8B","url":"https://huggingface.co/internlm/Intern-Decision-0.8B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"},"pub":38.96,"priv":33.71,"full":36.34,"base":{"name":"Qwen/Qwen3.5-0.8B-Base","pub":29.14,"priv":25.25},"caveat":"The vision patch embed was swapped for the identical linear map to run at normal speed; outputs agree to bf16 precision.","bench":{"CV-Bench":{"pub":51.67,"private":20.6},"BLINK":{"pub":24.93,"private":20.0},"RealWorldQA":{"pub":38.13,"private":19.33},"CharXiv":{"pub":48.37,"private":21.76},"InfographicVQA":{"pub":71.58,"private":59.33},"Mind2Web":{"pub":21.21,"private":19.33},"Winoground":{"pub":43.75,"private":71.0},"KIE (CORD+FUNSD)":{"pub":63.0,"private":61.02},"Moderation (Hateful Memes)":{"pub":12.45,"private":11.0},"R-Bench-M":{"pub":5.79,"private":null},"MMMU-Pro vision":{"pub":7.65,"private":null}},"params":873438784,"active":null,"params_estimated":false,"cal":{"acc":0.5515,"conf":0.4987,"ece":0.0613,"over95":0.0002,"n":13695,"benchmarks":11},"ms_row":74.4,"gpu_h":8.663,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":38.78,"priv":34.95,"full":36.86,"budget":"16.7 MP"},"rank_full":60,"rank_pub":60},{"engine":"clef-omni","name":"Cloudflare clef-omni (30B-A3B)","url":"https://huggingface.co/Cloudflare/clef-omni","kind":"full fine-tune","technique":"autoregressive","org":{"id":"Cloudflare","name":"Cloudflare","logo":"data/orgs/Cloudflare.png"},"pub":63.68,"priv":55.4,"full":59.54,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":73.37,"private":40.7},"BLINK":{"pub":48.45,"private":38.67},"RealWorldQA":{"pub":61.99,"private":36.67},"CharXiv":{"pub":61.15,"private":30.24},"InfographicVQA":{"pub":87.43,"private":85.33},"Mind2Web":{"pub":74.79,"private":57.33},"Winoground":{"pub":75.0,"private":84.0},"KIE (CORD+FUNSD)":{"pub":89.05,"private":79.7},"Moderation (Hateful Memes)":{"pub":31.24,"private":46.0},"R-Bench-M":{"pub":25.64,"private":null},"MMMU-Pro vision":{"pub":22.37,"private":null}},"params":31830000000.0,"active":"A3B","params_estimated":false,"cal":{"acc":0.7096,"conf":0.7519,"ece":0.0422,"over95":0.0166,"n":13695,"benchmarks":11},"ms_row":281.2,"gpu_h":1.505,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap 1,048,576 px lifted to the board cap via the documented media_kwargs (processor max 12.8 MP)","native":{"pub":63.68,"priv":55.4,"full":59.54,"budget":"1.6 MP"},"rank_full":32,"rank_pub":38},{"engine":"vllm-sr-d3-v310","name":"vLLM-SR d3 (27B)","url":"https://huggingface.co/vllm-sr/d3/tree/v3.1.0","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":74.08,"priv":68.45,"full":71.26,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":76.36,"private":43.72},"BLINK":{"pub":60.21,"private":65.33},"RealWorldQA":{"pub":73.92,"private":46.67},"CharXiv":{"pub":79.45,"private":71.72},"InfographicVQA":{"pub":97.81,"private":100.0},"Mind2Web":{"pub":83.28,"private":66.0},"Winoground":{"pub":85.75,"private":93.0},"KIE (CORD+FUNSD)":{"pub":92.59,"private":86.6},"Moderation (Hateful Memes)":{"pub":44.6,"private":43.0},"R-Bench-M":{"pub":33.4,"private":null},"MMMU-Pro vision":{"pub":46.37,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.7918,"conf":0.8052,"ece":0.0157,"over95":0.0102,"n":13695,"benchmarks":11},"ms_row":299.1,"gpu_h":1.601,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":74.08,"priv":68.45,"full":71.26,"budget":"1.6 MP"},"rank_full":10,"rank_pub":14},{"engine":"vllm-sr-d3-flash","name":"vLLM-SR d3-flash (9B)","url":"https://huggingface.co/vllm-sr/d3-flash","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":69.87,"priv":63.91,"full":66.89,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":75.39,"private":44.39},"BLINK":{"pub":56.02,"private":46.67},"RealWorldQA":{"pub":62.27,"private":37.33},"CharXiv":{"pub":73.93,"private":54.75},"InfographicVQA":{"pub":96.17,"private":100.0},"Mind2Web":{"pub":80.82,"private":64.0},"Winoground":{"pub":82.75,"private":90.0},"KIE (CORD+FUNSD)":{"pub":92.92,"private":87.01},"Moderation (Hateful Memes)":{"pub":40.97,"private":51.0},"R-Bench-M":{"pub":20.77,"private":null},"MMMU-Pro vision":{"pub":38.28,"private":null}},"params":8392695024.0,"active":null,"params_estimated":false,"cal":{"acc":0.7562,"conf":0.7789,"ece":0.0235,"over95":0.0085,"n":13695,"benchmarks":11},"ms_row":150.0,"gpu_h":0.803,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":69.87,"priv":63.91,"full":66.89,"budget":"1.6 MP"},"rank_full":20,"rank_pub":26},{"engine":"vllm-sr-d3-nano","name":"vLLM-SR d3-nano (2B)","url":"https://huggingface.co/vllm-sr/d3-nano","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":59.15,"priv":50.19,"full":54.67,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":71.91,"private":33.0},"BLINK":{"pub":50.38,"private":28.0},"RealWorldQA":{"pub":53.11,"private":28.67},"CharXiv":{"pub":65.16,"private":36.84},"InfographicVQA":{"pub":83.61,"private":84.67},"Mind2Web":{"pub":63.69,"private":50.67},"Winoground":{"pub":69.0,"private":78.0},"KIE (CORD+FUNSD)":{"pub":83.97,"private":80.91},"Moderation (Hateful Memes)":{"pub":23.66,"private":31.0},"R-Bench-M":{"pub":15.54,"private":null},"MMMU-Pro vision":{"pub":17.79,"private":null}},"params":2213241664.0,"active":null,"params_estimated":false,"cal":{"acc":0.6777,"conf":0.751,"ece":0.0733,"over95":0.0175,"n":13695,"benchmarks":11},"ms_row":93.4,"gpu_h":0.5,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":59.15,"priv":50.19,"full":54.67,"budget":"1.6 MP"},"rank_full":45,"rank_pub":45},{"engine":"vllm-sr-d3-mini","name":"vLLM-SR d3-mini (4B)","url":"https://huggingface.co/vllm-sr/d3-mini","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":67.67,"priv":58.65,"full":63.16,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":73.94,"private":40.7},"BLINK":{"pub":53.44,"private":40.67},"RealWorldQA":{"pub":57.83,"private":32.0},"CharXiv":{"pub":73.93,"private":53.81},"InfographicVQA":{"pub":94.54,"private":94.0},"Mind2Web":{"pub":77.53,"private":64.67},"Winoground":{"pub":81.5,"private":88.0},"KIE (CORD+FUNSD)":{"pub":92.19,"private":84.97},"Moderation (Hateful Memes)":{"pub":37.35,"private":29.0},"R-Bench-M":{"pub":18.78,"private":null},"MMMU-Pro vision":{"pub":32.71,"private":null}},"params":4539265536.0,"active":null,"params_estimated":false,"cal":{"acc":0.7378,"conf":0.765,"ece":0.0279,"over95":0.0075,"n":13695,"benchmarks":11},"ms_row":117.7,"gpu_h":0.63,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":67.67,"priv":58.65,"full":63.16,"budget":"1.6 MP"},"rank_full":29,"rank_pub":29},{"engine":"vllm-sr-d3-lite","name":"vLLM-SR d3-lite (0.8B)","url":"https://huggingface.co/vllm-sr/d3-lite","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":47.88,"priv":36.33,"full":42.11,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":65.03,"private":24.29},"BLINK":{"pub":38.14,"private":21.33},"RealWorldQA":{"pub":36.18,"private":20.67},"CharXiv":{"pub":53.13,"private":18.93},"InfographicVQA":{"pub":73.22,"private":57.33},"Mind2Web":{"pub":50.67,"private":36.67},"Winoground":{"pub":54.5,"private":67.0},"KIE (CORD+FUNSD)":{"pub":76.29,"private":72.79},"Moderation (Hateful Memes)":{"pub":13.11,"private":8.0},"R-Bench-M":{"pub":9.4,"private":null},"MMMU-Pro vision":{"pub":7.25,"private":null}},"params":852985920.0,"active":null,"params_estimated":false,"cal":{"acc":0.6034,"conf":0.6804,"ece":0.077,"over95":0.0155,"n":13695,"benchmarks":11},"ms_row":80.1,"gpu_h":0.429,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":47.88,"priv":36.33,"full":42.11,"budget":"1.6 MP"},"rank_full":56,"rank_pub":56},{"engine":"vllm-sr-d3-edge","name":"vLLM-SR d3-edge (0.6B)","url":"https://huggingface.co/vllm-sr/d3-edge","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":36.61,"priv":26.81,"full":31.71,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":54.02,"private":17.92},"BLINK":{"pub":34.76,"private":-6.67},"RealWorldQA":{"pub":25.64,"private":24.0},"CharXiv":{"pub":36.09,"private":17.05},"InfographicVQA":{"pub":62.84,"private":42.67},"Mind2Web":{"pub":38.47,"private":32.0},"Winoground":{"pub":27.75,"private":41.0},"KIE (CORD+FUNSD)":{"pub":62.46,"private":60.61},"Moderation (Hateful Memes)":{"pub":9.15,"private":6.0},"R-Bench-M":{"pub":9.4,"private":null},"MMMU-Pro vision":{"pub":4.2,"private":null}},"params":594321600.0,"active":null,"params_estimated":false,"cal":{"acc":0.5437,"conf":0.6412,"ece":0.0976,"over95":0.0186,"n":13695,"benchmarks":11},"ms_row":62.7,"gpu_h":0.335,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":36.61,"priv":26.81,"full":31.71,"budget":"1.6 MP"},"rank_full":61,"rank_pub":61},{"engine":"rsi-jev-v6.1-vl-27b-auto","name":"RSI-Jev v6.1 VL 27B (effort auto)","url":"https://huggingface.co/shgao/rsi-jev-v6.1-vl-27b","kind":"LoRA","technique":"autoregressive","org":null,"pub":68.67,"priv":59.36,"full":64.01,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":77.82,"private":38.69},"BLINK":{"pub":53.28,"private":58.0},"RealWorldQA":{"pub":65.6,"private":32.67},"CharXiv":{"pub":79.45,"private":61.35},"InfographicVQA":{"pub":93.99,"private":95.33},"Mind2Web":{"pub":75.75,"private":56.0},"Winoground":{"pub":82.25,"private":83.0},"KIE (CORD+FUNSD)":{"pub":91.05,"private":84.16},"Moderation (Hateful Memes)":{"pub":29.6,"private":25.0},"R-Bench-M":{"pub":23.48,"private":null},"MMMU-Pro vision":{"pub":35.96,"private":null}},"params":27781427952.0,"active":null,"params_estimated":false,"cal":{"acc":0.748,"conf":0.7846,"ece":0.0372,"over95":0.0201,"n":13695,"benchmarks":11},"ms_row":200.4,"gpu_h":1.073,"effective_max_pixels":1048576,"effective_budget":"1,024 image tokens (~1 MP)","effective_budget_source":"fixed by its engine (not settable), below the 1.6 MP board cap","native":{"pub":68.67,"priv":59.36,"full":64.01,"budget":"1,024 image tokens (~1 MP)"},"rank_full":26,"rank_pub":27},{"engine":"matilda-jev-v1.5","name":"matilda-jev-v1.5 (27B)","url":null,"kind":null,"technique":null,"org":null,"pub":65.42,"priv":58.67,"full":62.05,"base":null,"caveat":"Ran through the packaged server.py: the in-process text engine takes no images.","bench":{"CV-Bench":{"pub":76.44,"private":44.05},"BLINK":{"pub":53.28,"private":62.67},"RealWorldQA":{"pub":58.94,"private":34.67},"CharXiv":{"pub":75.94,"private":56.64},"InfographicVQA":{"pub":77.05,"private":91.33},"Mind2Web":{"pub":63.82,"private":42.67},"Winoground":{"pub":79.75,"private":82.0},"KIE (CORD+FUNSD)":{"pub":90.38,"private":74.01},"Moderation (Hateful Memes)":{"pub":36.85,"private":40.0},"R-Bench-M":{"pub":37.19,"private":null},"MMMU-Pro vision":{"pub":27.21,"private":null}},"params":null,"active":null,"params_estimated":false,"cal":{"acc":0.7278,"conf":0.7985,"ece":0.0714,"over95":0.0172,"n":13695,"benchmarks":11},"ms_row":137.8,"gpu_h":0.738,"effective_max_pixels":262144,"effective_budget":"0.26 MP","effective_budget_source":"fixed by its engine (not settable), below the 1.6 MP board cap","native":{"pub":65.42,"priv":58.67,"full":62.05,"budget":"0.26 MP"},"rank_full":30,"rank_pub":30},{"engine":"xor-26b-a4b-v1.1","name":"xor-26b-a4b-v1.1","url":null,"kind":null,"technique":null,"org":null,"pub":65.06,"priv":50.93,"full":58.0,"base":null,"caveat":"Non-RGB images are converted to RGB on the client: its SGLang Gemma-4 vision path crashes on them.","bench":{"CV-Bench":{"pub":71.99,"private":39.03},"BLINK":{"pub":52.96,"private":9.33},"RealWorldQA":{"pub":50.34,"private":24.0},"CharXiv":{"pub":71.93,"private":45.33},"InfographicVQA":{"pub":87.43,"private":92.67},"Mind2Web":{"pub":65.19,"private":52.0},"Winoground":{"pub":82.25,"private":82.0},"KIE (CORD+FUNSD)":{"pub":91.45,"private":76.04},"Moderation (Hateful Memes)":{"pub":34.87,"private":38.0},"R-Bench-M":{"pub":36.29,"private":null},"MMMU-Pro vision":{"pub":30.98,"private":null}},"params":null,"active":"A4B","params_estimated":false,"cal":{"acc":0.7249,"conf":0.6416,"ece":0.0833,"over95":0.0015,"n":13695,"benchmarks":11},"ms_row":135.0,"gpu_h":0.825,"effective_max_pixels":645120,"effective_budget":"280 soft tokens (~0.65 MP)","effective_budget_source":"fixed by its engine (not settable), below the 1.6 MP board cap","native":{"pub":65.06,"priv":50.93,"full":58.0,"budget":"280 soft tokens (~0.65 MP)"},"rank_full":37,"rank_pub":34},{"engine":"rsi-jev-v6.1-vl-4b","name":"RSI-Jev v6.1 VL 4B","url":"https://huggingface.co/shgao/rsi-jev-v6.1-vl-4b","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":63.01,"priv":51.83,"full":57.42,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":72.8,"private":36.01},"BLINK":{"pub":50.06,"private":38.67},"RealWorldQA":{"pub":56.44,"private":27.33},"CharXiv":{"pub":69.67,"private":47.21},"InfographicVQA":{"pub":92.35,"private":88.67},"Mind2Web":{"pub":59.3,"private":45.33},"Winoground":{"pub":76.5,"private":86.0},"KIE (CORD+FUNSD)":{"pub":86.04,"private":79.29},"Moderation (Hateful Memes)":{"pub":35.7,"private":18.0},"R-Bench-M":{"pub":18.97,"private":null},"MMMU-Pro vision":{"pub":24.02,"private":null}},"params":4660000000.0,"active":null,"params_estimated":false,"cal":{"acc":0.7036,"conf":0.7434,"ece":0.0412,"over95":0.0188,"n":13695,"benchmarks":11},"ms_row":86.0,"gpu_h":0.46,"effective_max_pixels":1048576,"effective_budget":"1,024 image tokens (~1 MP)","effective_budget_source":"fixed by its engine (not settable), below the 1.6 MP board cap","native":{"pub":63.01,"priv":51.83,"full":57.42,"budget":"1,024 image tokens (~1 MP)"},"rank_full":38,"rank_pub":39},{"engine":"aplomb-1","name":"Aplomb 1","url":"https://huggingface.co/empiriolabsai/aplomb-1","kind":"LoRA + head","technique":"autoregressive","org":{"id":"empiriolabsai","name":"EmpirioLabs AI","logo":"data/orgs/empiriolabsai.png"},"pub":63.75,"priv":50.8,"full":57.27,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":75.88,"private":40.03},"BLINK":{"pub":52.48,"private":46.67},"RealWorldQA":{"pub":60.32,"private":25.33},"CharXiv":{"pub":68.67,"private":49.1},"InfographicVQA":{"pub":90.71,"private":82.67},"Mind2Web":{"pub":60.12,"private":41.33},"Winoground":{"pub":79.5,"private":82.0},"KIE (CORD+FUNSD)":{"pub":88.51,"private":78.07},"Moderation (Hateful Memes)":{"pub":24.65,"private":12.0},"R-Bench-M":{"pub":29.61,"private":null},"MMMU-Pro vision":{"pub":23.49,"private":null}},"params":5295891968.0,"active":null,"params_estimated":false,"cal":{"acc":0.7139,"conf":0.7617,"ece":0.0487,"over95":0.0202,"n":13695,"benchmarks":11},"ms_row":130.4,"gpu_h":0.698,"effective_max_pixels":655360,"effective_budget":"640 image tokens (~0.66 MP)","effective_budget_source":"fixed by its engine (not settable), below the 1.6 MP board cap","native":{"pub":63.75,"priv":50.8,"full":57.27,"budget":"640 image tokens (~0.66 MP)"},"rank_full":40,"rank_pub":37},{"engine":"shisa-de-2-repeat","name":"Shisa DE-2 (repeat)","url":"https://huggingface.co/shisa-ai/shisa-de-2","kind":"LoRA","technique":"autoregressive","org":{"id":"shisa-ai","name":"Shisa.AI","logo":"data/orgs/shisa-ai.png"},"pub":58.26,"priv":47.44,"full":52.85,"base":null,"caveat":"Takes a single image per question: 1,696 multi-image rows count as failures. Read with 512 logprobs (README setting); non-RGB images converted to RGB on the client.","bench":{"CV-Bench":{"pub":68.59,"private":37.35},"BLINK":{"pub":-6.0,"private":-33.33},"RealWorldQA":{"pub":53.11,"private":20.67},"CharXiv":{"pub":67.42,"private":50.04},"InfographicVQA":{"pub":90.16,"private":93.33},"Mind2Web":{"pub":60.67,"private":40.67},"Winoground":{"pub":76.5,"private":83.0},"KIE (CORD+FUNSD)":{"pub":90.58,"private":78.88},"Moderation (Hateful Memes)":{"pub":38.83,"private":23.0},"R-Bench-M":{"pub":29.79,"private":null},"MMMU-Pro vision":{"pub":29.19,"private":null}},"params":25805933872,"active":"A4B","params_estimated":false,"cal":{"acc":0.7287,"conf":0.8779,"ece":0.1493,"over95":0.0815,"n":12321,"benchmarks":11},"ms_row":83.5,"gpu_h":0.447,"effective_max_pixels":645120,"effective_budget":"280 soft tokens (~0.65 MP)","effective_budget_source":"fixed by its engine (not settable), below the 1.6 MP board cap","native":{"pub":58.26,"priv":47.44,"full":52.85,"budget":"280 soft tokens (~0.65 MP)"},"rank_full":46,"rank_pub":46},{"engine":"pplx-decider-v1.2","name":"Perplexity Decider v1.2 (27B)","url":"https://huggingface.co/perplexity-ai/pplx-decider-v1.2","kind":"full fine-tune","technique":"autoregressive","org":{"id":"perplexity-ai","name":"Perplexity","logo":"data/orgs/perplexity-ai.png"},"pub":77.25,"priv":66.18,"full":71.72,"base":null,"caveat":"Ran through the checkpoint's own pplx_decider engine (the submitted text adapter), which accepts images.","bench":{"CV-Bench":{"pub":77.58,"private":45.39},"BLINK":{"pub":63.27,"private":63.33},"RealWorldQA":{"pub":73.64,"private":45.33},"CharXiv":{"pub":83.46,"private":72.66},"InfographicVQA":{"pub":98.36,"private":99.33},"Mind2Web":{"pub":93.29,"private":70.0},"Winoground":{"pub":87.75,"private":94.0},"KIE (CORD+FUNSD)":{"pub":92.45,"private":86.6},"Moderation (Hateful Memes)":{"pub":55.32,"private":19.0},"R-Bench-M":{"pub":34.49,"private":null},"MMMU-Pro vision":{"pub":43.38,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.8081,"conf":0.8085,"ece":0.011,"over95":0.0055,"n":13695,"benchmarks":11},"ms_row":394.5,"gpu_h":2.112,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap (decision_config.json max_pixels)","native":{"pub":77.25,"priv":66.18,"full":71.72,"budget":"1.6 MP"},"rank_full":6,"rank_pub":3},{"engine":"pplx-decider-flash-v1.2","name":"Perplexity Decider Flash v1.2 (4B)","url":"https://huggingface.co/perplexity-ai/pplx-decider-flash-v1.2","kind":"full fine-tune","technique":"autoregressive","org":{"id":"perplexity-ai","name":"Perplexity","logo":"data/orgs/perplexity-ai.png"},"pub":72.55,"priv":59.48,"full":66.02,"base":null,"caveat":"Ran through the checkpoint's own pplx_decider engine (the submitted text adapter), which accepts images.","bench":{"CV-Bench":{"pub":75.64,"private":43.38},"BLINK":{"pub":55.54,"private":44.67},"RealWorldQA":{"pub":63.1,"private":32.67},"CharXiv":{"pub":76.94,"private":50.04},"InfographicVQA":{"pub":93.99,"private":98.67},"Mind2Web":{"pub":92.87,"private":67.33},"Winoground":{"pub":83.5,"private":91.0},"KIE (CORD+FUNSD)":{"pub":91.72,"private":86.6},"Moderation (Hateful Memes)":{"pub":54.49,"private":21.0},"R-Bench-M":{"pub":23.48,"private":null},"MMMU-Pro vision":{"pub":31.51,"private":null}},"params":4539265536.0,"active":null,"params_estimated":false,"cal":{"acc":0.7663,"conf":0.766,"ece":0.0171,"over95":0.0035,"n":13695,"benchmarks":11},"ms_row":173.9,"gpu_h":0.931,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap (decision_config.json max_pixels)","native":{"pub":72.55,"priv":59.48,"full":66.02,"budget":"1.6 MP"},"rank_full":22,"rank_pub":21},{"engine":"startlux-decision-v1.1-27b-instant","name":"StartLux-Decision v1.1 27B","url":"https://huggingface.co/startlux-models/StartLux-Decision-v1.1-27B-Instant","kind":"full fine-tune","technique":"autoregressive","org":{"id":"startlux-models","name":"StartLux","logo":"data/orgs/startlux-models.png"},"pub":73.83,"priv":69.39,"full":71.61,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":76.2,"private":45.39},"BLINK":{"pub":60.53,"private":64.67},"RealWorldQA":{"pub":71.98,"private":50.67},"CharXiv":{"pub":79.2,"private":73.61},"InfographicVQA":{"pub":98.36,"private":100.0},"Mind2Web":{"pub":84.52,"private":68.0},"Winoground":{"pub":86.5,"private":91.0},"KIE (CORD+FUNSD)":{"pub":92.65,"private":84.16},"Moderation (Hateful Memes)":{"pub":43.28,"private":47.0},"R-Bench-M":{"pub":30.15,"private":null},"MMMU-Pro vision":{"pub":46.1,"private":null}},"params":27781427952.0,"active":null,"params_estimated":false,"cal":{"acc":0.7879,"conf":0.8121,"ece":0.0246,"over95":0.0117,"n":13695,"benchmarks":11},"ms_row":251.9,"gpu_h":1.349,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":73.83,"priv":69.39,"full":71.61,"budget":"1.6 MP"},"rank_full":7,"rank_pub":15},{"engine":"knowline-35b-a3b-gen1","name":"KnowLine 35B-A3B Gen1","url":"https://huggingface.co/PelaAI/KnowLine-35B-A3B-Gen1","kind":"LoRA","technique":"autoregressive","org":{"id":"PelaAI","name":"PelaAI","logo":"data/orgs/PelaAI.png"},"pub":72.64,"priv":63.59,"full":68.11,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":76.04,"private":45.73},"BLINK":{"pub":56.83,"private":56.0},"RealWorldQA":{"pub":70.87,"private":42.67},"CharXiv":{"pub":79.45,"private":62.29},"InfographicVQA":{"pub":99.45,"private":98.67},"Mind2Web":{"pub":79.99,"private":49.33},"Winoground":{"pub":79.25,"private":87.0},"KIE (CORD+FUNSD)":{"pub":92.72,"private":88.63},"Moderation (Hateful Memes)":{"pub":47.73,"private":42.0},"R-Bench-M":{"pub":29.61,"private":null},"MMMU-Pro vision":{"pub":44.24,"private":null}},"params":35951822704,"active":null,"params_estimated":false,"cal":{"acc":0.7793,"conf":0.7858,"ece":0.0105,"over95":0.005,"n":13695,"benchmarks":11},"ms_row":158.7,"gpu_h":0.849,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap applied client-side (kit data_url encoding); model processor allows 16.8 MP","native":{"pub":72.64,"priv":63.59,"full":68.11,"budget":"1.6 MP"},"rank_full":19,"rank_pub":20},{"engine":"amos-1-large","name":"Amos 1 Large (27B)","url":"https://huggingface.co/mukaddamzaid/amos-1-large","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":76.1,"priv":68.71,"full":72.4,"base":null,"caveat":"26 requests over the author's 4 MiB body limit count as wrong (9 public, 17 private).","bench":{"CV-Bench":{"pub":77.33,"private":44.72},"BLINK":{"pub":79.38,"private":58.67},"RealWorldQA":{"pub":76.14,"private":44.0},"CharXiv":{"pub":79.95,"private":76.43},"InfographicVQA":{"pub":97.27,"private":99.33},"Mind2Web":{"pub":82.19,"private":66.67},"Winoground":{"pub":86.25,"private":95.0},"KIE (CORD+FUNSD)":{"pub":92.39,"private":86.6},"Moderation (Hateful Memes)":{"pub":42.79,"private":47.0},"R-Bench-M":{"pub":33.76,"private":null},"MMMU-Pro vision":{"pub":45.5,"private":null}},"params":27781427952.0,"active":null,"params_estimated":false,"cal":{"acc":0.8024,"conf":0.797,"ece":0.0235,"over95":0.0032,"n":13686,"benchmarks":11},"ms_row":385.9,"gpu_h":2.066,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap applied client-side (kit data_url encoding), as in the submitted run","native":{"pub":76.1,"priv":68.71,"full":72.4,"budget":"1.6 MP"},"rank_full":4,"rank_pub":7},{"engine":"span-02","name":"span-02 (Gemma 4 26B-A4B)","url":"https://huggingface.co/lihe07/span-02","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":70.69,"priv":60.85,"full":65.77,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":75.64,"private":51.42},"BLINK":{"pub":63.91,"private":22.0},"RealWorldQA":{"pub":59.77,"private":28.0},"CharXiv":{"pub":78.2,"private":68.89},"InfographicVQA":{"pub":95.63,"private":97.33},"Mind2Web":{"pub":73.0,"private":66.0},"Winoground":{"pub":85.25,"private":86.0},"KIE (CORD+FUNSD)":{"pub":91.99,"private":84.97},"Moderation (Hateful Memes)":{"pub":37.35,"private":43.0},"R-Bench-M":{"pub":35.57,"private":null},"MMMU-Pro vision":{"pub":42.85,"private":null}},"params":25805936206.0,"active":null,"params_estimated":false,"cal":{"acc":0.7677,"conf":0.8571,"ece":0.0893,"over95":0.0271,"n":13695,"benchmarks":11},"ms_row":265.1,"gpu_h":1.419,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"author Gemma-4 560 soft-token image budget, under the kit 1.6 MP client cap","native":{"pub":70.69,"priv":60.85,"full":65.77,"budget":"560 soft tokens (~1.3 MP)"},"rank_full":23,"rank_pub":23},{"engine":"onejev1.5-0.8b","name":"OneJev1.5-0.8B","url":"https://huggingface.co/OmniJev/OneJev1.5-0.8B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pub":52.32,"priv":36.86,"full":44.59,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":62.68,"private":23.62},"BLINK":{"pub":53.77,"private":20.67},"RealWorldQA":{"pub":41.46,"private":14.67},"CharXiv":{"pub":57.39,"private":31.19},"InfographicVQA":{"pub":77.6,"private":54.67},"Mind2Web":{"pub":46.42,"private":38.0},"Winoground":{"pub":55.0,"private":68.0},"KIE (CORD+FUNSD)":{"pub":83.84,"private":71.98},"Moderation (Hateful Memes)":{"pub":24.65,"private":9.0},"R-Bench-M":{"pub":10.12,"private":null},"MMMU-Pro vision":{"pub":8.84,"private":null}},"params":1107265600,"active":null,"params_estimated":false,"cal":{"acc":0.6308,"conf":0.673,"ece":0.0423,"over95":0.0085,"n":13695,"benchmarks":11},"ms_row":134.0,"gpu_h":0.717,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":52.32,"priv":36.86,"full":44.59,"budget":"1.6 MP"},"rank_full":54,"rank_pub":54},{"engine":"onejev1.5-9b","name":"OneJev1.5-9B","url":"https://huggingface.co/OmniJev/OneJev1.5-9B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pub":71.82,"priv":54.95,"full":63.38,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":76.28,"private":41.71},"BLINK":{"pub":72.29,"private":34.67},"RealWorldQA":{"pub":63.65,"private":36.0},"CharXiv":{"pub":75.44,"private":56.64},"InfographicVQA":{"pub":93.99,"private":91.33},"Mind2Web":{"pub":77.39,"private":56.0},"Winoground":{"pub":77.75,"private":89.0},"KIE (CORD+FUNSD)":{"pub":92.92,"private":84.16},"Moderation (Hateful Memes)":{"pub":53.83,"private":5.0},"R-Bench-M":{"pub":16.62,"private":null},"MMMU-Pro vision":{"pub":33.57,"private":null}},"params":9409813744,"active":null,"params_estimated":false,"cal":{"acc":0.7588,"conf":0.7775,"ece":0.0187,"over95":0.0108,"n":13695,"benchmarks":11},"ms_row":202.7,"gpu_h":1.085,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":71.82,"priv":54.95,"full":63.38,"budget":"1.6 MP"},"rank_full":28,"rank_pub":22},{"engine":"onejev1.5-27b","name":"OneJev1.5-27B","url":"https://huggingface.co/OmniJev/OneJev1.5-27B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pub":75.65,"priv":62.15,"full":68.9,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":77.5,"private":39.36},"BLINK":{"pub":75.19,"private":58.0},"RealWorldQA":{"pub":67.54,"private":44.67},"CharXiv":{"pub":80.45,"private":58.52},"InfographicVQA":{"pub":95.63,"private":96.67},"Mind2Web":{"pub":83.97,"private":65.33},"Winoground":{"pub":83.0,"private":88.0},"KIE (CORD+FUNSD)":{"pub":93.39,"private":85.79},"Moderation (Hateful Memes)":{"pub":58.29,"private":23.0},"R-Bench-M":{"pub":24.92,"private":null},"MMMU-Pro vision":{"pub":40.86,"private":null}},"params":27356728560,"active":null,"params_estimated":false,"cal":{"acc":0.7906,"conf":0.8039,"ece":0.0192,"over95":0.0095,"n":13695,"benchmarks":11},"ms_row":365.0,"gpu_h":1.954,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":75.65,"priv":62.15,"full":68.9,"budget":"1.6 MP"},"rank_full":17,"rank_pub":8},{"engine":"aleria-judge-1.1.0","name":"Aleria Judge 1.1.0 (Gemma 4 31B)","url":"https://github.com/ALERIA-ML/aleria-judge/tree/4956428","kind":"inference technique","technique":"autoregressive","org":null,"pub":73.4,"priv":66.7,"full":70.05,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":76.12,"private":54.1},"BLINK":{"pub":59.89,"private":36.0},"RealWorldQA":{"pub":60.6,"private":34.0},"CharXiv":{"pub":81.2,"private":69.84},"InfographicVQA":{"pub":97.81,"private":96.67},"Mind2Web":{"pub":79.99,"private":64.67},"Winoground":{"pub":85.75,"private":87.0},"KIE (CORD+FUNSD)":{"pub":92.79,"private":87.01},"Moderation (Hateful Memes)":{"pub":51.2,"private":71.0},"R-Bench-M":{"pub":40.08,"private":null},"MMMU-Pro vision":{"pub":41.06,"private":null}},"params":31273088876.0,"active":null,"params_estimated":false,"cal":{"acc":0.7854,"conf":0.8107,"ece":0.0357,"over95":0.03,"n":13695,"benchmarks":11},"ms_row":318.2,"gpu_h":1.703,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap","native":{"pub":73.4,"priv":66.7,"full":70.05,"budget":"1.6 MP"},"rank_full":14,"rank_pub":16,"new":true},{"engine":"herbert-v0.5","name":"Herbert v0.5","url":"https://huggingface.co/leinenterprises/Herbert_v0.5","kind":"LoRA","technique":"autoregressive","org":null,"pub":75.54,"priv":64.39,"full":69.96,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":79.12,"private":43.38},"BLINK":{"pub":71.81,"private":55.33},"RealWorldQA":{"pub":73.64,"private":42.0},"CharXiv":{"pub":77.94,"private":67.01},"InfographicVQA":{"pub":96.17,"private":98.67},"Mind2Web":{"pub":83.42,"private":63.33},"Winoground":{"pub":83.75,"private":90.0},"KIE (CORD+FUNSD)":{"pub":92.32,"private":85.79},"Moderation (Hateful Memes)":{"pub":52.51,"private":34.0},"R-Bench-M":{"pub":30.52,"private":null},"MMMU-Pro vision":{"pub":42.19,"private":null}},"params":27781427952.0,"active":null,"params_estimated":false,"cal":{"acc":0.793,"conf":0.7991,"ece":0.04,"over95":0.0042,"n":13695,"benchmarks":11},"ms_row":223.8,"gpu_h":1.198,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap","native":{"pub":75.54,"priv":64.39,"full":69.96,"budget":"1.6 MP"},"rank_full":15,"rank_pub":9,"new":true},{"engine":"jev-d3-46k","name":"jev-d3-46k","url":"https://huggingface.co/tinnel123/jev-d3-46k","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":68.13,"priv":59.08,"full":63.6,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":74.58,"private":42.04},"BLINK":{"pub":55.05,"private":40.67},"RealWorldQA":{"pub":58.38,"private":30.67},"CharXiv":{"pub":72.68,"private":51.92},"InfographicVQA":{"pub":91.26,"private":92.67},"Mind2Web":{"pub":77.66,"private":63.33},"Winoground":{"pub":80.75,"private":87.0},"KIE (CORD+FUNSD)":{"pub":91.85,"private":85.38},"Moderation (Hateful Memes)":{"pub":45.42,"private":38.0},"R-Bench-M":{"pub":17.16,"private":null},"MMMU-Pro vision":{"pub":32.11,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.7385,"conf":0.7767,"ece":0.0383,"over95":0.009,"n":13695,"benchmarks":11},"ms_row":117.0,"gpu_h":0.626,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap","native":{"pub":68.13,"priv":59.08,"full":63.6,"budget":"1.6 MP"},"rank_full":27,"rank_pub":28,"new":true},{"engine":"saina-helm-2-omni","name":"Saina Helm 2 Omni (0.8B)","url":"https://huggingface.co/run-saina/saina-helm-2-omni-0.8b","kind":"LoRA + head","technique":"autoregressive","org":{"id":"run-saina","name":"Saina","logo":"data/orgs/run-saina.png"},"pub":46.89,"priv":34.1,"full":40.49,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":56.94,"private":24.29},"BLINK":{"pub":32.98,"private":6.0},"RealWorldQA":{"pub":43.12,"private":29.33},"CharXiv":{"pub":48.62,"private":25.53},"InfographicVQA":{"pub":77.05,"private":54.0},"Mind2Web":{"pub":50.81,"private":34.67},"Winoground":{"pub":50.75,"private":61.0},"KIE (CORD+FUNSD)":{"pub":75.42,"private":65.08},"Moderation (Hateful Memes)":{"pub":11.46,"private":7.0},"R-Bench-M":{"pub":16.44,"private":null},"MMMU-Pro vision":{"pub":7.12,"private":null}},"params":852985920.0,"active":null,"params_estimated":false,"cal":{"acc":0.6022,"conf":0.6639,"ece":0.0618,"over95":0.0084,"n":13695,"benchmarks":11},"ms_row":169.1,"gpu_h":0.905,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap","native":{"pub":46.89,"priv":34.1,"full":40.49,"budget":"1.6 MP"},"rank_full":59,"rank_pub":58,"new":true},{"engine":"vllm-sr-d3-turbo-v11","name":"vLLM-SR d3-turbo v1.1 (27B)","url":"https://huggingface.co/vllm-sr/d3-turbo","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":76.84,"priv":68.08,"full":72.46,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":82.35,"private":46.06},"BLINK":{"pub":73.58,"private":66.67},"RealWorldQA":{"pub":72.25,"private":46.67},"CharXiv":{"pub":80.95,"private":67.95},"InfographicVQA":{"pub":96.72,"private":99.33},"Mind2Web":{"pub":85.06,"private":64.67},"Winoground":{"pub":84.75,"private":89.0},"KIE (CORD+FUNSD)":{"pub":92.19,"private":83.35},"Moderation (Hateful Memes)":{"pub":50.87,"private":49.0},"R-Bench-M":{"pub":39.0,"private":null},"MMMU-Pro vision":{"pub":44.05,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.8069,"conf":0.8517,"ece":0.0448,"over95":0.0145,"n":13695,"benchmarks":11},"ms_row":384.0,"gpu_h":2.055,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":76.84,"priv":68.08,"full":72.46,"budget":"1.6 MP"},"rank_full":3,"rank_pub":4,"new":true},{"engine":"drex-max-v1","name":"Nace.AI Drex Max v1 (27B)","url":null,"kind":"full fine-tune","technique":"autoregressive","org":{"id":"Nace-DeepInfra","name":"Nace-Deepinfra","logo":"data/orgs/Nace-DeepInfra.png"},"pub":74.09,"priv":68.41,"full":71.25,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":77.33,"private":43.05},"BLINK":{"pub":59.73,"private":69.33},"RealWorldQA":{"pub":72.81,"private":49.33},"CharXiv":{"pub":81.2,"private":72.66},"InfographicVQA":{"pub":97.81,"private":100.0},"Mind2Web":{"pub":83.56,"private":69.33},"Winoground":{"pub":85.0,"private":94.0},"KIE (CORD+FUNSD)":{"pub":92.59,"private":84.97},"Moderation (Hateful Memes)":{"pub":40.31,"private":33.0},"R-Bench-M":{"pub":40.44,"private":null},"MMMU-Pro vision":{"pub":47.23,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.7958,"conf":0.8587,"ece":0.0628,"over95":0.024,"n":13695,"benchmarks":11},"ms_row":298.2,"gpu_h":1.596,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":74.09,"priv":68.41,"full":71.25,"budget":"1.6 MP"},"rank_full":11,"rank_pub":13,"new":true},{"engine":"jev-vision-27b-r2","name":"jev-vision-27b","url":"https://huggingface.co/e12ex2/jev-vision-27b","kind":"LoRA","technique":"autoregressive","org":null,"pub":73.07,"priv":64.32,"full":68.7,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":76.77,"private":42.38},"BLINK":{"pub":53.93,"private":56.0},"RealWorldQA":{"pub":72.81,"private":39.33},"CharXiv":{"pub":78.7,"private":60.41},"InfographicVQA":{"pub":97.27,"private":98.0},"Mind2Web":{"pub":84.1,"private":66.0},"Winoground":{"pub":83.5,"private":88.0},"KIE (CORD+FUNSD)":{"pub":91.18,"private":85.79},"Moderation (Hateful Memes)":{"pub":52.51,"private":43.0},"R-Bench-M":{"pub":24.38,"private":null},"MMMU-Pro vision":{"pub":38.01,"private":null}},"params":27781427952,"active":null,"params_estimated":false,"cal":{"acc":0.773,"conf":0.8142,"ece":0.0587,"over95":0.0054,"n":13695,"benchmarks":11},"ms_row":255.8,"gpu_h":1.369,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":73.07,"priv":64.32,"full":68.7,"budget":"1.6 MP"},"rank_full":18,"rank_pub":18,"new":true},{"engine":"visionjev-1.0.0","name":"VisionJEV 1.0.0 (27B)","url":"https://huggingface.co/taoszhang/VisionJEV","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":76.77,"priv":66.07,"full":71.42,"base":null,"caveat":"Questions over its declared 8,192-token limit count as failures.","bench":{"CV-Bench":{"pub":80.17,"private":45.06},"BLINK":{"pub":74.06,"private":50.0},"RealWorldQA":{"pub":74.2,"private":50.67},"CharXiv":{"pub":76.44,"private":70.78},"InfographicVQA":{"pub":98.91,"private":99.33},"Mind2Web":{"pub":84.52,"private":66.67},"Winoground":{"pub":85.75,"private":93.0},"KIE (CORD+FUNSD)":{"pub":93.19,"private":84.16},"Moderation (Hateful Memes)":{"pub":54.33,"private":35.0},"R-Bench-M":{"pub":31.6,"private":null},"MMMU-Pro vision":{"pub":44.64,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.8028,"conf":0.8244,"ece":0.0216,"over95":0.0098,"n":13695,"benchmarks":11},"ms_row":347.2,"gpu_h":1.859,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":76.77,"priv":66.07,"full":71.42,"budget":"1.6 MP"},"rank_full":9,"rank_pub":5,"new":true},{"engine":"autotrust-jev-27b-v11","name":"AutoTrust-JEV-27B v1.1","url":"https://huggingface.co/autotrust/AutoTrust-JEV-27B-v1.1","kind":"full fine-tune","technique":"autoregressive","org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"},"pub":74.44,"priv":69.35,"full":71.89,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":77.33,"private":46.06},"BLINK":{"pub":59.24,"private":68.0},"RealWorldQA":{"pub":70.87,"private":48.67},"CharXiv":{"pub":80.2,"private":75.49},"InfographicVQA":{"pub":98.91,"private":100.0},"Mind2Web":{"pub":84.52,"private":63.33},"Winoground":{"pub":86.5,"private":95.0},"KIE (CORD+FUNSD)":{"pub":92.59,"private":86.6},"Moderation (Hateful Memes)":{"pub":44.11,"private":41.0},"R-Bench-M":{"pub":39.0,"private":null},"MMMU-Pro vision":{"pub":48.09,"private":null}},"params":27356728560.0,"active":null,"params_estimated":false,"cal":{"acc":0.7975,"conf":0.8388,"ece":0.0413,"over95":0.0107,"n":13695,"benchmarks":11},"ms_row":289.8,"gpu_h":1.551,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":74.44,"priv":69.35,"full":71.89,"budget":"1.6 MP"},"rank_full":5,"rank_pub":12,"new":true},{"engine":"qev-2b","name":"QEV-2B","url":"https://huggingface.co/ken-jo/QEV-2B","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":62.01,"priv":49.86,"full":55.93,"base":null,"caveat":"Takes at most 4 images and 8,192 input tokens per question; larger questions count as failures.","bench":{"CV-Bench":{"pub":70.62,"private":39.36},"BLINK":{"pub":47.97,"private":35.33},"RealWorldQA":{"pub":55.88,"private":32.67},"CharXiv":{"pub":62.41,"private":37.78},"InfographicVQA":{"pub":83.06,"private":62.0},"Mind2Web":{"pub":83.97,"private":56.67},"Winoground":{"pub":64.75,"private":76.0},"KIE (CORD+FUNSD)":{"pub":94.06,"private":80.91},"Moderation (Hateful Memes)":{"pub":29.43,"private":28.0},"R-Bench-M":{"pub":16.8,"private":null},"MMMU-Pro vision":{"pub":16.13,"private":null}},"params":2213241664,"active":null,"params_estimated":false,"cal":{"acc":0.6951,"conf":0.7738,"ece":0.0805,"over95":0.0202,"n":13695,"benchmarks":11},"ms_row":120.5,"gpu_h":0.645,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":62.01,"priv":49.86,"full":55.93,"budget":"1.6 MP"},"rank_full":44,"rank_pub":42,"new":true},{"engine":"qev-0.8b","name":"QEV-0.8B","url":"https://huggingface.co/ken-jo/QEV-0.8B","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":53.36,"priv":38.67,"full":46.01,"base":null,"caveat":"Takes at most 4 images and 8,192 input tokens per question; larger questions count as failures.","bench":{"CV-Bench":{"pub":59.69,"private":24.29},"BLINK":{"pub":39.27,"private":14.0},"RealWorldQA":{"pub":40.9,"private":19.33},"CharXiv":{"pub":54.39,"private":34.01},"InfographicVQA":{"pub":75.41,"private":55.33},"Mind2Web":{"pub":73.42,"private":47.33},"Winoground":{"pub":48.0,"private":62.0},"KIE (CORD+FUNSD)":{"pub":88.85,"private":79.7},"Moderation (Hateful Memes)":{"pub":31.9,"private":12.0},"R-Bench-M":{"pub":12.29,"private":null},"MMMU-Pro vision":{"pub":9.17,"private":null}},"params":852985920,"active":null,"params_estimated":false,"cal":{"acc":0.64,"conf":0.7428,"ece":0.1028,"over95":0.0193,"n":13695,"benchmarks":11},"ms_row":101.8,"gpu_h":0.545,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":53.36,"priv":38.67,"full":46.01,"budget":"1.6 MP"},"rank_full":53,"rank_pub":52,"new":true},{"engine":"sieve-vision-4b","name":"Sieve-Vision-4B","url":"https://huggingface.co/sthanika-ai/Sieve-Vision-4B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"},"pub":65.17,"priv":52.02,"full":58.6,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":69.89,"private":28.31},"BLINK":{"pub":51.99,"private":48.67},"RealWorldQA":{"pub":60.32,"private":29.33},"CharXiv":{"pub":63.16,"private":30.24},"InfographicVQA":{"pub":92.35,"private":92.0},"Mind2Web":{"pub":92.6,"private":63.33},"Winoground":{"pub":75.5,"private":79.0},"KIE (CORD+FUNSD)":{"pub":89.58,"private":81.32},"Moderation (Hateful Memes)":{"pub":22.51,"private":16.0},"R-Bench-M":{"pub":20.41,"private":null},"MMMU-Pro vision":{"pub":29.19,"private":null}},"params":4539265536,"active":null,"params_estimated":false,"cal":{"acc":0.7231,"conf":0.7834,"ece":0.0604,"over95":0.0154,"n":13695,"benchmarks":11},"ms_row":126.0,"gpu_h":0.674,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":65.17,"priv":52.02,"full":58.6,"budget":"1.6 MP"},"rank_full":36,"rank_pub":33,"new":true},{"engine":"glide-v1.2","name":"Fastino GLiDE 1.2","url":"https://huggingface.co/fastino/GLiDE","kind":"full fine-tune","technique":"autoregressive","org":{"id":"fastino","name":"Fastino","logo":"data/orgs/fastino.png"},"pub":75.37,"priv":67.54,"full":71.46,"base":null,"caveat":"Public vision benchmarks run by the lab; there is no submitted vision run.","bench":{"CV-Bench":{"pub":76.53,"private":44.05},"BLINK":{"pub":59.57,"private":66.0},"RealWorldQA":{"pub":71.98,"private":45.33},"CharXiv":{"pub":83.21,"private":71.72},"InfographicVQA":{"pub":97.81,"private":98.0},"Mind2Web":{"pub":82.73,"private":69.33},"Winoground":{"pub":87.25,"private":92.0},"KIE (CORD+FUNSD)":{"pub":92.65,"private":87.41},"Moderation (Hateful Memes)":{"pub":42.95,"private":34.0},"R-Bench-M":{"pub":56.14,"private":null},"MMMU-Pro vision":{"pub":48.55,"private":null}},"params":27800000000.0,"active":null,"params_estimated":false,"cal":{"acc":0.8114,"conf":0.824,"ece":0.018,"over95":0.01,"n":13695,"benchmarks":11},"ms_row":346.6,"gpu_h":0.298,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels (author default 1,048,576 lifted)","native":{"pub":75.37,"priv":67.54,"full":71.46,"budget":"1.6 MP"},"rank_full":8,"rank_pub":11,"new":true},{"engine":"h2o-lightning-31b","name":"H2O Lightning 31B","url":"https://huggingface.co/h2oai/h2o-lightning-31b","kind":"full fine-tune","technique":"autoregressive","org":{"id":"h2oai","name":"H2O.ai","logo":"data/orgs/h2oai.png"},"pub":76.51,"priv":64.6,"full":70.56,"base":null,"caveat":"Takes at most 4 images per question; larger questions count as failures.","bench":{"CV-Bench":{"pub":81.38,"private":49.75},"BLINK":{"pub":68.75,"private":42.0},"RealWorldQA":{"pub":64.49,"private":32.67},"CharXiv":{"pub":83.71,"private":67.01},"InfographicVQA":{"pub":97.81,"private":98.0},"Mind2Web":{"pub":86.16,"private":64.0},"Winoground":{"pub":85.75,"private":91.0},"KIE (CORD+FUNSD)":{"pub":91.25,"private":87.01},"Moderation (Hateful Memes)":{"pub":55.48,"private":50.0},"R-Bench-M":{"pub":38.82,"private":null},"MMMU-Pro vision":{"pub":47.29,"private":null}},"params":31273086512,"active":null,"params_estimated":false,"cal":{"acc":0.8087,"conf":0.906,"ece":0.0981,"over95":0.0485,"n":13695,"benchmarks":11},"ms_row":275.9,"gpu_h":1.477,"effective_max_pixels":1290240,"effective_budget":"560 soft tokens (~1.3 MP)","effective_budget_source":"Gemma-4 server budget; images sent at the 1.6 MP board cap","native":{"pub":76.51,"priv":64.6,"full":70.56,"budget":"560 soft tokens (~1.3 MP)"},"rank_full":13,"rank_pub":6,"new":true},{"engine":"rbdash-decision-v0.2","name":"RBDash-Decision v0.2","url":"https://huggingface.co/RBDash-Team/RBDash-Decision-v0.2","kind":"full fine-tune","technique":"autoregressive","org":{"id":"RBDash-Team","name":"RBDash-Team","logo":"data/orgs/RBDash-Team.png"},"pub":77.7,"priv":68.91,"full":73.3,"base":null,"caveat":"Takes at most 4 images per question; larger questions count as failures.","bench":{"CV-Bench":{"pub":77.82,"private":41.71},"BLINK":{"pub":75.35,"private":64.0},"RealWorldQA":{"pub":73.36,"private":49.33},"CharXiv":{"pub":79.2,"private":67.95},"InfographicVQA":{"pub":97.81,"private":97.33},"Mind2Web":{"pub":87.67,"private":72.67},"Winoground":{"pub":87.0,"private":95.0},"KIE (CORD+FUNSD)":{"pub":92.99,"private":86.19},"Moderation (Hateful Memes)":{"pub":57.63,"private":46.0},"R-Bench-M":{"pub":35.21,"private":null},"MMMU-Pro vision":{"pub":44.44,"private":null}},"params":26085330160,"active":null,"params_estimated":false,"cal":{"acc":0.8096,"conf":0.8342,"ece":0.0298,"over95":0.0113,"n":13695,"benchmarks":11},"ms_row":324.9,"gpu_h":1.739,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":77.7,"priv":68.91,"full":73.3,"budget":"1.6 MP"},"rank_full":2,"rank_pub":2,"new":true},{"engine":"vllm-sr-d3-ultra","name":"vLLM-SR d3-ultra (Decision 3.0, 27B)","url":"https://huggingface.co/vllm-sr/d3-ultra","kind":"full fine-tune","technique":"autoregressive","org":{"id":"vllm-sr","name":"vLLM Semantic Router","logo":"data/orgs/vllm-sr.png"},"pub":75.46,"priv":65.73,"full":70.59,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":81.79,"private":44.72},"BLINK":{"pub":71.33,"private":67.33},"RealWorldQA":{"pub":69.48,"private":44.67},"CharXiv":{"pub":80.45,"private":66.06},"InfographicVQA":{"pub":96.72,"private":96.0},"Mind2Web":{"pub":85.89,"private":62.67},"Winoground":{"pub":84.0,"private":87.0},"KIE (CORD+FUNSD)":{"pub":92.25,"private":82.13},"Moderation (Hateful Memes)":{"pub":41.3,"private":41.0},"R-Bench-M":{"pub":43.15,"private":null},"MMMU-Pro vision":{"pub":43.65,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.8027,"conf":0.8622,"ece":0.06,"over95":0.0224,"n":13695,"benchmarks":11},"ms_row":380.7,"gpu_h":2.038,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":75.46,"priv":65.73,"full":70.59,"budget":"1.6 MP"},"rank_full":12,"rank_pub":10,"new":true},{"engine":"wemm-jev-27b","name":"WeMM-JEV-27B","url":"https://huggingface.co/Rover00/WeMM-JEV","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":77.93,"priv":69.7,"full":73.81,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":81.38,"private":44.72},"BLINK":{"pub":74.22,"private":68.67},"RealWorldQA":{"pub":74.2,"private":50.0},"CharXiv":{"pub":81.7,"private":73.61},"InfographicVQA":{"pub":98.36,"private":100.0},"Mind2Web":{"pub":85.89,"private":72.67},"Winoground":{"pub":87.0,"private":94.0},"KIE (CORD+FUNSD)":{"pub":93.79,"private":86.6},"Moderation (Hateful Memes)":{"pub":52.51,"private":37.0},"R-Bench-M":{"pub":38.46,"private":null},"MMMU-Pro vision":{"pub":46.23,"private":null}},"params":26085330160.0,"active":null,"params_estimated":false,"cal":{"acc":0.8145,"conf":0.836,"ece":0.0214,"over95":0.0106,"n":13695,"benchmarks":11},"ms_row":338.7,"gpu_h":1.813,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"author cap equals the board cap","native":{"pub":77.93,"priv":69.7,"full":73.81,"budget":"1.6 MP"},"rank_full":1,"rank_pub":1,"new":true},{"engine":"valen-0.8b-r2","name":"Valen-0.8B","url":"https://huggingface.co/Valen-Team/Valen-0.8B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"Valen-Team","name":"Valen","logo":"data/orgs/Valen-Team.png"},"pub":47.7,"priv":39.67,"full":43.69,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":62.52,"private":16.25},"BLINK":{"pub":35.88,"private":26.0},"RealWorldQA":{"pub":47.28,"private":28.0},"CharXiv":{"pub":57.64,"private":28.36},"InfographicVQA":{"pub":79.78,"private":71.33},"Mind2Web":{"pub":40.66,"private":32.0},"Winoground":{"pub":42.25,"private":66.0},"KIE (CORD+FUNSD)":{"pub":77.09,"private":80.1},"Moderation (Hateful Memes)":{"pub":11.79,"private":9.0},"R-Bench-M":{"pub":16.44,"private":null},"MMMU-Pro vision":{"pub":7.78,"private":null}},"params":854305641.0,"active":null,"params_estimated":false,"cal":{"acc":0.612,"conf":0.7202,"ece":0.1083,"over95":0.0271,"n":13695,"benchmarks":11},"ms_row":340.1,"gpu_h":1.82,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"native processor at the board cap (author cap lifted)","native":{"pub":47.7,"priv":39.67,"full":43.69,"budget":"1.6 MP"},"rank_full":55,"rank_pub":57,"new":true},{"engine":"valen-2b-r2","name":"Valen-2B","url":"https://huggingface.co/Valen-Team/Valen-2B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"Valen-Team","name":"Valen","logo":"data/orgs/Valen-Team.png"},"pub":56.37,"priv":42.69,"full":49.53,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":68.75,"private":28.98},"BLINK":{"pub":45.87,"private":27.33},"RealWorldQA":{"pub":58.66,"private":34.0},"CharXiv":{"pub":63.66,"private":34.01},"InfographicVQA":{"pub":83.61,"private":73.33},"Mind2Web":{"pub":50.39,"private":38.0},"Winoground":{"pub":63.75,"private":65.0},"KIE (CORD+FUNSD)":{"pub":86.64,"private":82.54},"Moderation (Hateful Memes)":{"pub":14.59,"private":1.0},"R-Bench-M":{"pub":19.69,"private":null},"MMMU-Pro vision":{"pub":15.34,"private":null}},"params":2214825577.0,"active":null,"params_estimated":false,"cal":{"acc":0.6635,"conf":0.7533,"ece":0.0898,"over95":0.0341,"n":13695,"benchmarks":11},"ms_row":384.1,"gpu_h":2.056,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"native processor at the board cap (author cap lifted)","native":{"pub":56.37,"priv":42.69,"full":49.53,"budget":"1.6 MP"},"rank_full":48,"rank_pub":48,"new":true},{"engine":"valen-4b-r2","name":"Valen-4B","url":"https://huggingface.co/Valen-Team/Valen-4B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"Valen-Team","name":"Valen","logo":"data/orgs/Valen-Team.png"},"pub":62.63,"priv":50.69,"full":56.66,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":74.66,"private":29.98},"BLINK":{"pub":51.19,"private":30.67},"RealWorldQA":{"pub":62.54,"private":30.0},"CharXiv":{"pub":70.93,"private":47.21},"InfographicVQA":{"pub":93.44,"private":93.33},"Mind2Web":{"pub":61.77,"private":54.0},"Winoground":{"pub":65.75,"private":76.0},"KIE (CORD+FUNSD)":{"pub":90.38,"private":84.97},"Moderation (Hateful Memes)":{"pub":21.19,"private":10.0},"R-Bench-M":{"pub":22.94,"private":null},"MMMU-Pro vision":{"pub":29.19,"private":null}},"params":4540981545.0,"active":null,"params_estimated":false,"cal":{"acc":0.7122,"conf":0.7883,"ece":0.0762,"over95":0.0287,"n":13695,"benchmarks":11},"ms_row":374.0,"gpu_h":2.002,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"native processor at the board cap (author cap lifted)","native":{"pub":62.63,"priv":50.69,"full":56.66,"budget":"1.6 MP"},"rank_full":41,"rank_pub":40,"new":true},{"engine":"sieve-vision-27b","name":"Sieve-Vision-27B","url":"https://huggingface.co/sthanika-ai/Sieve-Vision-27B","kind":"LoRA + head","technique":"autoregressive","org":{"id":"sthanika-ai","name":"Sthānika AI","logo":"data/orgs/sthanika-ai.png"},"pub":73.17,"priv":57.22,"full":65.19,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":80.9,"private":40.03},"BLINK":{"pub":68.43,"private":62.67},"RealWorldQA":{"pub":75.58,"private":40.67},"CharXiv":{"pub":72.18,"private":30.24},"InfographicVQA":{"pub":97.81,"private":96.0},"Mind2Web":{"pub":94.38,"private":70.0},"Winoground":{"pub":80.25,"private":88.0},"KIE (CORD+FUNSD)":{"pub":97.46,"private":85.38},"Moderation (Hateful Memes)":{"pub":23.17,"private":2.0},"R-Bench-M":{"pub":26.73,"private":null},"MMMU-Pro vision":{"pub":39.4,"private":null}},"params":26085330160,"active":null,"params_estimated":false,"cal":{"acc":0.7807,"conf":0.8158,"ece":0.0401,"over95":0.0245,"n":13695,"benchmarks":11},"ms_row":479.8,"gpu_h":2.568,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"per-image cap at the board cap; the images of one request share a 6,400-token budget","native":{"pub":73.17,"priv":57.22,"full":65.19,"budget":"1.6 MP"},"rank_full":24,"rank_pub":17,"new":true},{"engine":"rsi-jev-v5.0-vl-3b","name":"RSI-Jev v5.0 VL 3B","url":"https://huggingface.co/shgao/rsi-jev-v5.0-vl-3b","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":62.58,"priv":52.1,"full":57.34,"base":{"name":"Qwen/Qwen3.5-4B-Base","pub":28.77,"priv":21.9},"caveat":"Its base was read through the plain vision-language path, where a pretrained base answers letters poorly, so the gain over the base is overstated.","bench":{"CV-Bench":{"pub":72.96,"private":35.68},"BLINK":{"pub":50.7,"private":45.33},"RealWorldQA":{"pub":55.88,"private":31.33},"CharXiv":{"pub":66.67,"private":40.61},"InfographicVQA":{"pub":82.51,"private":67.33},"Mind2Web":{"pub":63.69,"private":41.33},"Winoground":{"pub":77.0,"private":82.0},"KIE (CORD+FUNSD)":{"pub":86.17,"private":81.32},"Moderation (Hateful Memes)":{"pub":42.46,"private":44.0},"R-Bench-M":{"pub":11.75,"private":null},"MMMU-Pro vision":{"pub":25.08,"private":null}},"params":4660000000.0,"active":null,"params_estimated":false,"cal":{"acc":0.6966,"conf":0.704,"ece":0.0226,"over95":0.0106,"n":13695,"benchmarks":11},"ms_row":144.2,"gpu_h":0.772,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":63.26,"priv":51.58,"full":57.42,"budget":"16.7 MP"},"rank_full":39,"rank_pub":41},{"engine":"jev-27b-vl","name":"JEV-27B-VL","url":"https://huggingface.co/autotrust/JEV-27B-VL","kind":"LoRA","technique":"autoregressive","org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"},"pub":72.8,"priv":66.4,"full":69.6,"base":{"name":"Qwen/Qwen3.8-27B","pub":68.54,"priv":60.25},"caveat":"Ran through the author's own server (vLLM + POST /v1/decide, System 1 only) with the context cap lifted to 256K; its base was read through a different (head-free autojev) path.","bench":{"CV-Bench":{"pub":76.85,"private":42.04},"BLINK":{"pub":63.27,"private":58.67},"RealWorldQA":{"pub":72.25,"private":48.0},"CharXiv":{"pub":79.7,"private":62.29},"InfographicVQA":{"pub":96.72,"private":98.67},"Mind2Web":{"pub":78.62,"private":59.33},"Winoground":{"pub":84.5,"private":89.0},"KIE (CORD+FUNSD)":{"pub":92.05,"private":86.6},"Moderation (Hateful Memes)":{"pub":40.31,"private":53.0},"R-Bench-M":{"pub":31.06,"private":null},"MMMU-Pro vision":{"pub":40.13,"private":null}},"params":27800000000,"active":null,"params_estimated":false,"cal":{"acc":0.7771,"conf":0.7901,"ece":0.018,"over95":0.0102,"n":13695,"benchmarks":11},"ms_row":263.9,"gpu_h":1.412,"effective_max_pixels":1638400,"effective_budget":"1.6 MP","effective_budget_source":"board cap 1,638,400 pixels","native":{"pub":72.78,"priv":66.86,"full":69.82,"budget":"16.7 MP"},"rank_full":16,"rank_pub":19}],"refs":[{"engine":"ref-qwen35-397b","name":"Qwen3.5-397B (VLM)","kind":"stock · API","url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B","note":"Stock model via API, read from its generated letter rather than letter probabilities.","label":null,"params":397000000000.0,"active":"A17B","params_estimated":null,"pub":68.52,"priv":58.37,"full":63.44,"bench":{"CV-Bench":{"pub":78.31,"private":46.73},"BLINK":{"pub":50.54,"private":4.67},"RealWorldQA":{"pub":67.82,"private":41.33},"CharXiv":{"pub":62.16,"private":51.92},"InfographicVQA":{"pub":95.08,"private":99.33},"Mind2Web":{"pub":76.57,"private":57.33},"Winoground":{"pub":82.25,"private":87.0},"KIE (CORD+FUNSD)":{"pub":94.06,"private":84.97},"Moderation (Hateful Memes)":{"pub":49.22,"private":52.0},"R-Bench-M":{"pub":0.38,"private":null},"MMMU-Pro vision":{"pub":47.43,"private":null}},"ref":true,"cal":null,"ms_row":null},{"engine":"ref-qwen3vl-8b","name":"Qwen3-VL-8B (VLM)","kind":"stock","url":"https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct","note":"Stock vision-language model, no fine-tune.","label":null,"params":8800000000.0,"active":null,"params_estimated":null,"pub":62.33,"priv":45.0,"full":53.66,"bench":{"CV-Bench":{"pub":72.72,"private":34.0},"BLINK":{"pub":51.35,"private":20.0},"RealWorldQA":{"pub":56.16,"private":24.67},"CharXiv":{"pub":58.9,"private":16.1},"InfographicVQA":{"pub":84.7,"private":70.67},"Mind2Web":{"pub":70.95,"private":50.67},"Winoground":{"pub":71.75,"private":80.0},"KIE (CORD+FUNSD)":{"pub":89.05,"private":80.91},"Moderation (Hateful Memes)":{"pub":34.21,"private":28.0},"R-Bench-M":{"pub":21.85,"private":null},"MMMU-Pro vision":{"pub":27.87,"private":null}},"ref":true,"cal":{"acc":0.7027,"conf":0.8963,"ece":0.1937,"over95":0.1058,"n":13695,"benchmarks":11},"ms_row":119.4},{"engine":"ref-siglip2","name":"SigLIP 2 (contrastive)","kind":"contrastive floor","url":"https://huggingface.co/google/siglip2-so400m-patch14-384","note":"Image-text similarity only, no language model: the floor.","label":null,"params":1100000000.0,"active":null,"params_estimated":true,"pub":6.16,"priv":6.17,"full":6.17,"bench":{"CV-Bench":{"pub":-5.39,"private":4.86},"BLINK":{"pub":-0.85,"private":-10.0},"RealWorldQA":{"pub":5.39,"private":-6.0},"CharXiv":{"pub":4.01,"private":3.85},"InfographicVQA":{"pub":7.65,"private":-3.33},"Mind2Web":{"pub":8.46,"private":25.33},"Winoground":{"pub":33.5,"private":19.0},"KIE (CORD+FUNSD)":{"pub":-2.25,"private":0.51},"Moderation (Hateful Memes)":{"pub":-2.23,"private":2.0},"R-Bench-M":{"pub":0.74,"private":null},"MMMU-Pro vision":{"pub":2.68,"private":null}},"ref":true,"cal":null,"ms_row":null}],"excluded":[{"name":"gevva-e4b","why":"its official path takes no images"},{"name":"gevva-e2b","why":"its official path takes no images"},{"name":"reflex-27b-pmscr","why":"its as-run backend takes no images"}],"bench_info":{"CV-Bench":{"what":"Basic spatial vision: count objects, say which is left or right, and judge which is closer to the camera.","example":"How many chairs are in the photo, or is the lamp closer to the camera than the sofa?","why":"Many decisions start with getting simple facts about a scene right.","tech":"Counting, 2D relation and distance items; 2 to 6 options."},"BLINK":{"what":"Perception puzzles that people solve at a glance but models often miss, sometimes across two or more images.","example":"Given two photos of the same room, how did the camera move between them?","why":"Tests whether a model really looks at the image instead of guessing from the wording.","tech":"Seven subtasks of the validation split; 2 to 4 options over 1 to 4 images."},"RealWorldQA":{"what":"Questions about everyday real-world photos, many taken from a car or a phone.","example":"From this dashboard photo, which lane should the car be in to turn right?","why":"Real scenes are where a decision model meets the physical world.","tech":"Multiple-choice and yes/no items parsed from the prompt."},"CharXiv":{"what":"Reasoning over charts from scientific papers: read values, compare trends and find the right series.","example":"In this plot, which method has the highest score at the largest dataset size?","why":"Decisions often hinge on reading a chart correctly.","tech":"Reasoning questions converted to 4-way multiple choice with distractors from the same figure."},"InfographicVQA":{"what":"Answering questions about infographics that mix text, numbers, icons and layout.","example":"According to this infographic, which age group spends the most time online?","why":"Combines reading and layout understanding on dense documents.","tech":"Short answers converted to multiple choice."},"Mind2Web":{"what":"Web navigation: given a task and a screenshot of a real website, pick the element to click or type into next.","example":"To book a one-way flight to Denver, which of these four marked buttons comes next?","why":"Choosing the next action on a screen is a decision in the most literal sense.","tech":"Four marked boxes per screenshot: the right element plus three real distractors from the page."},"Winoground":{"what":"Matching captions to images when the words are the same but their order changes the meaning.","example":"Which caption fits: 'a dog bites a person' or 'a person bites a dog'?","why":"Tests whether a model understands who does what to whom.","tech":"Caption choice for each of two images; pairs of images and captions."},"KIE (CORD+FUNSD)":{"what":"Reading receipts and forms: pull the right field (total, date, item) out of a scanned document.","example":"On this receipt, what is the total after tax?","why":"Document extraction feeds many routing and approval decisions.","tech":"Receipt fields and form-field presence questions, multiple choice."},"Moderation (Hateful Memes)":{"what":"Deciding whether a meme is hateful, where the meaning often needs both the image and its text.","example":"Is this meme, its picture and caption together, hateful or not?","why":"A content-moderation call is a binary decision with real consequences.","tech":"Binary hateful / not hateful; a caption-only control is reported separately."},"R-Bench-M":{"what":"Expert-level multi-discipline questions with an image, from science, engineering and medicine.","example":"Given this circuit diagram, which option gives the current through the resistor?","why":"Shows headroom: even strong models score low here.","tech":"Six options A to F, one image; low weight in the index."},"MMMU-Pro vision":{"what":"College-level exam questions where the whole question and its options are shown inside the image.","example":"A photographed exam page asks which of ten options explains the graph.","why":"The model has to read and reason from the image alone.","tech":"Ten options inside the image; a quarter weight in the index."}},"bench_rows":{"CV-Bench":{"public":2038},"BLINK":{"public":961},"RealWorldQA":{"public":589},"CharXiv":{"public":532},"InfographicVQA":{"public":244},"Mind2Web":{"public":973},"Winoground":{"public":800},"KIE (CORD+FUNSD)":{"public":2465},"Moderation (Hateful Memes)":{"public":1213},"R-Bench-M":{"public":665},"MMMU-Pro vision":{"public":1730}},"stock_drop":{"name":"Qwen3-VL-8B (VLM)","drop":17.3}} \ No newline at end of file diff --git a/huggies/decision-favicon.png b/huggies/decision-favicon.png deleted file mode 100644 index 289b188a5a42c435990cd493a334fd48a83a7be6..0000000000000000000000000000000000000000 --- a/huggies/decision-favicon.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e1dbc0757d579391f3f881da2c208caa0456f4243025f8e09f5e2012ab389355 -size 13916 diff --git a/huggies/decision.png b/huggies/decision.png deleted file mode 100644 index cfacfc1ef9d4720af74400feb3955926feae228f..0000000000000000000000000000000000000000 --- a/huggies/decision.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9d14673a4742f3a6ffcae41dbb83084f16a6e60573a930fa4e2068dd9be1f113 -size 365371 diff --git a/index.html b/index.html index 51ab879d8f522de0ede5de270ff7963e2ce0f5a6..4b04544d1607adfc354fcf6791f46a138cc94e8d 100644 --- a/index.html +++ b/index.html @@ -3,1880 +3,674 @@ -Decision Index 0.4 - - +Jev Reproductions Tracker + + - - - - + + + + + + + + + + -
-
Loading the index…
+ + + +
+
+ +
Category
+
+ Has + + + + + + Sort + + +
+
+
+ +
+
+
+
No artifacts match that
Try a model name, an author handle, or clear the chips above.
+ +
+
+

Still not in the open

+

Everything above approximates the interface (state + typed options → one-pass probabilities). None of it has these:

+
    +
  • TypeSafe's Jev weights. Every "openjev" / "-RLCD" Hub repo is either a stock model plus decoding code, or an independently trained head.
    • +
  • The RLCD algorithm, loss or data. Reinforcement Learning for Calibrated Decisions is described only at blog-post level; no public implementation.
    • +
  • A trained open model matching Jev's calibration claims. Open scorers softmax over candidate logits; the best local study reports ~74% agreement with Jev vs Jev's own 87% on the same subset, and finds confidence does not reliably flag errors.
    • +
  • Archer Hume's promised open-weight version. Announced with the architecture post; training "takes time".
    • +
+
+ +
+
+ +
+
+
+

Missing something? Open a PR against this Space's index.html (the data is one JS array at the bottom of the file) or ping @multimodalart. Hugging Face models and Spaces count as first-class artifacts here, not just GitHub repos.

+

Trending = ♥ likes on X + 5 × GitHub stars + 8 × Hub likes + views ÷ 500. All categories share one grid, color-coded; use the category chips to filter. Promised-but-unreleased items always sort last.

+

Engagement numbers are a one-time snapshot from public X post metadata on 2026-09-17 and will drift. GitHub stars and Hub likes were read the same day. Descriptions paraphrase each project's own README and launch post; corrections welcome.

+

Visual identity and outlined Huggies from HF Huggiverse (Chunte/HFBA). Unofficial, community-maintained, not affiliated with TypeSafe AI.

+
+ +
+
+ diff --git a/methodology.html b/methodology.html deleted file mode 100644 index d2d049b0d5a6f50c40ba38b431d447d54a604997..0000000000000000000000000000000000000000 --- a/methodology.html +++ /dev/null @@ -1,413 +0,0 @@ - - - - - -Methodology · Decision Index 0.4 - - - - - - - - - - - - - -
-
Loading the methodology…
- - - diff --git a/news.html b/news.html deleted file mode 100644 index ab7693c9a99370a8ed0b858cacfd46b02078e267..0000000000000000000000000000000000000000 --- a/news.html +++ /dev/null @@ -1,870 +0,0 @@ - - - - - -Jev Reproductions Tracker - - - - - - - - - - - - - - - - - - - - -
- - - -
-
- -
Category
-
-
- Has - - - - - -
- Sort - - -
-
-
- -
-
-
-
No artifacts match that
Try a model name, an author handle, or clear the chips above.
- -
-
-

Still not in the open

-

Everything above approximates the interface (state + typed options → one-pass probabilities). None of it has these:

-
    -
  • TypeSafe's Jev weights. Every "openjev" / "-RLCD" Hub repo is either a stock model plus decoding code, or an independently trained head.
    • -
  • The RLCD algorithm, loss or data. Reinforcement Learning for Calibrated Decisions is described only at blog-post level; no public implementation.
    • -
  • A trained open model matching Jev's calibration claims. The best open scorers now reach ~90% agreement with Jev (Bespoke Nimble; snapjudge measures 81.2% for Qwen3.8-27B against Jev's own 87.7%), but they still softmax over candidate logits and the studies find confidence does not reliably flag errors.
    • -
  • Archer Hume's promised open-weight version. Announced with the architecture post; training "takes time". A release from his company, Solomon, appeared 2026-09-21; its launch post calls it the best open-weight alternative to Jev, though the model card documents a document-QA model and never uses the word. Listed as a delivery of the promise, not as Jev reconstructed.
    • -
-
- -
-
- -
-
-
-

Missing something? Open a PR against this Space's news.html (the data is one JS array at the bottom of the file) or ping @multimodalart. Hugging Face models and Spaces count as first-class artifacts here, not just GitHub repos.

-

Trending = ♥ likes on X + 5 × GitHub stars + 8 × Hub likes + views ÷ 500. All categories share one grid, color-coded; use the category chips to filter. Promised-but-unreleased items always sort last.

-

Engagement numbers are a snapshot from public X post metadata on 2026-09-24 and will drift. GitHub stars and Hub likes were read the same day. Descriptions paraphrase each project's own README and launch post; corrections welcome.

-

Visual identity and outlined Huggies from HF Huggiverse (Chunte/HFBA). Unofficial, community-maintained, not affiliated with TypeSafe AI.

-
- -
-
- - - - diff --git a/og-index.html b/og-index.html deleted file mode 100644 index 3472b5b9e05ea105d21f2d943611630956e818c5..0000000000000000000000000000000000000000 --- a/og-index.html +++ /dev/null @@ -1,55 +0,0 @@ - -Decision Index 0.3 · social card - -
-

Decision Index 0.3

-
-
Full score · open reproductions of Jev
-
huggingface.co/spaces/multimodalart/jev-decision-index
-
- diff --git a/og-news.png b/og-news.png deleted file mode 100644 index c1d930a74a5f40b3ede3e9c13fa3d146dcea6b39..0000000000000000000000000000000000000000 --- a/og-news.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:17f7a3741ce3956256bdef566dd15a3314abc21ffda0fd60aec0c0d4736469af -size 381629 diff --git a/og-news.html b/og.html similarity index 73% rename from og-news.html rename to og.html index cebcce2df8802132f7f1e70c5dfad05c7b001487..485d74ac1ef5373f5508fc40a79967076ba85f9c 100644 --- a/og-news.html +++ b/og.html @@ -133,7 +133,6 @@ .badge.pr { background: var(--purple); color: #fff; } .badge.paper { background: var(--pink); } .badge.soon { background: var(--orange); border-style: dashed; font-weight: 500; } - .badge.new { background: var(--green); color: #fff; font-weight: 500; letter-spacing: .06em; text-transform: uppercase; } .card h3 { margin: 10px 0 2px; font-size: 18px; line-height: 1.2; font-weight: 700; letter-spacing: -.01em; } .card h3 a { text-decoration: none; } .card h3 a:hover { text-decoration: underline; text-decoration-thickness: 2.5px; text-decoration-color: var(--c); } @@ -250,245 +249,177 @@ const ITEMS = [ desc: 'Batches every JSON key at once and reads category probabilities from stock Qwen 2.5 1B logits, ~5× faster than generating JSON.', quote: 'They were building in stealth for 2 years, I was building in stealth for 2 hours… No new training required.', links: { tweet: 'https://x.com/harshagundal/status/2100044305536889015', hf: 'https://huggingface.co/harshatheg/Qwen-2.5-1B-RLCD', space: 'https://huggingface.co/spaces/drinkmoonshine/parallel-constrained-decoding' }, - m: { likes: 5811, rts: 357, views: 1276259, hfLikes: 475 }, date: '2026-09-16', tags: 'qwen json mac m4 on-device forks epsilon3 shreyansh26 drinkmoonshine' }, + m: { likes: 4922, rts: 259, views: 928013, hfLikes: 259 }, date: '2026-09-16', tags: 'qwen json mac m4 on-device forks epsilon3 shreyansh26 drinkmoonshine' }, { cat: 'decode', title: 'openjev-sglang: Jev-compatible public API', by: { name: 'Eric Zhang', handle: 'ekzhang1' }, desc: 'A drop-in /v1/systemone endpoint over Qwen3.6-35B-A3B. SGLang\'s radix cache keeps the shared prefill hot, so 64 parallel tasks finish in under a second. Prefill-only, no generation.', quote: 'It runs a comparable open model (Qwen3.6-35B-A3B), and just uses SGLang radix cache to preserve the prefill reuse — 64 tasks in <1s.', links: { tweet: 'https://x.com/ekzhang1/status/2100651678110515383', gh: 'https://github.com/ekzhang/openjev-sglang' }, - m: { likes: 978, rts: 61, views: 140654, stars: 221 }, date: '2026-09-17', tags: 'sglang qwen3.6 moe api systemone radix cache' }, + m: { likes: 358, rts: 12, views: 16449, stars: 39 }, date: '2026-09-17', tags: 'sglang qwen3.6 moe api systemone radix cache' }, { cat: 'decode', title: 'openjev + openjev.com', by: { name: 'Theo Lee', handle: 'theoleecj' }, desc: 'Reads option logits on Qwen3.5-4B instead of generating text: 21 decisions in 1.02s vs 5.33s for JSON. CUDA plus a WebGPU browser demo.', quote: 'On the Jev waitlist? You can use not Jev (but close enough) right here.', links: { tweet: 'https://x.com/theoleecj/status/2100243609060856264', gh: 'https://github.com/TheoLeeCJ/openjev', demo: 'https://openjev.com' }, - m: { likes: 10, rts: 0, views: 3319, stars: 2429 }, date: '2026-09-16', tags: 'webgpu browser rtx 3090 qwen3.5 logits mit' }, + m: { likes: 5, rts: 0, views: 2702, stars: 771 }, date: '2026-09-16', tags: 'webgpu browser rtx 3090 qwen3.5 logits mit' }, { cat: 'decode', title: 'PocketJev: on-device iPhone visual decisions', by: { name: 'aqty', handle: 'aqty' }, desc: 'Qwen3-VL running locally on an iPhone via MLX. Camera frame + question + 3 options → direct option logits in ~1s, no photos saved.', quote: 'iPhone上でQwen 2Bをローカル実行して、カメラ画像＋質問＋3択から文章生成なしで直接判定。', links: { tweet: 'https://x.com/aqty/status/2100549517171175807', gh: 'https://github.com/NullPo-jp/PocketJev' }, - m: { likes: 6, rts: 4, views: 439, stars: 0 }, date: '2026-09-17', tags: 'ios iphone mlx vision qwen3-vl camera japan' }, + m: { likes: 4, rts: 2, views: 177, stars: 0 }, date: '2026-09-17', tags: 'ios iphone mlx vision qwen3-vl camera japan' }, { cat: 'decode', title: 'jevmlx: Jev-style decisions for any MLX model', by: { name: 'Beni', handle: 'beni_il_' }, desc: 'Typed schema fields broadcast across the KV cache, one batch row per field, all scored in a single forward pass on any MLX model.', quote: 'If you can run Qwen on your machine, it can behave very similarly to Jev.', links: { tweet: 'https://x.com/beni_il_/status/2100617387116568956', gh: 'https://github.com/bnsd55/jevmlx' }, - m: { likes: 7, rts: 0, views: 1185, stars: 46 }, date: '2026-09-17', tags: 'mlx apple silicon kv cache hebrew' }, + m: { likes: 7, rts: 0, views: 912, stars: 9 }, date: '2026-09-17', tags: 'mlx apple silicon kv cache hebrew' }, { cat: 'decode', title: 'jev-on-a-laptop: the honest benchmark', by: { name: 'Richard Becker', handle: 'RichardBcker1' }, desc: 'Qwen 2.5 1.5B/7B and Qwen3 8B on a laptop vs Jev\'s published cases: 7B hits 73.8% agreement to Jev\'s 86.6%, 0.4–2s per decision. Finds confidence does not reliably flag errors.', quote: 'All code, benchmarks, model-size comparisons and the full write-up (unofficial, no affiliation with TypeSafe).', links: { tweet: 'https://x.com/RichardBcker1/status/2100195276107530393', gh: 'https://github.com/rorshopping/jev-on-a-laptop', space: 'https://huggingface.co/spaces/rorshopping/parallel-constrained-decisions' }, - m: { likes: 1, rts: 0, views: 68, stars: 22 }, date: '2026-09-16', tags: 'benchmark study agreement fraud apple silicon m5 writeup explainer' }, + m: { likes: 1, rts: 0, views: 58, stars: 12 }, date: '2026-09-16', tags: 'benchmark study agreement fraud apple silicon m5 writeup explainer' }, { cat: 'decode', title: 'jevfire: one context, many decisions on CUDA', by: { name: 'Enrique Alcázar', handle: 'enriquealcazarg' }, desc: 'Parallel decisions for CUDA LLMs behind a vLLM-style API, with game-agent examples. Claims 10×+ over regular inference, runs in the browser, accepts images.', quote: 'Here you have an open-source approach to the same problem… uses the same principles as JEV, is type-safe, runs in your browser, and accepts images.', links: { tweet: 'https://x.com/enriquealcazarg/status/2100267330022375858', gh: 'https://github.com/kikoncuo/jevfire', site: 'https://kikoncuo.github.io/jevfire/' }, - m: { likes: 1, rts: 0, views: 252, stars: 8 }, date: '2026-09-16', tags: 'cuda vllm browser images games primeagen' }, + m: { likes: 1, rts: 0, views: 243, stars: 3 }, date: '2026-09-16', tags: 'cuda vllm browser images games primeagen' }, { cat: 'decode', title: 'LFM2.5-RLCD (350M & 2.6B): PCD on Liquid models', by: { name: 'notnotsamuel · monotykamary', handle: null }, desc: 'Parallel constrained decoding on unchanged Liquid LFM2.5 weights, branching attention and convolution state. Inference-only: 8–63× faster, field accuracy still ~60%.', links: { hf: 'https://huggingface.co/notnotsamuel/LFM2.5-350M-RLCD', hf2: 'https://huggingface.co/monotykamary/LFM2.5-2.6B-RLCD' }, - m: { hfLikes: 25 }, date: '2026-09-16', tags: 'liquid lfm hub weights renamed inference only pcd' }, + m: { hfLikes: 6 }, date: '2026-09-16', tags: 'liquid lfm hub weights renamed inference only pcd' }, { cat: 'decode', title: 'parallelConstraintDecoding (Java + Python)', by: { name: 'Stephan Janssen', handle: 'Stephan007' }, desc: 'A Java port of the decode trick alongside a Python reference.', quote: 'Here\'s a java version of Jev 🤩', links: { tweet: 'https://x.com/Stephan007/status/2100673219967557906', gh: 'https://github.com/stephanj/parallelConstraintDecoding' }, - m: { likes: 3, rts: 0, views: 104, stars: 12 }, date: '2026-09-17', tags: 'java jvm port' }, + m: { likes: 2, rts: 0, views: 53, stars: 3 }, date: '2026-09-17', tags: 'java jvm port' }, { cat: 'decode', title: 'system-one (snellingio): "System One Lite"', by: { name: 'snellingio', handle: null }, desc: 'Choice / Score / Noul server over MLX Qwen3, mirroring TypeSafe\'s three primitives. No launch post found.', links: { gh: 'https://github.com/snellingio/system-one' }, - m: { stars: 39 }, date: '2026-09-16', tags: 'mlx qwen3 choice score noul server' }, + m: { stars: 26 }, date: '2026-09-16', tags: 'mlx qwen3 choice score noul server' }, { cat: 'decode', title: 'open-jev (daseinlabs): Gemma 3 4B on MLX', by: { name: 'Dasein Labs', handle: null }, desc: 'Prefix-cache option scoring on a local Gemma 3 4B via MLX, a Doom demo, and a /v1/systemone shim. Only seen in roundups.', links: { gh: 'https://github.com/daseinlabs/open-jev' }, - m: { stars: 69 }, date: '2026-09-17', tags: 'gemma mlx doom systemone shim' }, + m: { stars: 7 }, date: '2026-09-17', tags: 'gemma mlx doom systemone shim' }, { cat: 'decode', title: 'openvons (open-Jev, Japanese)', by: { name: 'genai-craft', handle: null }, desc: 'Decision layer answering finite-option questions with probabilities, from text, images or Japanese voice commands.', links: { gh: 'https://github.com/genai-craft/openvons' }, - m: { stars: 12 }, date: '2026-09-17', tags: 'japanese voice image multimodal' }, + m: { stars: 5 }, date: '2026-09-17', tags: 'japanese voice image multimodal' }, { cat: 'decode', title: 'openjev (zhihz): bilingual local decisions', by: { name: 'zhihz', handle: null }, desc: 'Local bilingual probability decisions from context, questions and candidate answers.', links: { gh: 'https://github.com/zhihz/openjev' }, - m: { stars: 22 }, date: '2026-09-16', tags: 'chinese bilingual local' }, + m: { stars: 3 }, date: '2026-09-16', tags: 'chinese bilingual local' }, { cat: 'decode', title: 'system-one (sgoedecke): batched single-token choice', by: { name: 'Sean Goedecke', handle: null }, desc: 'Batched single-token choice inference for open models, wire-compatible with TypeSafe.', links: { gh: 'https://github.com/sgoedecke/system-one' }, - m: { stars: 20 }, date: '2026-09-17', tags: 'compatible batched single token' }, - { cat: 'decode', title: 'reflex: stock Qwen3.5 / Qwen3-VL behind /v1/systemone', by: { name: 'Kshetrajna Raghavan', handle: 'kshetrajna' }, - desc: 'Runs Jev\'s shape on stock Qwen3.5: choice / score / noul in one batched pass, a /v1/systemone server, and a WebGPU demo on a 0.8B model. Temperature scaling gets Qwen3.5-4B to 0.039 ECE vs Jev\'s 0.031; a LoRA recipe trains honest probabilities on public data.', - quote: '"Wonder if we could build that?" is a pretty normal response to new tech at Shopify. Jev got me curious, so I built Reflex: a Qwen-based experiment in structured decisions + probabilities running on WebGPU.', - links: { tweet: 'https://x.com/kshetrajna/status/2100739853101195744', gh: 'https://github.com/kshetrajna12/reflex', demo: 'https://kshetrajna12.github.io/reflex/' }, - m: { likes: 330, rts: 22, views: 139746, stars: 87 }, date: '2026-09-17', tags: 'systemone choice score noul qwen3-vl image shared state server webgpu browser demo lora calibration ece shopify mmlu' }, - { cat: 'decode', title: 'LitJev: Jev-schema decision layer for the Qwen family', by: { name: 'Zhengxu Yu', handle: 'yuzxfred' }, - desc: 'Shared prefill, forked KV/recurrent state, one cached branch per question, option logits read from lm_head on any Qwen checkpoint (default Qwen3.8-27B), behind the exact Jev /v1/systemone schema. Playground, MMLU-Pro eval, optional temperature calibration.', - quote: 'Qwen3.8-27B，10 道 MMLU-Pro 一次完成，0.3 s。比让模型只生成答案快 2.9x。非官方，基于公开信息。', - links: { tweet: 'https://x.com/yuzxfred/status/2100652136878981337', gh: 'https://github.com/zhengxuyu/litjev', blog: 'https://x.com/yuzxfred/status/2100896621173047367' }, - m: { likes: 2, rts: 0, views: 161, stars: 28 }, date: '2026-09-17', tags: 'qwen qwen3.8 27b systemone schema compatible calibration mmlu-pro doom chess vision transformers chinese explainer writeup limitations' }, + m: { stars: 0 }, date: '2026-09-17', tags: 'compatible batched single token' }, /* ---------------- trained ---------------- */ - { cat: 'decode', title: 'snapjudge: TypeSafe-compatible decisions on a Mac, scored against Jev', by: { name: 'Michael Gross', handle: 'michagross' }, - desc: 'Choice / Score / Noul behind /v1/systemone on MLX Qwen3.5-family models, probabilities read straight from the logits. On TypeSafe\'s 20 public cases (373 decisions) Qwen3.8-27B reaches 81.2% agreement vs Jev\'s 87.7%: level on short texts, behind on invoices.', - links: { tweet: 'https://x.com/michagross/status/2101062157290856604', gh: 'https://github.com/Micha0827/snapjudge' }, - m: { stars: 7, likes: 1, rts: 0, views: 12 }, date: '2026-09-18', tags: 'mlx apple silicon qwen3.6 qwen3.8 systemone api benchmark agreement german runner game' }, - { cat: 'trained', title: 'Bespoke Nimble: open data, open model, open recipe', by: { name: 'Mahesh Sathiamoorthy', handle: 'madiator' }, - desc: 'The first fully open recipe: LoRA on Qwen3.5-9B with the data curation published too. Contrastive curation flips one fact to make negatives, so training needs no probabilities. 90.1% agreement with Jev vs 66.4% for base Qwen.', - quote: 'Introducing Bespoke Nimble: an open data, open model, open recipe for an open Jev.', - links: { tweet: 'https://x.com/madiator/status/2100990591215783946', gh: 'https://github.com/bespokelabsai/nimble', hf: 'https://huggingface.co/bespokelabs/Bespoke-Nimble-9B' }, - m: { likes: 1324, rts: 171, views: 495403, stars: 1073, hfLikes: 114 }, date: '2026-09-18', tags: 'bespokelabs nimble qwen3.5 9b lora contrastive data curation open recipe synthetic distillation free parallel scorer 100ms h100 macbook' }, { cat: 'trained', title: 'jevlike: train your own one-pass option scorer', by: { name: 'Vinny LaRouge', handle: 'vinnylarouge' }, desc: 'A tiny byte encoder (or frozen HF encoder) plus an option head maps context + N options to one probability each. Doom, chess and Wikispeedia demos; parent of several forks.', quote: 'I reverse-engineered a jev-like architecture given its type. You can find the repo here to train your own jevlikes.', links: { tweet: 'https://x.com/vinnylarouge/status/2100170846346097083', gh: 'https://github.com/vinnylarouge/jevlike', demo: 'https://x.com/vinnylarouge/status/2100281651930513460' }, - m: { likes: 2017, rts: 168, views: 165419, stars: 1077 }, date: '2026-09-16', tags: 'train scorer encoder doom chess wikispeedia rl' }, + m: { likes: 1888, rts: 156, views: 152455, stars: 661 }, date: '2026-09-16', tags: 'train scorer encoder doom chess wikispeedia rl' }, { cat: 'trained', title: 'openjev (AlexWortega): Qwen3.5 + MLP head, real weights', by: { name: 'Alex Wortega', handle: 'justALEXWORTEGA' }, desc: 'The only trained "openjev" checkpoint on the Hub: Qwen3.5-4B as an NLI cross-encoder, argmax entailment is the move. Plays Doom zero-shot from text and pixels; 35B-A3B heads included.', quote: 'Typesafe: pnewed 💨 Jev: liberated 🫡 I trained an MLP on top of qwen 4b and it works literally like JEV', links: { tweet: 'https://x.com/justALEXWORTEGA/status/2100341039986798930', hf: 'https://huggingface.co/AlexWortega/openjev', gh: 'https://github.com/mikesmullin/openjev' }, - m: { likes: 773, rts: 47, views: 104277, hfLikes: 310, stars: 3 }, date: '2026-09-16', tags: 'nli cross-encoder mlp qwen3.5 doom vision weights hub' }, + m: { likes: 760, rts: 46, views: 96076, hfLikes: 84, stars: 1 }, date: '2026-09-16', tags: 'nli cross-encoder mlp qwen3.5 doom vision weights hub' }, { cat: 'trained', title: 'decider-2b: full fine-tune of Qwen3.5-2B-Base', by: { name: 'Mapika', handle: null }, desc: 'Full fine-tune of Qwen3.5-2B-Base on 942k examples from 64 decision datasets, plus a vision variant. Weights on the Hub.', links: { gh: 'https://github.com/Mapika/decider', hf: 'https://huggingface.co/Mapika/decider-2b', hf2: 'https://huggingface.co/Mapika/decider-2b-vision' }, - m: { stars: 156, hfLikes: 64 }, date: '2026-09-16', tags: 'fine-tune qwen3.5 2b calibrated vision gh200 hub weights' }, - { cat: 'trained', title: 'jeff: a self-hosted drop-in jev on a 400M encoder', by: { name: 'Logan Markewich', handle: 'LoganMarkewich' }, - desc: 'A drop-in /v1/systemone endpoint on GLiFormer-large (400M), the encoder-classifier route: ~$2.6 per 1M requests against jev\'s ~$15.6. Officially on JevBench v1.2.2 at 66.9 (#9 of 18, #14 on intelligence) vs jev\'s 75.3; far behind on irony and the hard tier.', - quote: 'jev itself can basically be boiled down to a classfier or an encoder, just like GliFormer. Latency is the same, cost is cheaper, and with only a mild hit in accuracy.', - links: { tweet: 'https://x.com/LoganMarkewich/status/2101142971223781587', gh: 'https://github.com/logan-markewich/jeff', hf: 'https://huggingface.co/knowledgator/gliformer-large-v1' }, - m: { likes: 462, rts: 46, views: 32400, stars: 167, hfLikes: 137 }, date: '2026-09-19', tags: 'gliformer encoder classifier gliner knowledgator cost latency modal self-hosted typesafe-sdk jevbench drop-in' }, - { cat: 'trained', title: 'Kev: a 0.6B/4B/8B family you can train yourself', by: { name: 'Jared Palmer', handle: 'jaredpalmer' }, - desc: 'LoRA plus a small pointer head on Qwen3, scaled to 0.6B/4B/8B with training code, frozen eval suites and Apache-2.0 weights, plus a later Qwen3.5 9B. Drop-in System One API. Kev-4B trains in 40 min on one H100; out of domain Kev-8B hits 79.6% vs Jev\'s 85.7%.', - quote: 'UPDATE: Kev-0.6B, 4B, and 8B are now available. Kev is a family of small open source Jev-like decision models you can train and run yourself.', - links: { tweet: 'https://x.com/jaredpalmer/status/2101715352258232539', gh: 'https://github.com/jaredpalmer/kev', hf: 'https://huggingface.co/collections/jaredpalmer/kev-6aad9d0ea49f2589665e07cd' }, - m: { likes: 1188, rts: 98, views: 78473, stars: 993 }, date: '2026-09-20', tags: 'qwen3 qwen3.5 lora pointer head apache train your own typesafe sdk drop-in apple silicon kv cache h100 mmlu calibration' }, - { cat: 'trained', title: 'rlcd-modernbert-151m ("OpenJev Verdict")', by: { name: 'heman10x', handle: 'heman10x' }, + m: { stars: 4, hfLikes: 1 }, date: '2026-09-16', tags: 'fine-tune qwen3.5 2b calibrated vision gh200 hub weights' }, + { cat: 'trained', title: 'rlcd-modernbert-151m ("OpenJev Verdict")', by: { name: 'heman10x', handle: null }, desc: 'Post-trained ModernBERT (151M) scoring 25 candidate slots, abstention included, in under 35ms. WebGPU playground, Apache-2.0.', - links: { tweet: 'https://x.com/heman10x/status/2100836659533336676', hf: 'https://huggingface.co/heman10x/rlcd-modernbert-151m', gh: 'https://github.com/Heman10x-NGU/Verdict-open-jev' }, - m: { likes: 17, rts: 1, views: 7780, hfLikes: 16, stars: 42 }, date: '2026-09-17', tags: 'modernbert encoder webgpu abstention gliclass hub weights' }, + links: { hf: 'https://huggingface.co/heman10x/rlcd-modernbert-151m', gh: 'https://github.com/Heman10x-NGU/Verdict-open-jev' }, + m: { hfLikes: 0, stars: 1 }, date: '2026-09-17', tags: 'modernbert encoder webgpu abstention gliclass hub weights' }, { cat: 'trained', title: 'open-jev (pngwn): Qwen3.5-4B decision scorer + demo Space', by: { name: 'pngwn', handle: 'evilpingwin' }, desc: 'Qwen3.5-4B LoRA fine-tune that scores each candidate answer for a typed question, with temperature scaling so confidence tracks accuracy. Weights (v1, v2b) and a demo Space on the Hub.', quote: 'Since everyone is open-jevving. I set ML Intern (on hugging chat) to work. After a few experiments and a little guidance and we did ok.', links: { tweet: 'https://x.com/evilpingwin/status/2100721265107751121', space: 'https://huggingface.co/spaces/pngwn/open-jev', hf: 'https://huggingface.co/pngwn/system-one-qwen3.5-4b-scorer-v2b', hf2: 'https://huggingface.co/pngwn/system-one-qwen3.5-4b-scorer' }, - m: { likes: 243, rts: 37, views: 14097, hfLikes: 14 }, date: '2026-09-17', tags: 'qwen3.5 scoring head temperature scaling calibration ml intern huggingchat hub weights space demo' }, + m: { likes: 0, rts: 0, views: 56, hfLikes: 0 }, date: '2026-09-17', tags: 'qwen3.5 scoring head temperature scaling calibration ml intern huggingchat hub weights space demo' }, { cat: 'trained', title: 'jevbetter: a stronger jevlike', by: { name: 'olanotolu', handle: null }, desc: 'jevlike with a fancier head: hashed n-gram encoder, rival-aware attention, temperature scaling. Benchmarked against jevlike.', links: { gh: 'https://github.com/olanotolu/jevbetter' }, - m: { stars: 13 }, date: '2026-09-16', tags: 'n-gram attention temperature scaling calibration' }, + m: { stars: 9 }, date: '2026-09-16', tags: 'n-gram attention temperature scaling calibration' }, { cat: 'trained', title: 'system-one-open: Gemma 4 E2B / Gemma 3 270M + head', by: { name: 'mithalouni', handle: null }, desc: 'Typed calibrated decisions in one pass on Gemma 4 E2B and Gemma 3 270M, trained on Modal.', links: { gh: 'https://github.com/mithalouni/system-one-open' }, - m: { stars: 18 }, date: '2026-09-17', tags: 'gemma modal scoring head' }, + m: { stars: 1 }, date: '2026-09-17', tags: 'gemma modal scoring head' }, { cat: 'trained', title: 'qwen-rlcd: Qwen3.5-0.8B Choice/Score/Noul', by: { name: 'shamazharikh', handle: null }, desc: 'Choice / Score / Noul on Qwen3.5-0.8B. Inference works; training marked on hold.', links: { gh: 'https://github.com/shamazharikh/qwen-rlcd' }, - m: { stars: 3 }, date: '2026-09-16', tags: 'qwen3.5 0.8b small' }, + m: { stars: 1 }, date: '2026-09-16', tags: 'qwen3.5 0.8b small' }, { cat: 'trained', title: 'system-one-gemma: Gemma 3 270M scoring head', by: { name: 'akash-kamat', handle: null }, desc: 'Gemma 3 270M with a scoring head; single forward pass, no text generation.', links: { gh: 'https://github.com/akash-kamat/system-one-gemma' }, - m: { stars: 3 }, date: '2026-09-17', tags: 'gemma 270m tiny' }, - { cat: 'trained', title: 'Laya: 421M RLCD decision model + demo Space', by: { name: 'Convai Innovations', handle: null }, - desc: 'ModernBERT-large fine-tuned plus a decision head trained from scratch (421M total) with RLCD against proper scoring rules: choice/score/noul with calibrated probabilities, all questions in one pass. Weights and a 7-tab Gradio demo on the Hub.', - links: { gh: 'https://github.com/NandhaKishorM/laya', hf: 'https://huggingface.co/convaiinnovations/laya', space: 'https://huggingface.co/spaces/convaiinnovations/laya-demo', blog: 'https://dev.to/nandakishor_m_6cc0adfde9f/i-built-non-autoregressive-decision-models-a-year-ago-then-a-frontier-lab-called-it-a-18me', site: 'https://www.reddit.com/r/LocalLLaMA/comments/1wjieap/made_the_horizontal_opensource_model_for_jev_with/' }, - m: { stars: 3908, hfLikes: 1065 }, date: '2026-09-18', tags: 'modernbert rlcd calibrated choice score noul support triage guardrails rag moderation routing hub weights space demo github pypi reddit devto' }, - { cat: 'trained', title: 'Decision 1.0: four open decision models + Studio Space', by: { name: 'vLLM Semantic Router', handle: null }, - desc: 'Four Apache-2.0 models from the vLLM Semantic Router team: Kai/Lex (572M encoders) and Sol (1.88B) / Nox (4.21B) decoders doing Choice, Noul and Score in one pass, 50 languages, plus a Studio Space. Kai outscores both Laya baselines in five of six panels.', - links: { hf: 'https://huggingface.co/collections/llm-semantic-router/decision-10-6ab12177bd0002394d8409f9', hf2: 'https://huggingface.co/llm-semantic-router/Decision-1.0-Kai', space: 'https://huggingface.co/spaces/llm-semantic-router/decision-studio' }, - m: { hfLikes: 2 }, date: '2026-09-21', tags: 'vllm semantic router decision 1.0 kai lex sol nox vela encoder qwen3.5 choice noul score ordinal regression multilingual belebele 50 languages apache system one adapter decision studio amd rocm' }, + m: { stars: 0 }, date: '2026-09-17', tags: 'gemma 270m tiny' }, { cat: 'trained', title: 'System One Mini: 69M DistilBERT, five fixed decisions', by: { name: 'David Hatley', handle: null }, desc: '69M DistilBERT with five fixed typed heads over synthetic software-diagnosis summaries. Its card says: not a reproduction of Jev or RLCD.', links: { hf: 'https://huggingface.co/DavidHatley/system-one-mini' }, m: { hfLikes: 0 }, date: '2026-09-16', tags: 'distilbert fixed schema multitask hub weights' }, - { cat: 'trained', title: 'NanoJev: 0.6B Qwen3 with a real decision head', by: { name: 'TianyuCodings', handle: null }, - desc: 'A 0.6B Qwen3 backbone with an actual trained head emitting Choice / Boolean / Score across 2–255 candidates in one forward pass. Full training pipeline, maze and snake runs, weights and data on the Hub. The one Chinese and Korean roundups are pushing hardest.', - links: { gh: 'https://github.com/TianyuCodings/NanoJev', hf: 'https://huggingface.co/C-Tianyu/NanoJev' }, - m: { stars: 1386, hfLikes: 63 }, date: '2026-09-17', tags: 'nanojev 0.6b qwen3 decision head choice boolean score maze snake training pipeline weights hub china korea' }, - { cat: 'trained', title: 'mini-jev: a preregistered measurement of the trick', by: { name: 'r-ms', handle: null }, - desc: 'Not another clone: a preregistered study of letter-logit versus grammar-JSON decoding on frozen Qwen3-4B and CLINC150. An actual measurement, no weights trained.', - links: { gh: 'https://github.com/r-ms/mini-jev' }, - m: { stars: 33 }, date: '2026-09-17', tags: 'preregistered study letter logits grammar json clinc150 qwen3-4b measurement' }, - - { cat: 'trained', title: 'Solomon: the open-weight release Archer Hume promised', by: { name: 'Doccy (Archer Hume)', handle: '4rcherhume' }, - desc: 'Archer Hume\'s promised open-weight release, from his company Doccy: a rank-64 LoRA plus trained heads scoring five question types as letter-logit probabilities, no generation. His post calls it the best open-weight alternative to Jev; real panel 0.8131.', - quote: 'Yes we are a healthcare company. Yes we just released the best open-weight alternative to Jev, Solomon 27b.', - links: { tweet: 'https://x.com/4rcherhume/status/2101888238357237798', hf: 'https://huggingface.co/DoccyHealth/Solomon', blog: 'https://archerhume.com/posts/jevs-architecture-unmasked/' }, - m: { likes: 15, rts: 3, views: 304, hfLikes: 1 }, date: '2026-09-21', tags: 'qwen3.8 27b lora trained heads letter logits document decisions calibration apache promised delivered doccy archer melbourne alternative' }, { cat: 'trained', promised: true, title: 'Archer Hume\'s open-weight Jev (promised)', by: { name: 'Archer Hume', handle: '4rcherhume' }, - desc: 'Announced alongside the architecture teardown: an open-weight version trained on the inferred design. Delivered 2026-09-21 as Solomon, from Hume\'s company Doccy, whose launch post calls it the best open-weight alternative to Jev.', + desc: 'Announced alongside the architecture teardown: an open-weight version trained on the inferred design. Training in progress, nothing released yet.', quote: 'I decided I\'d release an open-weight version. BUT training takes time, so while we all wait I decided I\'d drop the sauce.', links: { tweet: 'https://x.com/4rcherhume/status/2100555442061820286', blog: 'https://archerhume.com/posts/jevs-architecture-unmasked/' }, - m: { likes: 2248, rts: 212, views: 379074 }, date: '2026-09-17', tags: 'promised coming soon open-weight training archer' }, + m: { likes: 1512, rts: 132, views: 197990 }, date: '2026-09-17', tags: 'promised coming soon open-weight training archer' }, /* ---------------- prior ---------------- */ { cat: 'prior', title: 'GLiNER 2.5: "Weights are on Hugging Face."', by: { name: 'George (Fastino / GLiNER)', handle: 'george_onx' }, desc: 'Points the launch thread at the GLiNER 2.5 collection: a pretrained zero-shot encoder-classifier already doing single-pass classification over caller-supplied labels.', quote: 'We\'ve been reading the comments. Weights are on Hugging Face.', links: { tweet: 'https://x.com/george_onx/status/2100293114808119379', hf: 'https://huggingface.co/collections/fastino/gliner25-models' }, - m: { likes: 1587, rts: 94, views: 224731 }, date: '2026-09-16', tags: 'gliner fastino zero-shot encoder classifier collection' }, + m: { likes: 1392, rts: 81, views: 197776 }, date: '2026-09-16', tags: 'gliner fastino zero-shot encoder classifier collection' }, { cat: 'prior', title: 'DeepMost sales-conversion RL model + papers', by: { name: 'Nandakishor M', handle: 'Nandakishorm1' }, desc: 'Claims the idea shipped in 2025 papers (arXiv 2503.23303, 2510.01237) with a Hub model and dataset. PPO over embeddings; loosely related.', quote: 'Also the second work published in September 2025 was exactly the same one jev proposed now.', links: { tweet: 'https://x.com/Nandakishorm1/status/2100451360873611666', hf: 'https://huggingface.co/DeepMostInnovations/sales-conversion-model-reinf-learning', paper: 'https://arxiv.org/abs/2510.01237' }, - m: { likes: 5, rts: 0, views: 251, hfLikes: 147 }, date: '2026-09-17', tags: 'ppo embeddings arxiv prior art sales' }, + m: { likes: 0, rts: 0, views: 16, hfLikes: 77 }, date: '2026-09-17', tags: 'ppo embeddings arxiv prior art sales' }, /* ---------------- arch ---------------- */ { cat: 'explain', title: 'Jev\'s Architecture Unmasked', by: { name: 'Archer Hume', handle: '4rcherhume' }, desc: 'Argues Jev is a causal transformer with shared-state encoding, per-question branches and direct probability readouts, likely on a sparse MoE backbone, inferred from latency probing.', quote: 'All the grifters are completely wrong about Jev\'s architecture so I decided I\'d release an open-weight version. BUT training takes time…', links: { tweet: 'https://x.com/4rcherhume/status/2100555442061820286', blog: 'https://archerhume.com/posts/jevs-architecture-unmasked/' }, - m: { likes: 2246, rts: 212, views: 378647 }, date: '2026-09-17', tags: 'moe teardown speculation open-weight promise blog' }, + m: { likes: 1512, rts: 132, views: 197990 }, date: '2026-09-17', tags: 'moe teardown speculation open-weight promise blog' }, /* ---------------- diffusion ---------------- */ { cat: 'diffusion', title: 'DiffusionGemma-as-Jev + vLLM structured-generation patch', by: { name: 'Matt Mastracci', handle: 'mmastrac' }, desc: 'Adds a Jev-like structured mode for DiffusionGemma to vLLM (PR #57250) and runs live evals: not slower than Jev\'s API and roughly tied on quality.', quote: 'Is Jev faster than DiffusionGemma? No ❌ Is Jev smarter than DiffusionGemma? No ❌ (they\'re roughly tied!)', links: { tweet: 'https://x.com/mmastrac/status/2100626193943052784', pr: 'https://github.com/vllm-project/vllm/pull/57250' }, - m: { likes: 1094, rts: 84, views: 414483 }, date: '2026-09-17', tags: 'diffusion gemma vllm dgx spark evals' }, - { cat: 'diffusion', title: 'razorback16/openjev + api.codiv.ai host', by: { name: 'razorback16', handle: null }, - desc: 'The same vLLM PR #57250 route, packaged as a Jev-wire /v1/systemone service: a DiffusionGemma 26B-A4B canvas with only the label slots left as noise, one read-only denoise step reads out the distribution. Public host at api.codiv.ai.', - links: { gh: 'https://github.com/razorback16/openjev', site: 'https://api.codiv.ai/v1/systemone' }, - m: { stars: 201 }, date: '2026-09-17', tags: 'diffusiongemma 26b a4b nvfp4 vllm 57250 hosted api codiv systemone denoise canvas' }, + m: { likes: 352, rts: 20, views: 21545 }, date: '2026-09-17', tags: 'diffusion gemma vllm dgx spark evals' }, { cat: 'diffusion', title: 'open-jev (JoshuaSP): typed JSON with DiffusionGemma', by: { name: 'Joshua SP', handle: null }, desc: 'Typed JSON inference with DiffusionGemma, benchmarked against Every and Jev.', links: { gh: 'https://github.com/JoshuaSP/open-jev' }, - m: { stars: 20 }, date: '2026-09-16', tags: 'diffusion gemma benchmark every' }, - { cat: 'diffusion', title: 'LocalJev (GitHub Next): oMLX Jev API + model bake-off', by: { name: 'GitHub Next', handle: 'GitHubNext' }, - desc: 'Official GitHub Next bridge, since not everyone there has Jev access: a Bun server putting oMLX DiffusionGemma behind /v1/systemone. Prompts for JSON probabilities instead of reading logits: wire-compatible, not equivalent. Ships a 1,200-request bake-off.', - quote: 'Not everyone on the team has access to Jev yet. Spent a morning cobbling together a poor man\'s Jev on top of omlx for local use.', - links: { tweet: 'https://x.com/GitHubNext/status/2101193436816920798', gh: 'https://github.com/githubnext/localjev' }, - m: { likes: 1500, rts: 188, views: 221937, stars: 606 }, date: '2026-09-19', tags: 'github next localjev omlx bun typescript diffusiongemma 26b a4b json probabilities bakeoff m5 max benchmark qwen gemma moe wire compatible promptcoding' }, + m: { stars: 9 }, date: '2026-09-16', tags: 'diffusion gemma benchmark every' }, /* ---------------- explain ---------------- */ - { cat: 'explain', title: 'jev-ood-calibration: does Jev know when it doesn\'t know?', by: { name: 'scienthoon', handle: null }, - desc: 'Calibration test on 900 rule-generated tickets Jev cannot have seen plus OBQA/CSQA/HellaSwag via AI Gateway. Refit temperature 2.7 on the unseen task vs 0.96–1.35 on the benchmarks; Choice/Score overconfident, Boolean underconfident. Raw responses published.', - links: { gh: 'https://github.com/scienthoon/jev-ood-calibration' }, - m: { stars: 3 }, date: '2026-09-19', tags: 'calibration ece temperature ood synthetic benchmark gateway' }, { cat: 'explain', title: '45-second TL;DR on Jev (video)', by: { name: 'Matija Sosic', handle: 'MatijaSosic' }, desc: 'The most-shared explainer of the launch: a 45-second video that strips Jev down to its core idea, made because the official launch video was hard to follow.', quote: 'I find the core idea beautifully simple, but the video made it really hard to understand.', links: { tweet: 'https://x.com/MatijaSosic/status/2100190746389135772' }, - m: { likes: 9450, rts: 691, views: 1658695 }, date: '2026-09-16', tags: 'video tldr explainer launch' }, + m: { likes: 7279, rts: 514, views: 1089551 }, date: '2026-09-16', tags: 'video tldr explainer launch' }, { cat: 'explain', title: 'typesafe-ai-benchmark: an "imposter Jev" on Cerebras', by: { name: 'Shannon Duncan', handle: 'iamMrDuncan' }, desc: 'An "imposter Jev" gateway on Cerebras with Qwen 3.8 27B, compared side by side. Similar quality and speed; TypeSafe far cheaper.', quote: 'Built an alternative version of @typesafeai but on @cerebras with Qwen 3.8 27b. Similar quality, similar performance, but vastly different cost.', links: { tweet: 'https://x.com/iamMrDuncan/status/2100467548298899918', gh: 'https://github.com/iammrduncan/typesafe-ai-benchmark' }, - m: { likes: 209, rts: 25, views: 26087, stars: 34 }, date: '2026-09-17', tags: 'cerebras qwen gateway benchmark comparison cost' }, + m: { likes: 201, rts: 24, views: 23611, stars: 25 }, date: '2026-09-16', tags: 'cerebras qwen gateway benchmark comparison cost' }, { cat: 'explain', title: 'awesome-jev lists (AnotiaWang · hellogumbo)', by: { name: 'AnotiaWang · hellogumbo', handle: null }, desc: 'Two curated indexes; AnotiaWang\'s "Research & Open Models" section is the tight one, hellogumbo\'s catches the long tail.', links: { gh: 'https://github.com/AnotiaWang/awesome-jev', gh2: 'https://github.com/hellogumbo/awesome-jev' }, - m: { stars: 215 }, date: '2026-09-16', tags: 'awesome list index curated roundup' }, + m: { stars: 16 }, date: '2026-09-16', tags: 'awesome list index curated roundup' }, { cat: 'explain', title: 'jev-benchmarks: probability-aware eval harness', by: { name: 'AbdelStark', handle: null }, desc: 'Calibration, selective risk and latency evals for typed decision models, Jev or open.', links: { gh: 'https://github.com/AbdelStark/jev-benchmarks' }, - m: { stars: 12 }, date: '2026-09-17', tags: 'calibration selective risk eval harness' }, + m: { stars: 6 }, date: '2026-09-17', tags: 'calibration selective risk eval harness' }, { cat: 'explain', title: 'Thread: System 1 / System 2 agents and the open recreations', by: { name: 'hhkkmon', handle: 'hhkkmon' }, desc: '14-part explainer framing agents as "expensive LLM calls pretending to be if statements", listing the open recreations at the end.', quote: 'Big models think. Small decision models react.', links: { tweet: 'https://x.com/hhkkmon/status/2100443314957038010' }, - m: { likes: 2, rts: 0, views: 120 }, date: '2026-09-17', tags: 'thread explainer system 1 system 2 agents' }, + m: { likes: 2, rts: 0, views: 83 }, date: '2026-09-17', tags: 'thread explainer system 1 system 2 agents' }, { cat: 'explain', title: 'Long-form: "OpenJev is NOT Jev open-sourced"', by: { name: 'mygtmhire', handle: 'mygtmhire' }, desc: 'Most thorough single post on TheoLeeCJ/openjev, including the warning that its probabilities are not calibrated confidence.', quote: 'State in → probabilities out → code acts. No conversation required.', links: { tweet: 'https://x.com/mygtmhire/status/2100666488625734103' }, - m: { likes: 2, rts: 0, views: 126 }, date: '2026-09-17', tags: 'explainer openjev routing calibration' }, - { cat: 'explain', title: 'jev-behavior-study: probing the closed model', by: { name: 'RINNECODER', handle: null }, - desc: 'Black-box behavioural study of the closed Jev 1.13.0 itself. Not a replica — it measures the thing everyone is trying to reproduce.', - links: { gh: 'https://github.com/RINNECODER/jev-behavior-study' }, - m: { stars: 3 }, date: '2026-09-17', tags: 'behavior study black box probing jev 1.13.0 closed model' }, + m: { likes: 0, rts: 0, views: 55 }, date: '2026-09-17', tags: 'explainer openjev routing calibration' }, ]; /* ===================================================================== */ @@ -514,7 +445,6 @@ function typesOf(it) { } function badgesOf(it) { const L = it.links || {}; const b = []; - if (isNew(it)) b.push(['new', 'new']); if (it.promised) b.push(['soon', 'coming soon / promised']); if (L.gh || L.gh2) b.push(['gh', 'GitHub']); if (L.hf || L.hf2) b.push(['hf', '🤗 weights']); @@ -528,28 +458,9 @@ function badgesOf(it) { } const likesOf = it => it.m?.likes ?? -1; /* Trending: one weighted number across platforms. A GitHub star or a Hub like is a - stronger signal than a like on X, so they are weighted up; views add a little. - A recency credit is added on top, so a fresh release is not buried under older, - louder ones. It decays daily (halving every day), so a week-old item keeps only - a fraction of it. It is also scaled by the item's own engagement: a brand-new - entry nobody has reacted to yet cannot climb on its date alone. - The credit therefore rewards "new AND real", never novelty by itself. - Note the credit has to outweigh real engagement to lift a fresh item over a - well-established one, so it is deliberately large. It is a nudge in the ranking, - not a tie-breaker: engagement still decides between items of similar age. */ + stronger signal than a like on X, so they are weighted up; views add a little. */ const W = { likes: 1, stars: 5, hfLikes: 8, views: 1/500 }; -const RECENCY = { credit: 8000, halfLifeDays: 1, ref: 400 }; -const BASE_DATE = '2026-09-20'; /* the snapshot the metrics were last refreshed against */ -const ageDays = it => Math.max(0, (new Date(BASE_DATE) - new Date(it.date)) / 864e5); -const recency = it => { - const m = it.m || {}; - const raw = (m.likes || 0) * W.likes + (m.stars || 0) * W.stars + (m.hfLikes || 0) * W.hfLikes + (m.views || 0) * W.views; - const reach = Math.min(1, raw / RECENCY.ref); /* saturates fast: any real traction gets the full credit */ - return RECENCY.credit * reach * Math.pow(2, -ageDays(it) / RECENCY.halfLifeDays); -}; -const trending = it => { const m = it.m || {}; return Math.round((m.likes || 0) * W.likes + (m.stars || 0) * W.stars + (m.hfLikes || 0) * W.hfLikes + (m.views || 0) * W.views + recency(it)); }; -/* "new" badge: added within the last day of the snapshot */ -const isNew = it => ageDays(it) <= 1; +const trending = it => { const m = it.m || {}; return Math.round((m.likes || 0) * W.likes + (m.stars || 0) * W.stars + (m.hfLikes || 0) * W.hfLikes + (m.views || 0) * W.views); }; const hasPost = it => !!it.links?.tweet; function sortItems(arr, mode) { diff --git a/og.png b/og.png index 90da2931d41d61f7ac6192f4052cdc1f94195cb8..df2bf90fa8e2929b17652194c70c0bed3701ca89 100644 --- a/og.png +++ b/og.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:190a2e60e3993e1e3b5ca86ae79a415a0a1961b9a2e66c526da411d5981b7944 -size 298233 +oid sha256:2db24233ceed3465f24fc97862fdb618ff8c6e05d56a1fe104ea531ac7b80504 +size 340347 diff --git a/refresh.js b/refresh.js deleted file mode 100644 index 51c465e3bfb7c32488ae6a25619e5151616eda0c..0000000000000000000000000000000000000000 --- a/refresh.js +++ /dev/null @@ -1,34 +0,0 @@ -const fs=require('fs'),{execFileSync}=require('child_process'); -const P='news.html'; let src=fs.readFileSync(P,'utf8'); -const js=src.slice(src.indexOf('')); -const head=js.slice(0,js.indexOf('/* ---- state ---- */')); -const ITEMS=eval(head+';ITEMS'); -const sleep=ms=>new Promise(r=>setTimeout(r,ms)); -const clean=t=>t.split('').filter(c=>{const k=c.charCodeAt(0);return k>=32||c==='\n'||c==='\t'||c==='\r';}).join(''); -async function j(url,opt){const r=await fetch(url,opt);if(!r.ok)throw new Error(r.status+' '+url);return JSON.parse(clean(await r.text()));} -/* GitHub stars go through the gh CLI so the whole run fits in one authenticated - hour (5,000 requests); the unauthenticated API allows only 60. */ -const ghStars=repo=>{try{return parseInt(execFileSync('gh',['api','repos/'+repo,'--jq','.stargazers_count'],{encoding:'utf8',stdio:['ignore','pipe','ignore']}).trim(),10);}catch(e){return null;}}; -const esc=t=>t.replace(/\\/g,'\\\\').replace(/'/g,"\\'"); -const changes=[]; -(async()=>{ - for(const it of ITEMS){ - const m={...(it.m||{})}; const L=it.links||{}; const notes=[]; - if(L.tweet){const id=L.tweet.match(/status\/(\d+)/)[1]; - try{const d=await j('https://api.fxtwitter.com/status/'+id);const t=d.tweet;m.likes=t.likes;m.rts=t.retweets;m.views=t.views;}catch(e){notes.push('tweet:'+e.message);}} - const ghs=[L.gh,L.gh2].filter(Boolean).map(u=>u.match(/github\.com\/([^\/]+\/[^\/#?]+)/)?.[1]).filter(Boolean); - if(ghs.length){let sum=0,ok=true;for(const r of ghs){const s=ghStars(r);if(s===null){ok=false;notes.push('gh:'+r);}else sum+=s;}if(ok)m.stars=sum;} - const hfs=[L.hf,L.hf2].filter(Boolean).filter(u=>!u.includes('/collections/')&&!u.includes('/spaces/')).map(u=>u.match(/huggingface\.co\/([^\/]+\/[^\/#?]+)/)?.[1]).filter(Boolean); - if(hfs.length){let sum=0,ok=true;for(const r of hfs){try{const d=await j('https://huggingface.co/api/models/'+r);sum+=d.likes||0;}catch(e){ok=false;notes.push('hf:'+r);}}if(ok)m.hfLikes=sum;} - const before=JSON.stringify(it.m||{}),after=JSON.stringify(m); - if(before!==after||notes.length)changes.push([it.title.slice(0,44).padEnd(44),before,'->',after,notes.join(' ')]); - const ti=src.indexOf("title: '"+esc(it.title)+"'"); if(ti<0){console.log('TITLE NOT FOUND',it.title);continue;} - const mi=src.indexOf('m: {',ti); const me=src.indexOf('}',mi)+1; - const lit='m: { '+Object.entries(m).map(([k,v])=>k+': '+v).join(', ')+' }'; - src=src.slice(0,mi)+lit+src.slice(me); - await sleep(60); - } - fs.writeFileSync(P,src); - for(const c of changes)console.log(c.join(' ')); - console.log('changed',changes.length,'of',ITEMS.length); -})();