| {"generated_utc":"2026-10-07T16:59:09Z","weights":{"public":0.5,"private":0.5},"bench_weights":{"CV-Bench":1.0,"BLINK":1.0,"RealWorldQA":1.0,"CharXiv":1.0,"InfographicVQA":1.0,"Mind2Web":1.0,"Winoground":1.0,"KIE (CORD+FUNSD)":1.0,"Moderation (Hateful Memes)":1.0,"R-Bench-M":0.5,"MMMU-Pro vision":0.25},"private_sets":["CV-Bench","BLINK","RealWorldQA","CharXiv","InfographicVQA","Mind2Web","Winoground","KIE (CORD+FUNSD)","Moderation (Hateful Memes)"],"entrants":[{"engine":"solomon-v11-bf16-encoding","name":"Solomon v1.1 (27B)","url":"https://huggingface.co/DoccyHealth/Solomon","kind":"head / adapter","technique":"autoregressive","org":{"id":"DoccyHealth","name":"Doccy Health","logo":"data/orgs/DoccyHealth.png"},"pub":69.8,"priv":65.52,"full":67.66,"base":null,"caveat":"Answers up to 8 options: MMMU-Pro rows with more options count as failures.","bench":{"CV-Bench":{"pub":74.91,"private":42.71},"BLINK":{"pub":59.08,"private":59.33},"RealWorldQA":{"pub":75.31,"private":54.0},"CharXiv":{"pub":74.94,"private":63.24},"InfographicVQA":{"pub":95.63,"private":87.33},"Mind2Web":{"pub":79.72,"private":58.67},"Winoground":{"pub":83.75,"private":91.0},"KIE (CORD+FUNSD)":{"pub":91.78,"private":85.38},"Moderation (Hateful Memes)":{"pub":36.36,"private":48.0},"R-Bench-M":{"pub":18.24,"private":null},"MMMU-Pro vision":{"pub":-2.56,"private":null}},"params":27781427952,"active":null,"params_estimated":false,"cal":{"acc":0.7731,"conf":0.8488,"ece":0.0758,"over95":0.0287,"n":12255,"benchmarks":11},"ms_row":561.6,"gpu_h":3.006,"rank_full":2,"rank_pub":3},{"engine":"xor-1.2","name":"Xor 1.2 (35B-A3B)","url":"https://huggingface.co/juspay/xor","kind":"LoRA","technique":"autoregressive","org":{"id":"juspay","name":"Juspay Technologies","logo":"data/orgs/juspay.png"},"pub":70.32,"priv":58.27,"full":64.3,"base":{"name":"Qwen/Qwen3.6-35B-A3B","pub":69.49,"priv":56.53},"caveat":"Environment rebuilt from the reference image (no lane recipe existed); same model code.","bench":{"CV-Bench":{"pub":76.28,"private":44.05},"BLINK":{"pub":57.15,"private":47.33},"RealWorldQA":{"pub":66.43,"private":44.0},"CharXiv":{"pub":71.18,"private":54.75},"InfographicVQA":{"pub":95.63,"private":75.33},"Mind2Web":{"pub":75.61,"private":55.33},"Winoground":{"pub":84.25,"private":93.0},"KIE (CORD+FUNSD)":{"pub":92.72,"private":88.63},"Moderation (Hateful Memes)":{"pub":38.5,"private":22.0},"R-Bench-M":{"pub":35.39,"private":null},"MMMU-Pro vision":{"pub":40.73,"private":null}},"params":35950000000,"active":"A3B","params_estimated":false,"cal":{"acc":0.7638,"conf":0.7659,"ece":0.0084,"over95":0.0078,"n":13693,"benchmarks":11},"ms_row":242.2,"gpu_h":1.297,"rank_full":3,"rank_pub":2},{"engine":"rune-26b-a4b-v3","name":"Surogate Rune 26B-A4B v3","url":"https://huggingface.co/surogate/rune-26b-a4b-GGUF","kind":"full fine-tune","technique":"autoregressive","org":{"id":"surogate","name":"Surogate","logo":"data/orgs/surogate.png"},"pub":65.27,"priv":51.99,"full":58.63,"base":{"name":"google/gemma-4-26B-A4B-it","pub":58.25,"priv":47.34},"caveat":null,"bench":{"CV-Bench":{"pub":70.21,"private":42.04},"BLINK":{"pub":53.12,"private":4.67},"RealWorldQA":{"pub":54.22,"private":28.0},"CharXiv":{"pub":73.43,"private":52.87},"InfographicVQA":{"pub":89.62,"private":94.0},"Mind2Web":{"pub":63.82,"private":52.0},"Winoground":{"pub":78.5,"private":81.0},"KIE (CORD+FUNSD)":{"pub":90.98,"private":79.29},"Moderation (Hateful Memes)":{"pub":38.17,"private":34.0},"R-Bench-M":{"pub":33.22,"private":null},"MMMU-Pro vision":{"pub":30.72,"private":null}},"params":25805936206,"active":"A4B","params_estimated":false,"cal":{"acc":0.7265,"conf":0.8042,"ece":0.0797,"over95":0.0298,"n":13693,"benchmarks":11},"ms_row":143.4,"gpu_h":0.768,"rank_full":8,"rank_pub":6},{"engine":"jpt-9b","name":"JPT-9B","url":"https://huggingface.co/kirp/jpt-9b","kind":"LoRA","technique":"autoregressive","org":null,"pub":65.5,"priv":58.24,"full":61.87,"base":{"name":"Qwen/Qwen3.5-9B-Base","pub":63.47,"priv":54.09},"caveat":null,"bench":{"CV-Bench":{"pub":76.04,"private":45.06},"BLINK":{"pub":52.96,"private":38.0},"RealWorldQA":{"pub":65.6,"private":42.0},"CharXiv":{"pub":69.17,"private":45.33},"InfographicVQA":{"pub":93.44,"private":94.67},"Mind2Web":{"pub":69.3,"private":52.67},"Winoground":{"pub":79.25,"private":90.0},"KIE (CORD+FUNSD)":{"pub":91.92,"private":87.41},"Moderation (Hateful Memes)":{"pub":23.17,"private":29.0},"R-Bench-M":{"pub":18.42,"private":null},"MMMU-Pro vision":{"pub":34.3,"private":null}},"params":9653104368,"active":null,"params_estimated":false,"cal":{"acc":0.7273,"conf":0.8022,"ece":0.0749,"over95":0.0227,"n":13695,"benchmarks":11},"ms_row":300.8,"gpu_h":1.61,"rank_full":5,"rank_pub":5},{"engine":"jpt-4b","name":"JPT-4B","url":"https://huggingface.co/kirp/jpt-4b","kind":"LoRA","technique":"autoregressive","org":null,"pub":64.83,"priv":54.47,"full":59.65,"base":{"name":"Qwen/Qwen3.5-4B-Base","pub":59.56,"priv":46.8},"caveat":null,"bench":{"CV-Bench":{"pub":75.88,"private":43.38},"BLINK":{"pub":53.77,"private":43.33},"RealWorldQA":{"pub":63.1,"private":36.0},"CharXiv":{"pub":69.67,"private":45.33},"InfographicVQA":{"pub":91.26,"private":84.67},"Mind2Web":{"pub":66.84,"private":41.33},"Winoground":{"pub":78.0,"private":88.0},"KIE (CORD+FUNSD)":{"pub":90.32,"private":84.16},"Moderation (Hateful Memes)":{"pub":26.3,"private":24.0},"R-Bench-M":{"pub":19.51,"private":null},"MMMU-Pro vision":{"pub":29.0,"private":null}},"params":4659865088,"active":null,"params_estimated":false,"cal":{"acc":0.7183,"conf":0.7768,"ece":0.0584,"over95":0.0183,"n":13695,"benchmarks":11},"ms_row":177.9,"gpu_h":0.952,"rank_full":7,"rank_pub":7},{"engine":"winnow-12b-q8","name":"Winnow-12B","url":"https://huggingface.co/EldanRing/Winnow-12B","kind":"LoRA","technique":"autoregressive","org":null,"pub":64.38,"priv":55.29,"full":59.84,"base":{"name":"google/gemma-4-12B","pub":18.73,"priv":11.9},"caveat":"Its base answers at chance through the converted projector, so the base comparison is not meaningful.","bench":{"CV-Bench":{"pub":67.78,"private":30.99},"BLINK":{"pub":50.87,"private":22.0},"RealWorldQA":{"pub":53.39,"private":28.0},"CharXiv":{"pub":67.42,"private":60.41},"InfographicVQA":{"pub":87.43,"private":88.67},"Mind2Web":{"pub":76.29,"private":60.0},"Winoground":{"pub":82.0,"private":80.0},"KIE (CORD+FUNSD)":{"pub":90.85,"private":82.54},"Moderation (Hateful Memes)":{"pub":31.57,"private":45.0},"R-Bench-M":{"pub":24.92,"private":null},"MMMU-Pro vision":{"pub":30.65,"private":null}},"params":11959730224,"active":null,"params_estimated":false,"cal":{"acc":0.7196,"conf":0.7928,"ece":0.0788,"over95":0.029,"n":13695,"benchmarks":11},"ms_row":567.5,"gpu_h":3.038,"rank_full":6,"rank_pub":8},{"engine":"neohorse-jev-4b","name":"NeoHorse-Jev-4B","url":"https://huggingface.co/TokenRhythm/NeoHorse-Jev-4B","kind":"head / adapter","technique":"autoregressive","org":{"id":"TokenRhythm","name":"TokenRhythm","logo":"data/orgs/TokenRhythm.png"},"pub":57.21,"priv":44.25,"full":50.73,"base":null,"caveat":"Takes a single image only: multi-image rows (BLINK, Winoground) count as failures.","bench":{"CV-Bench":{"pub":74.1,"private":35.34},"BLINK":{"pub":-2.13,"private":-33.33},"RealWorldQA":{"pub":58.66,"private":22.0},"CharXiv":{"pub":64.16,"private":37.78},"InfographicVQA":{"pub":89.62,"private":76.0},"Mind2Web":{"pub":73.0,"private":54.0},"Winoground":{"pub":76.5,"private":84.0},"KIE (CORD+FUNSD)":{"pub":89.11,"private":82.13},"Moderation (Hateful Memes)":{"pub":19.87,"private":7.0},"R-Bench-M":{"pub":13.55,"private":null},"MMMU-Pro vision":{"pub":23.82,"private":null}},"params":4659865088,"active":null,"params_estimated":false,"cal":{"acc":0.7197,"conf":0.7286,"ece":0.0245,"over95":0.0129,"n":12321,"benchmarks":11},"ms_row":113.6,"gpu_h":0.608,"rank_full":11,"rank_pub":11},{"engine":"jev-omni","name":"Jev-Omni (12B)","url":"https://huggingface.co/akhilaaa3/Jev-Omni","kind":"LoRA + head","technique":"autoregressive","org":null,"pub":51.9,"priv":44.73,"full":48.32,"base":null,"caveat":"Takes a single image only: multi-image rows (BLINK, Winoground) count as failures.","bench":{"CV-Bench":{"pub":69.81,"private":34.0},"BLINK":{"pub":-5.52,"private":-33.33},"RealWorldQA":{"pub":47.56,"private":20.67},"CharXiv":{"pub":62.91,"private":38.73},"InfographicVQA":{"pub":70.49,"private":68.67},"Mind2Web":{"pub":46.15,"private":35.33},"Winoground":{"pub":75.5,"private":80.0},"KIE (CORD+FUNSD)":{"pub":90.45,"private":71.17},"Moderation (Hateful Memes)":{"pub":35.53,"private":54.0},"R-Bench-M":{"pub":5.79,"private":null},"MMMU-Pro vision":{"pub":19.05,"private":null}},"params":11959730224,"active":null,"params_estimated":false,"cal":{"acc":0.6722,"conf":0.8001,"ece":0.1279,"over95":0.0334,"n":12321,"benchmarks":11},"ms_row":102.9,"gpu_h":0.551,"rank_full":13,"rank_pub":13},{"engine":"winnow-e4b-q8","name":"Winnow-E4B","url":"https://huggingface.co/EldanRing/Winnow-E4B","kind":"LoRA","technique":"autoregressive","org":null,"pub":48.46,"priv":38.37,"full":43.41,"base":{"name":"google/gemma-4-E4B","pub":45.56,"priv":39.79},"caveat":null,"bench":{"CV-Bench":{"pub":49.08,"private":17.25},"BLINK":{"pub":34.92,"private":10.0},"RealWorldQA":{"pub":35.35,"private":19.33},"CharXiv":{"pub":43.36,"private":28.36},"InfographicVQA":{"pub":83.61,"private":61.33},"Mind2Web":{"pub":53.13,"private":46.0},"Winoground":{"pub":65.5,"private":68.0},"KIE (CORD+FUNSD)":{"pub":80.43,"private":76.04},"Moderation (Hateful Memes)":{"pub":15.91,"private":19.0},"R-Bench-M":{"pub":15.36,"private":null},"MMMU-Pro vision":{"pub":14.01,"private":null}},"params":7996156490,"active":null,"params_estimated":false,"cal":{"acc":0.6144,"conf":0.6313,"ece":0.0478,"over95":0.0187,"n":13695,"benchmarks":11},"ms_row":152.6,"gpu_h":0.817,"rank_full":14,"rank_pub":14},{"engine":"onejev-0.8b","name":"OneJev 0.8B","url":"https://huggingface.co/OmniJev/OneJev-0.8B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"OmniJev","name":"OmniJev","logo":"data/orgs/OmniJev.png"},"pub":46.84,"priv":36.39,"full":41.61,"base":{"name":"Qwen/Qwen3.5-0.8B","pub":35.08,"priv":29.74},"caveat":null,"bench":{"CV-Bench":{"pub":55.8,"private":26.63},"BLINK":{"pub":39.43,"private":15.33},"RealWorldQA":{"pub":40.07,"private":24.67},"CharXiv":{"pub":52.13,"private":26.47},"InfographicVQA":{"pub":78.69,"private":54.67},"Mind2Web":{"pub":45.05,"private":29.33},"Winoground":{"pub":53.5,"private":69.0},"KIE (CORD+FUNSD)":{"pub":75.76,"private":72.39},"Moderation (Hateful Memes)":{"pub":12.61,"private":9.0},"R-Bench-M":{"pub":3.26,"private":null},"MMMU-Pro vision":{"pub":8.11,"private":null}},"params":870000000,"active":null,"params_estimated":false,"cal":{"acc":0.5954,"conf":0.611,"ece":0.0166,"over95":0.0042,"n":13695,"benchmarks":11},"ms_row":90.3,"gpu_h":0.483,"rank_full":16,"rank_pub":15},{"engine":"jpt-0.8b","name":"JPT-0.8B","url":"https://huggingface.co/kirp/jpt-0.8b","kind":"LoRA","technique":"autoregressive","org":null,"pub":44.72,"priv":40.98,"full":42.85,"base":{"name":"Qwen/Qwen3.5-0.8B-Base","pub":34.51,"priv":30.92},"caveat":null,"bench":{"CV-Bench":{"pub":51.35,"private":25.63},"BLINK":{"pub":33.31,"private":38.67},"RealWorldQA":{"pub":41.46,"private":26.67},"CharXiv":{"pub":52.13,"private":36.84},"InfographicVQA":{"pub":76.5,"private":54.67},"Mind2Web":{"pub":37.79,"private":33.33},"Winoground":{"pub":47.75,"private":62.0},"KIE (CORD+FUNSD)":{"pub":73.95,"private":74.01},"Moderation (Hateful Memes)":{"pub":12.78,"private":17.0},"R-Bench-M":{"pub":13.73,"private":null},"MMMU-Pro vision":{"pub":8.64,"private":null}},"params":873438784,"active":null,"params_estimated":false,"cal":{"acc":0.59,"conf":0.6536,"ece":0.0636,"over95":0.0067,"n":13695,"benchmarks":11},"ms_row":115.7,"gpu_h":0.619,"rank_full":15,"rank_pub":16},{"engine":"djev","name":"djev (26B-A4B)","url":"https://github.com/Davipar/djev-dev","kind":"inference technique","technique":"diffusion","org":null,"pub":44.51,"priv":29.21,"full":36.86,"base":null,"caveat":"Ran with its published canvas 128 and 32K context, since the long-context engine takes no images.","bench":{"CV-Bench":{"pub":50.38,"private":19.26},"BLINK":{"pub":27.18,"private":6.0},"RealWorldQA":{"pub":38.4,"private":18.67},"CharXiv":{"pub":35.34,"private":10.45},"InfographicVQA":{"pub":62.3,"private":60.67},"Mind2Web":{"pub":42.31,"private":32.67},"Winoground":{"pub":35.25,"private":38.0},"KIE (CORD+FUNSD)":{"pub":84.71,"private":56.14},"Moderation (Hateful Memes)":{"pub":38.5,"private":21.0},"R-Bench-M":{"pub":25.46,"private":null},"MMMU-Pro vision":{"pub":27.34,"private":null}},"params":25823778864,"active":"A4B","params_estimated":false,"cal":{"acc":0.6088,"conf":0.7669,"ece":0.1581,"over95":0.0548,"n":13695,"benchmarks":11},"ms_row":77.0,"gpu_h":0.412,"rank_full":17,"rank_pub":17},{"engine":"razorback-one-read-3e296f08","name":"razorback16 diffgemma (26B-A4B)","url":"https://github.com/vllm-project/vllm/pull/57250","kind":"inference technique","technique":"diffusion","org":null,"pub":43.91,"priv":28.8,"full":36.36,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":51.03,"private":20.6},"BLINK":{"pub":36.85,"private":-2.67},"RealWorldQA":{"pub":35.63,"private":21.33},"CharXiv":{"pub":25.81,"private":8.56},"InfographicVQA":{"pub":62.84,"private":63.33},"Mind2Web":{"pub":38.47,"private":32.67},"Winoground":{"pub":35.0,"private":47.0},"KIE (CORD+FUNSD)":{"pub":84.57,"private":53.71},"Moderation (Hateful Memes)":{"pub":41.14,"private":12.0},"R-Bench-M":{"pub":21.31,"private":null},"MMMU-Pro vision":{"pub":24.49,"private":null}},"params":25823778864,"active":"A4B","params_estimated":false,"cal":{"acc":0.6006,"conf":0.7814,"ece":0.1808,"over95":0.0598,"n":13695,"benchmarks":11},"ms_row":39.0,"gpu_h":0.209,"rank_full":19,"rank_pub":18},{"engine":"vllm-pr57250","name":"mmastrac diffgemma (26B-A4B)","url":"https://github.com/vllm-project/vllm/pull/57250","kind":"inference technique","technique":"diffusion","org":null,"pub":36.15,"priv":24.41,"full":30.28,"base":null,"caveat":null,"bench":{"CV-Bench":{"pub":47.38,"private":18.93},"BLINK":{"pub":25.09,"private":10.67},"RealWorldQA":{"pub":30.08,"private":17.33},"CharXiv":{"pub":25.31,"private":12.33},"InfographicVQA":{"pub":53.55,"private":50.0},"Mind2Web":{"pub":24.77,"private":15.33},"Winoground":{"pub":25.75,"private":35.0},"KIE (CORD+FUNSD)":{"pub":76.42,"private":39.09},"Moderation (Hateful Memes)":{"pub":28.94,"private":21.0},"R-Bench-M":{"pub":24.2,"private":null},"MMMU-Pro vision":{"pub":12.09,"private":null}},"params":25823778864,"active":"A4B","params_estimated":false,"cal":{"acc":0.5521,"conf":0.7132,"ece":0.1611,"over95":0.043,"n":13695,"benchmarks":11},"ms_row":84.8,"gpu_h":0.454,"rank_full":20,"rank_pub":20},{"engine":"intern-decision-4b","name":"Intern-Decision-4B","url":"https://huggingface.co/internlm/Intern-Decision-4B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"},"pub":61.83,"priv":51.95,"full":56.89,"base":{"name":"Qwen/Qwen3.5-4B-Base","pub":56.89,"priv":41.26},"caveat":"The vision patch embed was swapped for the identical linear map to run at normal speed; outputs agree to bf16 precision.","bench":{"CV-Bench":{"pub":73.85,"private":36.68},"BLINK":{"pub":53.6,"private":36.0},"RealWorldQA":{"pub":60.32,"private":34.67},"CharXiv":{"pub":57.14,"private":42.5},"InfographicVQA":{"pub":85.25,"private":67.33},"Mind2Web":{"pub":65.88,"private":43.33},"Winoground":{"pub":75.25,"private":87.0},"KIE (CORD+FUNSD)":{"pub":87.98,"private":78.07},"Moderation (Hateful Memes)":{"pub":26.63,"private":42.0},"R-Bench-M":{"pub":19.15,"private":null},"MMMU-Pro vision":{"pub":29.33,"private":null}},"params":4659865088,"active":null,"params_estimated":false,"cal":{"acc":0.6994,"conf":0.6899,"ece":0.0134,"over95":0.0021,"n":13695,"benchmarks":11},"ms_row":142.4,"gpu_h":0.762,"rank_full":9,"rank_pub":10},{"engine":"intern-decision-2b","name":"Intern-Decision-2B","url":"https://huggingface.co/internlm/Intern-Decision-2B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"},"pub":53.29,"priv":43.42,"full":48.36,"base":{"name":"Qwen/Qwen3.5-2B-Base","pub":43.77,"priv":34.45},"caveat":"The vision patch embed was swapped for the identical linear map to run at normal speed; outputs agree to bf16 precision.","bench":{"CV-Bench":{"pub":65.35,"private":36.35},"BLINK":{"pub":43.13,"private":26.0},"RealWorldQA":{"pub":49.78,"private":42.67},"CharXiv":{"pub":58.15,"private":25.53},"InfographicVQA":{"pub":81.42,"private":58.0},"Mind2Web":{"pub":49.98,"private":42.0},"Winoground":{"pub":63.75,"private":77.0},"KIE (CORD+FUNSD)":{"pub":84.57,"private":75.23},"Moderation (Hateful Memes)":{"pub":13.93,"private":8.0},"R-Bench-M":{"pub":10.48,"private":null},"MMMU-Pro vision":{"pub":17.19,"private":null}},"params":2274069824,"active":null,"params_estimated":false,"cal":{"acc":0.641,"conf":0.6175,"ece":0.0339,"over95":0.0015,"n":13695,"benchmarks":11},"ms_row":106.8,"gpu_h":0.572,"rank_full":12,"rank_pub":12},{"engine":"intern-decision-0.8b","name":"Intern-Decision-0.8B","url":"https://huggingface.co/internlm/Intern-Decision-0.8B","kind":"full fine-tune","technique":"autoregressive","org":{"id":"internlm","name":"Intern Large Models","logo":"data/orgs/internlm.png"},"pub":38.78,"priv":34.95,"full":36.86,"base":{"name":"Qwen/Qwen3.5-0.8B-Base","pub":29.58,"priv":26.81},"caveat":"The vision patch embed was swapped for the identical linear map to run at normal speed; outputs agree to bf16 precision.","bench":{"CV-Bench":{"pub":51.92,"private":20.6},"BLINK":{"pub":25.41,"private":20.0},"RealWorldQA":{"pub":37.57,"private":28.67},"CharXiv":{"pub":48.37,"private":21.76},"InfographicVQA":{"pub":70.49,"private":59.33},"Mind2Web":{"pub":21.07,"private":19.33},"Winoground":{"pub":43.25,"private":71.0},"KIE (CORD+FUNSD)":{"pub":62.6,"private":61.83},"Moderation (Hateful Memes)":{"pub":12.45,"private":12.0},"R-Bench-M":{"pub":5.79,"private":null},"MMMU-Pro vision":{"pub":8.18,"private":null}},"params":873438784,"active":null,"params_estimated":false,"cal":{"acc":0.5517,"conf":0.5003,"ece":0.0595,"over95":0.0002,"n":13695,"benchmarks":11},"ms_row":1618.4,"gpu_h":8.663,"rank_full":18,"rank_pub":19},{"engine":"pplx-decider-v1.1-27b","name":"Perplexity Decider v1.1 (27B)","url":"https://huggingface.co/perplexity-ai/pplx-decider-v1.1-27b","kind":"full fine-tune","technique":"autoregressive","org":{"id":"perplexity-ai","name":"Perplexity","logo":"data/orgs/perplexity-ai.png"},"pub":67.0,"priv":59.4,"full":63.2,"base":{"name":"Qwen/Qwen3.8-27B","pub":61.3,"priv":51.68},"caveat":"Ran through the checkpoint's own decision model, which accepts images.","bench":{"CV-Bench":{"pub":74.74,"private":41.71},"BLINK":{"pub":54.73,"private":62.0},"RealWorldQA":{"pub":57.83,"private":26.67},"CharXiv":{"pub":77.19,"private":56.64},"InfographicVQA":{"pub":79.78,"private":93.33},"Mind2Web":{"pub":68.76,"private":50.0},"Winoground":{"pub":84.5,"private":84.0},"KIE (CORD+FUNSD)":{"pub":91.32,"private":75.23},"Moderation (Hateful Memes)":{"pub":42.79,"private":45.0},"R-Bench-M":{"pub":29.25,"private":null},"MMMU-Pro vision":{"pub":28.0,"private":null}},"params":27800000000,"active":null,"params_estimated":false,"cal":{"acc":0.7345,"conf":0.784,"ece":0.0496,"over95":0.0157,"n":13695,"benchmarks":11},"ms_row":146.8,"gpu_h":0.786,"rank_full":4,"rank_pub":4},{"engine":"rsi-jev-v5.0-vl-3b","name":"RSI-Jev v5.0 VL 3B","url":"https://huggingface.co/shgao/rsi-jev-v5.0-vl-3b","kind":"full fine-tune","technique":"autoregressive","org":null,"pub":61.85,"priv":51.22,"full":56.54,"base":{"name":"Qwen/Qwen3.5-4B-Base","pub":29.06,"priv":23.1},"caveat":"Its base was read through the plain vision-language path, where a pretrained base answers letters poorly, so the gain over the base is overstated.","bench":{"CV-Bench":{"pub":72.96,"private":35.68},"BLINK":{"pub":47.64,"private":46.67},"RealWorldQA":{"pub":55.33,"private":23.33},"CharXiv":{"pub":66.67,"private":33.07},"InfographicVQA":{"pub":87.98,"private":66.67},"Mind2Web":{"pub":56.56,"private":40.67},"Winoground":{"pub":75.75,"private":88.0},"KIE (CORD+FUNSD)":{"pub":85.91,"private":80.91},"Moderation (Hateful Memes)":{"pub":42.46,"private":46.0},"R-Bench-M":{"pub":11.93,"private":null},"MMMU-Pro vision":{"pub":23.36,"private":null}},"params":4660000000,"active":null,"params_estimated":false,"cal":{"acc":0.6914,"conf":0.698,"ece":0.0252,"over95":0.0108,"n":13695,"benchmarks":11},"ms_row":64.9,"gpu_h":0.347,"rank_full":10,"rank_pub":9},{"engine":"jev-27b-vl","name":"JEV-27B-VL","url":"https://huggingface.co/autotrust/JEV-27B-VL","kind":"LoRA","technique":"autoregressive","org":{"id":"autotrust","name":"AutoTrust AI Lab","logo":"data/orgs/autotrust.png"},"pub":72.78,"priv":66.86,"full":69.82,"base":{"name":"Qwen/Qwen3.8-27B","pub":61.3,"priv":51.68},"caveat":"Ran through the author's own server (vLLM + POST /v1/decide, System 1 only) with the context cap lifted to 256K; its base was read through a different (head-free autojev) path.","bench":{"CV-Bench":{"pub":76.61,"private":42.04},"BLINK":{"pub":62.95,"private":58.67},"RealWorldQA":{"pub":71.98,"private":48.0},"CharXiv":{"pub":79.7,"private":66.06},"InfographicVQA":{"pub":98.36,"private":98.67},"Mind2Web":{"pub":78.62,"private":59.33},"Winoground":{"pub":83.75,"private":89.0},"KIE (CORD+FUNSD)":{"pub":91.99,"private":87.01},"Moderation (Hateful Memes)":{"pub":40.31,"private":53.0},"R-Bench-M":{"pub":30.7,"private":null},"MMMU-Pro vision":{"pub":40.07,"private":null}},"params":27800000000,"active":null,"params_estimated":false,"cal":{"acc":0.7775,"conf":0.7921,"ece":0.0187,"over95":0.0105,"n":13695,"benchmarks":11},"ms_row":263.9,"gpu_h":1.412,"rank_full":1,"rank_pub":1}],"refs":[{"engine":"ref-qwen35-397b","name":"Qwen3.5-397B (VLM)","kind":"stock · API","url":"https://huggingface.co/Qwen/Qwen3.5-397B-A17B","note":"Stock model via API, read from its generated letter rather than letter probabilities.","label":null,"params":397000000000.0,"active":"A17B","params_estimated":null,"pub":68.52,"priv":58.37,"full":63.44,"bench":{"CV-Bench":{"pub":78.31,"private":46.73},"BLINK":{"pub":50.54,"private":4.67},"RealWorldQA":{"pub":67.82,"private":41.33},"CharXiv":{"pub":62.16,"private":51.92},"InfographicVQA":{"pub":95.08,"private":99.33},"Mind2Web":{"pub":76.57,"private":57.33},"Winoground":{"pub":82.25,"private":87.0},"KIE (CORD+FUNSD)":{"pub":94.06,"private":84.97},"Moderation (Hateful Memes)":{"pub":49.22,"private":52.0},"R-Bench-M":{"pub":0.38,"private":null},"MMMU-Pro vision":{"pub":47.43,"private":null}},"ref":true,"cal":null,"ms_row":null},{"engine":"ref-qwen3vl-8b","name":"Qwen3-VL-8B (VLM)","kind":"stock","url":"https://huggingface.co/Qwen/Qwen3-VL-8B-Instruct","note":"Stock vision-language model, no fine-tune.","label":null,"params":8800000000.0,"active":null,"params_estimated":null,"pub":62.47,"priv":44.84,"full":53.66,"bench":{"CV-Bench":{"pub":72.96,"private":34.0},"BLINK":{"pub":51.03,"private":20.0},"RealWorldQA":{"pub":54.77,"private":22.0},"CharXiv":{"pub":58.9,"private":16.1},"InfographicVQA":{"pub":85.25,"private":70.67},"Mind2Web":{"pub":71.22,"private":50.67},"Winoground":{"pub":73.75,"private":80.0},"KIE (CORD+FUNSD)":{"pub":89.05,"private":82.13},"Moderation (Hateful Memes)":{"pub":34.21,"private":28.0},"R-Bench-M":{"pub":21.85,"private":null},"MMMU-Pro vision":{"pub":28.13,"private":null}},"ref":true,"cal":{"acc":0.7032,"conf":0.8984,"ece":0.1952,"over95":0.1085,"n":13695,"benchmarks":11},"ms_row":119.4},{"engine":"ref-siglip2","name":"SigLIP 2 (contrastive)","kind":"contrastive floor","url":"https://huggingface.co/google/siglip2-so400m-patch14-384","note":"Image-text similarity only, no language model: the floor.","label":null,"params":1100000000.0,"active":null,"params_estimated":true,"pub":6.16,"priv":6.17,"full":6.17,"bench":{"CV-Bench":{"pub":-5.39,"private":4.86},"BLINK":{"pub":-0.85,"private":-10.0},"RealWorldQA":{"pub":5.39,"private":-6.0},"CharXiv":{"pub":4.01,"private":3.85},"InfographicVQA":{"pub":7.65,"private":-3.33},"Mind2Web":{"pub":8.46,"private":25.33},"Winoground":{"pub":33.5,"private":19.0},"KIE (CORD+FUNSD)":{"pub":-2.25,"private":0.51},"Moderation (Hateful Memes)":{"pub":-2.23,"private":2.0},"R-Bench-M":{"pub":0.74,"private":null},"MMMU-Pro vision":{"pub":2.68,"private":null}},"ref":true,"cal":null,"ms_row":null}],"excluded":[{"name":"reflex-27b-v2","why":"its official path takes no images"},{"name":"gevva-e4b","why":"its official path takes no images"},{"name":"gevva-e2b","why":"its official path takes no images"},{"name":"matilda-jev","why":"its base is undisclosed and its results return 404"}],"bench_info":{"CV-Bench":{"what":"Basic spatial vision: count objects, say which is left or right, and judge which is closer to the camera.","example":"How many chairs are in the photo, or is the lamp closer to the camera than the sofa?","why":"Many decisions start with getting simple facts about a scene right.","tech":"Counting, 2D relation and distance items; 2 to 6 options."},"BLINK":{"what":"Perception puzzles that people solve at a glance but models often miss, sometimes across two or more images.","example":"Given two photos of the same room, how did the camera move between them?","why":"Tests whether a model really looks at the image instead of guessing from the wording.","tech":"Seven subtasks of the validation split; 2 to 4 options over 1 to 4 images."},"RealWorldQA":{"what":"Questions about everyday real-world photos, many taken from a car or a phone.","example":"From this dashboard photo, which lane should the car be in to turn right?","why":"Real scenes are where a decision model meets the physical world.","tech":"Multiple-choice and yes/no items parsed from the prompt."},"CharXiv":{"what":"Reasoning over charts from scientific papers: read values, compare trends and find the right series.","example":"In this plot, which method has the highest score at the largest dataset size?","why":"Decisions often hinge on reading a chart correctly.","tech":"Reasoning questions converted to 4-way multiple choice with distractors from the same figure."},"InfographicVQA":{"what":"Answering questions about infographics that mix text, numbers, icons and layout.","example":"According to this infographic, which age group spends the most time online?","why":"Combines reading and layout understanding on dense documents.","tech":"Short answers converted to multiple choice."},"Mind2Web":{"what":"Web navigation: given a task and a screenshot of a real website, pick the element to click or type into next.","example":"To book a one-way flight to Denver, which of these four marked buttons comes next?","why":"Choosing the next action on a screen is a decision in the most literal sense.","tech":"Four marked boxes per screenshot: the right element plus three real distractors from the page."},"Winoground":{"what":"Matching captions to images when the words are the same but their order changes the meaning.","example":"Which caption fits: 'a dog bites a person' or 'a person bites a dog'?","why":"Tests whether a model understands who does what to whom.","tech":"Caption choice for each of two images; pairs of images and captions."},"KIE (CORD+FUNSD)":{"what":"Reading receipts and forms: pull the right field (total, date, item) out of a scanned document.","example":"On this receipt, what is the total after tax?","why":"Document extraction feeds many routing and approval decisions.","tech":"Receipt fields and form-field presence questions, multiple choice."},"Moderation (Hateful Memes)":{"what":"Deciding whether a meme is hateful, where the meaning often needs both the image and its text.","example":"Is this meme, its picture and caption together, hateful or not?","why":"A content-moderation call is a binary decision with real consequences.","tech":"Binary hateful / not hateful; a caption-only control is reported separately."},"R-Bench-M":{"what":"Expert-level multi-discipline questions with an image, from science, engineering and medicine.","example":"Given this circuit diagram, which option gives the current through the resistor?","why":"Shows headroom: even strong models score low here.","tech":"Six options A to F, one image; low weight in the index."},"MMMU-Pro vision":{"what":"College-level exam questions where the whole question and its options are shown inside the image.","example":"A photographed exam page asks which of ten options explains the graph.","why":"The model has to read and reason from the image alone.","tech":"Ten options inside the image; a quarter weight in the index."}},"bench_rows":{"CV-Bench":{"public":2038},"BLINK":{"public":961},"RealWorldQA":{"public":589},"CharXiv":{"public":532},"InfographicVQA":{"public":244},"Mind2Web":{"public":973},"Winoground":{"public":800},"KIE (CORD+FUNSD)":{"public":2465},"Moderation (Hateful Memes)":{"public":1213},"R-Bench-M":{"public":665},"MMMU-Pro vision":{"public":1730}},"stock_drop":{"name":"Qwen3-VL-8B (VLM)","drop":17.6}} |