import json
from collections import defaultdict
from pathlib import Path
import gradio as gr
import pandas as pd
from ui import (
render_about,
render_benchmarks,
render_footer,
render_header,
render_home,
)
custom_css = """
.logo {
width: 300px;
height: auto;
max-width: 100%;
margin: 0 auto;
object-fit: contain;
padding-bottom: 0;
}
.text {
font-size: 16px !important;
}
.tabs button,
.tab-nav button,
.subtabs button {
font-size: 20px;
color: #581c87 !important;
}
.tabs button.selected,
.tabs button[aria-selected="true"],
.tab-nav button.selected,
.tab-nav button[aria-selected="true"],
.subtabs button.selected,
.subtabs button[aria-selected="true"] {
color: #ffffff !important;
background: #7c3aed !important;
border-color: #6d28d9 !important;
}
.dark .tabs button,
.dark .tab-nav button,
.dark .subtabs button {
color: #e9d5ff !important;
}
.dark .tabs button.selected,
.dark .tabs button[aria-selected="true"],
.dark .tab-nav button.selected,
.dark .tab-nav button[aria-selected="true"],
.dark .subtabs button.selected,
.dark .subtabs button[aria-selected="true"] {
color: #ffffff !important;
background: #6d28d9 !important;
}
.benchmark-catalogue-row {
width: 100%;
gap: 16px;
}
.benchmark-card {
width: 100%;
max-width: none;
min-height: 220px;
border: 1px solid rgba(124, 58, 237, 0.45);
border-left: 4px solid #7c3aed;
border-radius: 12px;
background: rgba(139, 92, 246, 0.14);
padding: 16px 18px;
box-shadow: 0 4px 12px rgba(76, 29, 149, 0.12);
}
.benchmark-card:hover {
border-color: #5b21b6;
box-shadow: 0 6px 18px rgba(76, 29, 149, 0.22);
}
.community-footer {
margin-top: 24px;
padding: 18px 20px;
border: 1px solid #c4b5fd;
border-radius: 12px;
background: rgba(139, 92, 246, 0.16);
}
.community-footer h3 {
margin: 0 0 10px;
color: #581c87;
font-size: 1.05rem;
}
.community-footer-links {
display: flex;
flex-wrap: wrap;
gap: 10px 14px;
align-items: center;
}
.dark .community-footer {
border-color: #6d28d9;
background: rgba(76, 29, 149, 0.35);
}
.dark .community-footer h3 {
color: #f5f3ff;
}
.home-callouts {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 12px;
margin: 8px 0 20px;
}
.home-callouts > div {
padding: 14px 16px;
border: 1px solid #a78bfa;
border-radius: 10px;
background: #f5f3ff;
}
.home-callouts span {
color: #5b21b6;
font-size: 11px;
font-weight: 700;
letter-spacing: 0.05em;
}
.home-callouts strong {
display: block;
margin-top: 6px;
color: #1e1b4b;
font-size: 18px;
word-break: break-word;
}
.home-callouts em {
display: block;
margin-top: 4px;
color: #4c1d95;
font-style: normal;
font-size: 13px;
}
.home-benchmark-card {
height: 100%;
padding: 18px 18px 16px;
border: 1px solid #a78bfa;
border-left: 4px solid #7c3aed;
border-radius: 12px;
background: #f5f3ff;
box-shadow: 0 4px 12px rgba(76, 29, 149, 0.12);
}
.home-benchmark-title {
color: #1e1b4b;
font-size: 1.35rem;
font-weight: 700;
}
.home-benchmark-blurb {
margin: 8px 0 14px;
color: #312e81;
font-size: 0.95rem;
line-height: 1.45;
}
.home-top-label {
color: #5b21b6;
font-size: 11px;
font-weight: 700;
letter-spacing: 0.05em;
text-transform: uppercase;
}
.home-top-list {
list-style: none;
margin: 8px 0 0;
padding: 0;
}
.home-top-list li {
display: grid;
grid-template-columns: 28px 1fr auto;
gap: 10px;
align-items: center;
padding: 8px 0;
border-bottom: 1px solid #ddd6fe;
color: #1e1b4b;
}
.home-rank {
width: 28px;
height: 28px;
border-radius: 999px;
background: #ede9fe;
color: #5b21b6;
font-weight: 700;
font-size: 13px;
display: inline-flex;
align-items: center;
justify-content: center;
}
.home-model {
font-weight: 600;
word-break: break-word;
color: #1e1b4b;
}
.home-score {
font-variant-numeric: tabular-nums;
color: #4c1d95;
font-weight: 600;
}
.home-empty {
color: #4c1d95;
}
.dark .home-callouts > div,
.dark .home-benchmark-card {
border-color: #7c3aed;
background: #1e1b4b;
}
.dark .home-callouts span,
.dark .home-top-label,
.dark .home-rank {
color: #c4b5fd;
}
.dark .home-rank {
background: #312e81;
}
.dark .home-callouts strong,
.dark .home-benchmark-title,
.dark .home-model,
.dark .home-top-list li {
color: #faf5ff;
}
.dark .home-callouts em,
.dark .home-benchmark-blurb,
.dark .home-score,
.dark .home-empty {
color: #ddd6fe;
}
.dark .home-top-list li {
border-bottom-color: #4c1d95;
}
@media (max-width: 700px) {
.home-callouts {
grid-template-columns: 1fr;
}
}
.pareto-plot {
width: 100%;
}
.pareto-plot .plotly {
width: 100% !important;
}
.pareto-plot .js-plotly-plot,
.pareto-plot .plot-container {
width: 100% !important;
}
/* Hide the Plotly toolbar so it doesn't cover points in the top-right. */
.pareto-plot .modebar {
display: none !important;
}
.leaderboard-summary {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 12px;
margin: 8px 0 16px;
}
.leaderboard-summary > div {
padding: 14px 16px;
border: 1px solid #c4b5fd;
border-radius: 10px;
background: rgba(139, 92, 246, 0.14);
}
.leaderboard-summary span,
.results-heading {
color: #7e22ce;
font-size: 11px;
font-weight: 700;
letter-spacing: 0.05em;
}
.leaderboard-summary strong {
display: block;
margin-top: 4px;
color: #581c87;
font-size: 22px;
}
.leaderboard-controls {
align-items: end;
margin-bottom: 8px;
}
.ranking-table-host {
width: 100%;
}
.ranking-table {
margin-top: 4px;
width: 100%;
}
.results-heading {
display: flex;
justify-content: space-between;
margin: 16px 0 8px;
text-transform: uppercase;
}
.ranking-table-scroll {
overflow-x: auto;
border: 1px solid #c4b5fd;
border-radius: 10px;
background: rgba(139, 92, 246, 0.12);
}
.ranking-table table {
width: 100%;
min-width: 800px;
border-collapse: collapse;
color: #581c87;
font-size: 14px;
}
.ranking-table th {
padding: 11px 14px;
border-bottom: 1px solid #c4b5fd;
color: #7e22ce;
font-size: 11px;
letter-spacing: 0.04em;
text-align: left;
text-transform: uppercase;
}
.ranking-table th.sortable-col {
cursor: pointer;
user-select: none;
white-space: nowrap;
}
.ranking-table th.sortable-col:hover {
color: #4c1d95;
}
.ranking-table th.sortable-col.sorted-asc::after,
.ranking-table th.sortable-col.sorted-desc::after {
margin-left: 0.35em;
font-size: 0.85em;
opacity: 0.85;
}
.ranking-table th.sortable-col.sorted-asc::after {
content: "▲";
}
.ranking-table th.sortable-col.sorted-desc::after {
content: "▼";
}
.ranking-table td {
padding: 13px 14px;
border-bottom: 1px solid #e9d5ff;
}
.ranking-table tbody tr:last-child td {
border-bottom: 0;
}
.ranking-table tbody tr:hover {
background: #6d28d9;
}
.ranking-table tbody tr:hover td,
.ranking-table tbody tr:hover .rank,
.ranking-table tbody tr:hover .metric-score,
.ranking-table tbody tr:hover .model-cell strong,
.ranking-table tbody tr:hover .model-cell span {
color: #ffffff;
}
.ranking-table .rank {
color: #7c3aed;
font-weight: 700;
}
.ranking-table .model-cell strong,
.ranking-table .model-cell span {
display: block;
}
.ranking-table .model-cell span {
margin-top: 3px;
color: #9333ea;
font-size: 12px;
}
.ranking-table .metric-score {
color: #4c1d95;
font-variant-numeric: tabular-nums;
font-weight: 700;
}
.ranking-table .empty-state {
color: #9333ea;
padding: 32px;
text-align: center;
}
@media (prefers-color-scheme: dark) {
.leaderboard-summary > div,
.ranking-table-scroll {
border-color: rgba(221, 214, 254, 0.45);
}
.leaderboard-summary span,
.results-heading,
.ranking-table th,
.ranking-table .model-cell span,
.ranking-table .empty-state {
color: #ddd6fe;
}
.leaderboard-summary strong,
.ranking-table table,
.ranking-table .rank,
.ranking-table .metric-score {
color: #f5f3ff;
}
.ranking-table td {
border-bottom-color: rgba(221, 214, 254, 0.2);
}
}
.dark .leaderboard-summary > div,
.dark .ranking-table-scroll {
border-color: rgba(221, 214, 254, 0.45);
}
.dark .leaderboard-summary span,
.dark .results-heading,
.dark .ranking-table th,
.dark .ranking-table .model-cell span,
.dark .ranking-table .empty-state {
color: #ddd6fe;
}
.dark .leaderboard-summary strong,
.dark .ranking-table table,
.dark .ranking-table .rank,
.dark .ranking-table .metric-score {
color: #f5f3ff;
}
.compare-samples-help {
color: #6b21a8;
margin-bottom: 8px;
}
.compare-prompt-block {
margin: 0 0 22px;
padding: 14px 16px;
border: 1px solid #c4b5fd;
border-radius: 12px;
background: rgba(139, 92, 246, 0.10);
}
.compare-prompt-meta {
display: flex;
justify-content: space-between;
gap: 12px;
margin-bottom: 8px;
color: #7e22ce;
font-size: 12px;
font-weight: 700;
letter-spacing: 0.04em;
text-transform: uppercase;
}
.compare-prompt-text {
margin: 0 0 14px;
color: #3b0764;
font-size: 15px;
line-height: 1.45;
}
.compare-row {
display: grid;
gap: 12px;
}
.compare-cell {
min-width: 0;
}
.compare-model-label {
margin-bottom: 6px;
color: #5b21b6;
font-size: 13px;
font-weight: 700;
word-break: break-word;
}
.compare-cell img {
display: block;
width: 100%;
aspect-ratio: 1 / 1;
object-fit: cover;
border-radius: 10px;
border: 1px solid #ddd6fe;
background: #f5f3ff;
}
.compare-empty {
padding: 18px;
border: 1px dashed #c4b5fd;
border-radius: 10px;
color: #6b21a8;
text-align: center;
}
.dark .compare-samples-help,
.dark .compare-prompt-meta,
.dark .compare-model-label,
.dark .compare-empty {
color: #e9d5ff;
}
.dark .compare-prompt-block {
border-color: #6d28d9;
background: rgba(76, 29, 149, 0.35);
}
.dark .compare-prompt-text {
color: #f5f3ff;
}
.dark .compare-cell img {
border-color: #6d28d9;
background: #2e1065;
}
.dark .ranking-table td {
border-bottom-color: rgba(221, 214, 254, 0.2);
}
@media (max-width: 700px) {
.leaderboard-summary {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
}
h1, h2 {
margin: 0;
padding-top: 0;
}
"""
theme = gr.Theme.from_hub("Werli/Purple-Crimson-Gradio-Theme").set(
body_background_fill="#c4b5fd",
body_background_fill_dark="#120B1B",
body_text_color="#3b0764",
body_text_color_dark="#ffffff",
block_background_fill="#ede9fe",
block_background_fill_dark="#2e1065",
)
space_root = Path(__file__).resolve().parent
data_dir = space_root / "data"
def _resolve_data_path(*candidates):
"""Prefer Space-local data/, fall back to monorepo parent for local edits."""
for path in candidates:
if path.exists():
return path
return candidates[0]
oneig_path = _resolve_data_path(
data_dir / "oneig_model_price_and_median_generation_time.csv",
space_root.parent / "oneig_model_price_and_median_generation_time.csv",
)
def load_sample_comparison_data(folder):
"""Load prompts + generations for the Compare samples tab."""
folder = Path(folder)
prompts_path = folder / "prompts.jsonl"
generations_path = folder / "generations.jsonl"
if not prompts_path.exists() or not generations_path.exists():
return None
prompts = {}
with prompts_path.open() as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
prompts[row["prompt_id"]] = row.get("text", "")
images = defaultdict(dict)
with generations_path.open() as handle:
for line in handle:
if not line.strip():
continue
row = json.loads(line)
model_id = row["model_id"]
prompt_id = row["prompt_id"]
image_url = row.get("image")
if model_id and prompt_id and image_url:
images[model_id][prompt_id] = image_url
models = sorted(images)
if not models or not prompts:
return None
return {
"prompts": prompts,
"images": {model: dict(prompt_map) for model, prompt_map in images.items()},
"models": models,
"prompt_ids": sorted(prompts),
}
def format_url_links(value):
if pd.isna(value) or not value:
return ""
links = [url.strip() for url in str(value).split(",") if url.strip()]
if not links:
return ""
return " ".join(
f'run {idx}'
for idx, url in enumerate(links, start=1)
)
def load_oneig_dataframe(path):
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
df = df.rename(
columns={
"Owner": "Endpoint Owner",
"Optimization": "Optimization Details",
"Anime Alignment Score": "OneIG (Anime Alignment)",
"Human Alignment Score": "OneIG (Human Alignment)",
"Object Alignment Score": "OneIG (Object Alignment)",
"OneIG (Anime and Stylization) (Alignment Score)": "OneIG (Anime Alignment)",
"OneIG (Portrait) (Alignment Score)": "OneIG (Human Alignment)",
"OneIG (General Object) (Alignment Score)": "OneIG (Object Alignment)",
}
)
if "URL" in df.columns:
df["URL"] = df["URL"].apply(format_url_links)
if "Optimized" in df.columns:
df["Optimized"] = df["Optimized"].map(
{True: "Yes", False: "No", "TRUE": "Yes", "FALSE": "No"}
).fillna(df["Optimized"])
for column in [
"Price / Image (USD)",
"Median Generation Time (s)",
"Min Generation Time (s)",
"OneIG (Anime Alignment)",
"OneIG (Human Alignment)",
"OneIG (Object Alignment)",
"OneIG Anime Elo",
"OneIG Human Elo",
"OneIG Object Elo",
"P-Judge Overall",
"Rapidata Elo",
]:
if column in df.columns:
df[column] = pd.to_numeric(df[column], errors="coerce")
preferred_columns = [
"Platform",
"Endpoint Owner",
"Device",
"Model",
"Optimized",
"Optimization Details",
"OneIG (Anime Alignment)",
"OneIG (Human Alignment)",
"OneIG (Object Alignment)",
"OneIG Anime Elo",
"OneIG Human Elo",
"OneIG Object Elo",
"Median Inference Time",
"Median Inference Time (s)",
"Median Generation Time (s)",
"Min Generation Time (s)",
"Median Total Duration (s)",
"Price per Image",
"Price / Image (USD)",
"Evaluation Date (UTC)",
"URL",
]
present_preferred_columns = [col for col in preferred_columns if col in df.columns]
df = df[
present_preferred_columns
+ [col for col in df.columns.tolist() if col not in present_preferred_columns]
]
if "OneIG (Human Alignment)" in df.columns:
df = df.sort_values(
by="OneIG (Human Alignment)", ascending=False, na_position="last"
)
numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
for col in numeric_cols:
df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
return df
def load_qwen_combined_dataframe(path):
"""Load the combined Qwen Image Bench metrics table."""
df = pd.read_csv(path, na_values=["N/A", "n/a", ""])
model_column = "Rapidata Model" if "Rapidata Model" in df.columns else "Model"
df = df.rename(columns={model_column: "Model"})
# Rows prefixed with '#' are commented-out variants in the source exports.
df = df[~df["Model"].astype(str).str.startswith("#")].copy()
df["Model"] = df["Model"].astype(str).str.strip()
for column in [
"Price / Image (USD)",
"Median Generation Time (s)",
"Min Generation Time (s)",
"P-Judge Overall",
"Rapidata Elo",
"Datapoint Elo",
]:
if column in df.columns:
df[column] = pd.to_numeric(df[column], errors="coerce")
sort_column = next(
(
col
for col in [
"Datapoint Elo",
"P-Judge Overall",
"Rapidata Elo",
]
if col in df.columns
),
None,
)
if sort_column:
df = df.sort_values(by=sort_column, ascending=False, na_position="last")
numeric_cols = df.select_dtypes(include=[float, int]).columns.tolist()
for col in numeric_cols:
df[col] = df[col].apply(lambda x: round(x, 4) if pd.notna(x) else x)
return df.reset_index(drop=True)
df = load_oneig_dataframe(oneig_path)
oneig_metric_columns = [
col
for col in [
"OneIG (Anime Alignment)",
"OneIG (Human Alignment)",
"OneIG (Object Alignment)",
]
if col in df.columns
]
oneig_df = df.copy()
if oneig_metric_columns:
oneig_df["OneIG Overall Score"] = oneig_df[oneig_metric_columns].mean(
axis=1, skipna=True
)
oneig_df = oneig_df.sort_values(
by="OneIG Overall Score", ascending=False, na_position="last"
)
oneig_display_columns = [
col
for col in [
"Platform",
"Endpoint Owner",
"Model",
"Optimized",
*oneig_metric_columns,
"OneIG Anime Elo",
"OneIG Human Elo",
"OneIG Object Elo",
"P-Judge Overall",
"Rapidata Elo",
"Evaluation Date (UTC)",
"Median Generation Time (s)",
"Min Generation Time (s)",
"Price / Image (USD)",
"URL",
]
if col in oneig_df.columns
]
# Top-level Leaderboard tab uses the same OneIG table.
display_columns = oneig_display_columns
oneig_combined_dir = _resolve_data_path(
data_dir / "oneig_combined",
space_root.parent / "oneig_combined",
)
qwen_combined_dir = _resolve_data_path(
data_dir / "qwen_image_bench_combined",
space_root.parent / "qwen_image_bench_combined",
)
qwen_path = _resolve_data_path(
data_dir / "qwen_image_bench_model_price_and_median_generation_time.csv",
space_root.parent / "qwen_image_bench_model_price_and_median_generation_time.csv",
)
qwen_df = load_qwen_combined_dataframe(qwen_path)
qwen_score_columns = [
col
for col in [
"P-Judge Overall",
"Datapoint Elo",
"Rapidata Elo",
]
if col in qwen_df.columns
]
qwen_display_columns = [
col
for col in [
"Model",
*qwen_score_columns,
"Raw Win Rate",
"Median Generation Time (s)",
"Min Generation Time (s)",
"Price / Image (USD)",
]
if col in qwen_df.columns
]
qwen_overall_column = (
"Datapoint Elo"
if "Datapoint Elo" in qwen_df.columns
else (qwen_score_columns[0] if qwen_score_columns else None)
)
oneig_samples = load_sample_comparison_data(oneig_combined_dir)
qwen_samples = load_sample_comparison_data(qwen_combined_dir)
# Dataset-first catalogue: each card is a prompt suite; metrics are leaderboard columns.
benchmarks = [
{
"id": "oneig",
"title": "OneIG Alignment",
"emoji": "🎯",
"card_description": (
"Anime/stylization, portrait, and general-object alignment prompts — "
"alignment scores and Datapoint Elo (not the full OneIG suite), plus "
"side-by-side generations."
),
"intro": (
"OneIG Alignment covers the alignment slice of OneIG (not every OneIG "
"dimension). The leaderboard shows category alignment scores and Datapoint "
"Elo columns; Compare samples uses the combined alignment generations."
),
"data": oneig_df,
"columns": oneig_display_columns,
"score_columns": oneig_metric_columns,
"overall_column": "OneIG Overall Score",
"best_score_label": "BEST ONEIG ALIGNMENT",
"metrics_same_scale": True,
"note": (
"> Rankings are ordered by the mean of the available alignment category "
"scores. Missing category scores are not included in that model's mean."
),
"samples": oneig_samples,
},
{
"id": "qwen_image_bench",
"title": "Qwen Image Bench",
"emoji": "🖼️",
"card_description": (
"Qwen image-bench prompts with P-Judge, Datapoint Elo, and Rapidata Elo "
"as metric columns, plus combined generations for comparison."
),
"intro": (
"Qwen Image Bench is a shared prompt suite. The leaderboard joins every "
"available metric for this benchmark; Compare samples uses the combined "
"Qwen generations."
),
"data": qwen_df,
"columns": qwen_display_columns,
"score_columns": qwen_score_columns,
"overall_column": qwen_overall_column,
"best_score_label": "BEST DATAPOINT ELO",
"metrics_same_scale": False,
"note": (
"> Models are ordered by Datapoint Elo when available. Other metric "
"columns come from the P-Judge and Rapidata evaluations on the same "
"prompt suite."
),
"samples": qwen_samples,
},
]
custom_head = """
"""
with gr.Blocks(
theme=theme, fill_width=True, css=custom_css, head=custom_head
) as demo:
render_header()
with gr.Tabs():
with gr.TabItem("Home"):
render_home(benchmarks)
with gr.TabItem("Benchmarks") as benchmarks_tab:
reset_benchmarks, reset_benchmark_outputs = render_benchmarks(benchmarks)
benchmarks_tab.select(
reset_benchmarks,
outputs=reset_benchmark_outputs,
)
with gr.TabItem("About"):
render_about()
render_footer()
if __name__ == "__main__":
demo.launch(ssr_mode=False)