Initial clone with modifications

.ipynb_checkpoints/Gen_llm_eval_output-checkpoint.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+
+#python Gen_llm_eval_output.py   --p1 csv_files/llm_scores_p1.xlsx   --p2 csv_files/llm_scores_p2.xlsx   --p3 csv_files/llm_scores_p3.xlsx   --output-dir csv_files/outputs
+import argparse
+import os
+import re
+import math
+import pandas as pd
+import numpy as np
+
+REQUIRED_COLS = ["model", "task", "language", "configuration", "prompts", "f1"]
+
+def read_scores(path: str) -> pd.DataFrame:
+    df = pd.read_excel(path)
+    # normalize columns
+    df.columns = [c.strip().lower() for c in df.columns]
+    if "prompts" not in df.columns and "prompt" in df.columns:
+        df["prompts"] = df["prompt"]
+    missing = [c for c in REQUIRED_COLS if c not in df.columns]
+    if missing:
+        raise ValueError(f"{path} is missing required columns: {missing}")
+    # keep only required, coerce f1 to numeric
+    df = df[REQUIRED_COLS].copy()
+    df["f1"] = pd.to_numeric(df["f1"], errors="coerce")
+    df = df.dropna(subset=["f1"])
+    return df
+
+def sanitize_filename(s: str) -> str:
+    return re.sub(r"[^0-9A-Za-z._\-+]+", "_", str(s).strip())
+
+def format_float(x):
+    if x is None or (isinstance(x, float) and (math.isnan(x) or math.isinf(x))):
+        return "nan"
+    return f"{x:.4f}"
+
+def prompt_order_key(label: str):
+    # Sort by the number in "prompt-<n>" if present; fallback to string
+    m = re.search(r"(\d+)", str(label))
+    return (0, int(m.group(1))) if m else (1, str(label))
+
+def render_group_table(g: pd.DataFrame, model: str, language: str, configuration: str) -> str:
+    # Collect all prompt-level f1 values (across tasks and prompts)
+    prompt_values = g["f1"].to_numpy(dtype=float)
+    if prompt_values.size > 0:
+        gen_value = float(np.mean(prompt_values))
+        gen_stderr = float(np.std(prompt_values, ddof=1) / math.sqrt(len(prompt_values))) if len(prompt_values) > 1 else 0.0
+    else:
+        gen_value, gen_stderr = float("nan"), 0.0
+
+    # Build table text
+    if configuration=="0shot" : configuration='0'
+    if configuration=="10shot" : configuration='10'
+    model = model.split("__")[0]+'/'+model.split("__")[1]
+    #if model =='Henrychur__MMed-Llama-3-8B' : model='Henrychur/MMed-Llama-3-8B'
+    #if model =='HiTZ__Medical-mT5-large' : model=''
+    #if model =='Qwen__Qwen2.5-14B-Instruct-1M' : model='Qwen/'+model
+    #if model =='Qwen__Qwen2.5-32B-Instruct' : model='Qwen/'+model
+    #if model =='Qwen__Qwen3-30B-A3B-Instruct-2507' : model='Qwen/'+model
+    #if model =='deepseek-ai__DeepSeek-R1-Distill-Qwen-32B' : model=''
+    #if model =='epfl-llm__meditron-7b' : model=''
+    #if model =='google__gemma-2-9b-it' : model=''
+    #if model =='google__gemma-3-27b-it' : model=''
+    #if model =='google__medgemma-27b-text-it' : model=''
+    #if model =='google__medgemma-4b-it' : model=''
+    #if model =='microsoft__MediPhi-Clinical' : model=''
+    #if model =='microsoft__MediPhi-Instruct' : model=''
+    #if model =='mistralai__Mistral-7B-Instruct-v0.2' : model=''
+    #if model =='mistralai__Mistral-Nemo-Instruct-2407' : model=''
+    #if model =='tiiuae__Falcon3-10B-Instruct' : model=''
+    #if model =='unsloth__phi-4' : model=''
+    #if model =='Henrychur__MMed-Llama-3-8B' : model=''
+    
+    header = f"hf (pretrained={model} ), num_fewshot: {configuration}, batch_size: 1"
+    lines = [
+        "|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|",
+        "|-------|-------|------|------|------|----|------|---|------|",
+        #f"|Gen   |       |      |      |f1    |    |{format_float(gen_value)} |---| {format_float(gen_stderr)} |",
+    ]
+
+    # For each task, add task row (mean over prompts) then prompt rows
+    for task, df_task in g.groupby("task", sort=False):
+        f1s = df_task["f1"].to_numpy(dtype=float)
+        task_mean = float(np.mean(f1s)) if f1s.size else float("nan")
+        lines.append(f"| - {task.upper()}        |       |      |      |f1    |   | {format_float(task_mean)} |   |0 |")
+
+        # Prompt-level rows, sorted by prompt number if available
+        df_task = df_task.copy()
+        df_task["_order"] = df_task["prompts"].map(prompt_order_key)
+        df_task = df_task.sort_values("_order")
+        for _, r in df_task.iterrows():
+            prompt_label = str(r["prompts"])
+            lines.append(f"|   - {prompt_label}  |       |      |      |f1    |   | {format_float(r['f1'])} |   | 0 |")
+
+    return header + "\n" + "\n".join(lines) + "\n"
+
+def main():
+    ap = argparse.ArgumentParser(description="Build per-(model,language,configuration) summaries from three prompt Excel files.")
+    ap.add_argument("--p1", required=True, help="Path to llm_scores_p1.xlsx")
+    ap.add_argument("--p2", required=True, help="Path to llm_scores_p2.xlsx")
+    ap.add_argument("--p3", required=True, help="Path to llm_scores_p3.xlsx")
+    ap.add_argument("--output-dir", required=True, help="Directory to write output files")
+    args = ap.parse_args()
+
+    os.makedirs(args.output_dir, exist_ok=True)
+
+    df = pd.concat([read_scores(args.p1), read_scores(args.p2), read_scores(args.p3)], ignore_index=True)
+
+    # One file per (model, language, configuration)
+    for (model, language, config), g in df.groupby(["model", "language", "configuration"], sort=False):
+        content = render_group_table(g, model, language, config)
+        fname = f"{sanitize_filename(model)}__{sanitize_filename(language)}__{sanitize_filename(config)}.txt"
+        out_path = os.path.join(args.output_dir, fname)
+        with open(out_path, "w", encoding="utf-8") as f:
+            f.write(content)
+
+if __name__ == "__main__":
+    main()

.ipynb_checkpoints/app-checkpoint.py

@@ -0,0 +1,1144 @@
+import gradio as gr
+from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
+import pandas as pd
+from apscheduler.schedulers.background import BackgroundScheduler
+from huggingface_hub import snapshot_download
+from src.about import CITATION_BUTTON_LABEL, CITATION_BUTTON_TEXT, EVALUATION_QUEUE_TEXT, INTRODUCTION_TEXT, LLM_BENCHMARKS_TEXT, TITLE
+from src.tasks import TASK_DESCRIPTIONS, MEASURE_DESCRIPTION
+from src.display.css_html_js import custom_css
+from src.display.utils import BENCHMARK_COLS, COLS, EVAL_COLS, EVAL_TYPES, AutoEvalColumn, ModelType, fields, WeightType, Precision
+from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.submission.submit import add_new_eval
+import random
+import matplotlib.pyplot as plt
+import re
+import plotly.express as px
+import plotly.graph_objects as go
+import numpy as np
+
+
+
+
+# === NEW: helper for prompt sensitivity (simple: only NER/REL and 3 prompts) ===
+def calculate_prompt_sensitivity(dataframe, tasks, prompt_ids):
+    """
+    Computes a simple Prompt Sensitivity Index (PSI) over the tasks 
+    using the distribution of 'Best Prompt Id' across the provided prompt_ids.
+    """
+    cv_per_task = []
+    for task in tasks:
+        prompt_col = f"{task} Best Prompt Id"
+        task_accuracies = []
+        for pid in prompt_ids:
+            total = len(dataframe[prompt_col].dropna()) if prompt_col in dataframe.columns else 0
+            count = (dataframe[prompt_col] == pid).sum() if prompt_col in dataframe.columns else 0
+            acc = (count / total * 100) if total > 0 else 0
+            task_accuracies.append(acc)
+        if task_accuracies:
+            mean_acc = np.mean(task_accuracies)
+            std_acc = np.std(task_accuracies)
+            cv_per_task.append((std_acc / mean_acc) if mean_acc > 0 else 0)
+        else:
+            cv_per_task.append(0)
+    mean_cv = np.mean(cv_per_task) if cv_per_task else 0
+    psi = 1.0 if mean_cv >= 0.5 else (mean_cv / 0.5)
+    return psi, mean_cv, cv_per_task
+
+def create_best_model_comparison_table(dataframe, lang: str | None = None, shot: str | None = None):
+    """
+    Table with best overall model per task  and the model with the best prompt score.
+    Applies optional filters:
+      - lang in {EN, IT, SL, SK, GR, PL} or None/"All"
+      - shot in {"0","10"} or None/"All" (mapped to IS_FS False/True)
+    """
+    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+    df = dataframe.copy()
+    
+    if lang and lang != "All" and "LANG" in df.columns:
+        df = df[df["LANG"] == lang]
+    if shot and shot != "All" and "IS_FS" in df.columns:
+        df = df[df["IS_FS"] == (shot == "10")]
+
+    table_data = {'Task': [], 'Best Overall Model': [], 'CPS': [], 'Best Prompt Model': [], 'Acc.': []}
+
+    for task in tasks:
+        if task not in df.columns or df.empty:
+            continue
+        # Best overall on task
+        #max_idx = df[task].idxmax()
+        max_idx = pd.to_numeric(df[task], errors='coerce').idxmax()
+        try: 
+          model_raw = df.loc[max_idx, 'Model']
+        except Exception as e:
+          break
+
+        if isinstance(model_raw, str) and '<' in model_raw:
+            match = re.search(r'>([^<]+)<', model_raw)
+            model_name = match.group(1) if match else model_raw
+        else:
+            model_name = str(model_raw)
+        comb_perf_value = df.loc[max_idx, task]
+
+        # Best prompt row for task
+        best_prompt_column = f"{task} Best Prompt"
+        if best_prompt_column in df.columns and df[best_prompt_column].notna().any():
+            best_prompt_idx= pd.to_numeric(df[best_prompt_column],errors='coerce').idxmax()
+            try:
+             best_prompt_model_raw = df.loc[best_prompt_idx, 'Model']
+            except Exception as e:
+             break
+            if isinstance(best_prompt_model_raw, str) and '<' in best_prompt_model_raw:
+                match = re.search(r'>([^<]+)<', best_prompt_model_raw)
+                best_prompt_model = match.group(1) if match else best_prompt_model_raw
+            else:
+                best_prompt_model = str(best_prompt_model_raw)
+            best_prompt_accuracy = df.loc[best_prompt_idx, best_prompt_column]
+        else:
+            best_prompt_model = "n/a"
+            best_prompt_accuracy = float('nan')
+
+        table_data['Task'].append(task)
+        table_data['Best Overall Model'].append(model_name)
+        table_data['CPS'].append(f"{comb_perf_value:.2f}")
+        table_data['Best Prompt Model'].append(best_prompt_model)
+        table_data['Acc.'].append(f"{best_prompt_accuracy:.2f}" if isinstance(best_prompt_accuracy, (int, float)) else "n/a")
+
+    fig = go.Figure(data=[go.Table(
+        columnwidth=[60, 220, 60, 220, 60],
+        header=dict(
+            values=[f'<b>{col}</b>' for col in table_data.keys()],
+            fill_color=['#2171b5', '#2171b5', '#2171b5', '#4292c6', '#4292c6'],
+            font=dict(color='white', size=12, family='Arial'),
+            align='center', height=30
+        ),
+        cells=dict(
+            values=list(table_data.values()),
+            fill_color=[['#f0f0f0' if i % 2 == 0 else 'white' for i in range(len(table_data['Task']))]],
+            font=dict(color='#2c3e50', size=11, family='Arial'),
+            align=['center', 'left', 'center', 'left', 'center'],
+            height=30
+        )
+    )])
+
+    subtitle = []
+    subtitle.append(lang if (lang and lang != "All") else "All languages")
+    subtitle.append(f"{shot}-shot" if (shot and shot != "All") else "All shots")
+
+    fig.update_layout(
+        title={'text': f"Top Model per Task: CPS & Best Prompt  — {', '.join(subtitle)}",
+               'font': {'family': 'Arial', 'size': 14, 'color': '#2c3e50'}},
+        font=dict(family="Arial", size=11),
+        height=420, margin=dict(l=20, r=20, t=50, b=80)
+    )
+    return fig
+
+
+
+# === NEW: Best-model comparison table (only NER, REL) ===
+def create_best_model_comparison_table_without_lang(dataframe):
+    """
+    Table with the best overall model per task (NER, REL,) and the model that
+    achieves the best score with its own best prompt.
+    """
+    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+    table_data = {'Task': [], 'Best Overall Model': [], 'CPS': [], 'Best Prompt Model': [], 'Acc.': []}
+
+    for task in tasks:
+        if task not in dataframe.columns:
+            continue
+
+        # Best overall on the task's combined performance
+        max_idx = dataframe[task].idxmax()
+        model_raw = dataframe.loc[max_idx, 'Model']
+        if isinstance(model_raw, str) and '<' in model_raw:
+            match = re.search(r'>([^<]+)<', model_raw)
+            model_name = match.group(1) if match else model_raw
+        else:
+            model_name = str(model_raw)
+        comb_perf_value = dataframe.loc[max_idx, task]
+
+        # Model with the best prompt for this task
+        best_prompt_column = f"{task} Best Prompt"
+        if best_prompt_column in dataframe.columns:
+            best_prompt_idx = dataframe[best_prompt_column].idxmax()
+            best_prompt_model_raw = dataframe.loc[best_prompt_idx, 'Model']
+            if isinstance(best_prompt_model_raw, str) and '<' in best_prompt_model_raw:
+                match = re.search(r'>([^<]+)<', best_prompt_model_raw)
+                best_prompt_model = match.group(1) if match else best_prompt_model_raw
+            else:
+                best_prompt_model = str(best_prompt_model_raw)
+            best_prompt_accuracy = dataframe.loc[best_prompt_idx, best_prompt_column]
+        else:
+            best_prompt_model = "n/a"
+            best_prompt_accuracy = float('nan')
+
+        table_data['Task'].append(task)
+        table_data['Best Overall Model'].append(model_name)
+        table_data['CPS'].append(f"{comb_perf_value:.2f}")
+        table_data['Best Prompt Model'].append(best_prompt_model)
+        table_data['Acc.'].append(f"{best_prompt_accuracy:.2f}" if isinstance(best_prompt_accuracy, (int, float)) else "n/a")
+
+    fig = go.Figure(data=[go.Table(
+        columnwidth=[60, 220, 60, 220, 60],
+        header=dict(
+            values=[f'<b>{col}</b>' for col in table_data.keys()],
+            fill_color=['#2171b5', '#2171b5', '#2171b5', '#4292c6', '#4292c6'],
+            font=dict(color='white', size=12, family='Arial'),
+            align='center', height=30
+        ),
+        cells=dict(
+            values=list(table_data.values()),
+            fill_color=[['#f0f0f0' if i % 2 == 0 else 'white' for i in range(len(table_data['Task']))]],
+            font=dict(color='#2c3e50', size=11, family='Arial'),
+            align=['center', 'left', 'center', 'left', 'center'],
+            height=30
+        )
+    )])
+    fig.update_layout(
+        title={'text': "Top Model per Task: CPS & Best Prompt (NER/REL)",
+               'font': {'family': 'Arial', 'size': 14, 'color': '#2c3e50'}},
+        font=dict(family="Arial", size=11),
+        height=420, margin=dict(l=20, r=20, t=50, b=80)
+    )
+    fig.add_annotation(
+        text=("Best Overall Model uses the task's primary metric (CPS). "
+              "Best Prompt Model is the one whose own best prompt yields the highest score."),
+        xref="paper", yref="paper", x=0.5, y=-0.20, showarrow=False,
+        font=dict(size=11, color="gray", family="Arial"), align="center", xanchor="center"
+    )
+    return fig
+
+def create_prompt_heatmap(dataframe, lang: str | None = None, shot: str | None = None):
+    """
+    Heatmap of share (%) of models whose BEST prompt is each pid, for NER/REL with prompts p1..p3.
+    Optional filters:
+      - lang: None or one of EN/IT/SL/SK/GR/PL (None means All)
+      - shot: None or "0"/"10" (None means All) mapped to IS_FS False/True
+    """
+    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+
+    df = dataframe.copy()
+    # Language filter
+    if lang and lang != "All" and "LANG" in df.columns:
+        df = df[df["LANG"] == lang]
+    # Shot filter -> IS_FS (10-shot=True, 0-shot=False)
+    if shot and shot != "All" and "IS_FS" in df.columns:
+        df = df[df["IS_FS"] == (shot == "10")]
+
+    # Collect prompt ids present, normalize labels to p1..p3
+    def label_for(pid):
+        if isinstance(pid, str): return pid
+        try: return f"p{int(pid)}"
+        except Exception: return str(pid)
+
+    all_ids = set()
+    for task in tasks:
+        col = f"{task} Best Prompt Id"
+        if col in df.columns:
+            all_ids.update(df[col].dropna().unique())
+    prompt_ids_raw = sorted(list(all_ids), key=lambda x: int(re.sub(r'[^0-9]', '', str(x)) or 0))
+    prompt_ids_raw = [pid for pid in prompt_ids_raw if label_for(pid) in {"p1", "p2", "p3"}] or [1, 2, 3]
+    y_tick_labels = [label_for(pid) for pid in prompt_ids_raw]
+
+    matrix, hovers = [], []
+    for pid in prompt_ids_raw:
+        row, hover_row = [], []
+        for task in tasks:
+            col = f"{task} Best Prompt Id"
+            if col in df.columns and len(df[col].dropna()) > 0:
+                series = df[col].dropna()
+
+                def same_pid(v):
+                    a = re.sub(r'[^0-9]', '', str(v))
+                    b = re.sub(r'[^0-9]', '', str(pid))
+                    return a == b and a != ""
+
+                total = len(series)
+                count = sum(same_pid(v) for v in series)
+                pct = (count / total * 100) if total > 0 else 0
+                row.append(pct)
+                hover_row.append(f"<b>{task} — {label_for(pid)}</b><br>Models: {count}/{total}<br>Percentage: {pct:.1f}%")
+            else:
+                row.append(0); hover_row.append(f"<b>{task} — {label_for(pid)}</b><br>No data")
+        matrix.append(row); hovers.append(hover_row)
+
+    fig = go.Figure(data=go.Heatmap(
+        z=matrix, x=tasks, y=y_tick_labels,
+        colorscale=[[0,'#f7fbff'],[0.2,'#deebf7'],[0.4,'#9ecae1'],[0.6,'#4292c6'],[0.8,'#2171b5'],[1,'#08519c']],
+        text=[[f"{val:.0f}%" if val is not None else "" for val in row] for row in matrix],
+        texttemplate="%{text}", textfont={"size": 11, "family": "Arial"},
+        hovertemplate='%{customdata}<extra></extra>', customdata=hovers,
+        colorbar=dict(title="% Models", ticksuffix="%"),
+        zmin=0, zmax=100
+    ))
+
+    title_parts = []
+    title_parts.append(lang if (lang and lang != "All") else "All languages")
+    title_parts.append(f"{shot}-shot" if (shot and shot != "All") else "All shots")
+    fig.update_layout(
+        title={'text': f"Most Effective Prompts  — {', '.join(title_parts)}",
+               'font': {'family': 'Arial', 'size': 14, 'color': '#2c3e50'}},
+        xaxis_title="Task", yaxis_title="Prompt",
+        font=dict(family="Arial", size=11), margin=dict(b=100),
+        template="plotly_white", dragmode=False, height=420
+    )
+    fig.update_xaxes(fixedrange=True); fig.update_yaxes(fixedrange=True)
+    return fig
+
+
+# === NEW: Prompt heatmap (only NER, REL; 3 prompts p1, p2, p3) ===
+def create_prompt_heatmap_without_lang(dataframe):
+    """
+    Heatmap of the share of models (in %) whose BEST prompt for the task is each prompt id,
+    for tasks NER and REL, with exactly 3 prompts (p1, p2, p3). It supports columns storing
+    ids as integers (1/2/3) or strings ('p1'/'p2'/'p3').
+    """
+    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+
+    # Collect unique prompt ids as they appear (int or 'pX'); restrict to 3 prompts
+    all_ids = set()
+    for task in tasks:
+        col = f"{task} Best Prompt Id"
+        if col in dataframe.columns:
+            all_ids.update(dataframe[col].dropna().unique())
+
+    # Normalize to display labels and preserve the original values as keys
+    def label_for(pid):
+        if isinstance(pid, str):
+            return pid  # e.g., 'p1'
+        try:
+            return f"p{int(pid)}"
+        except Exception:
+            return str(pid)
+
+    prompt_ids_raw = sorted(list(all_ids), key=lambda x: int(re.sub(r'[^0-9]', '', str(x)) or 0))
+    # Optional: hard-limit to p1/p2/p3 if extra noise exists
+    prompt_ids_raw = [pid for pid in prompt_ids_raw if label_for(pid) in {"p1", "p2", "p3"}]
+
+    if not prompt_ids_raw:
+        # Fallback to p1..p3
+        prompt_ids_raw = [1, 2, 3]
+
+    y_tick_labels = [label_for(pid) for pid in prompt_ids_raw]
+
+    matrix, hovers = [], []
+    for pid in prompt_ids_raw:
+        row, hover_row = [], []
+        for task in tasks:
+            col = f"{task} Best Prompt Id"
+            if col in dataframe.columns:
+                series = dataframe[col].dropna()
+                # match values regardless of 'p1' vs 1 vs '1'
+                def same_pid(v):
+                    a = re.sub(r'[^0-9]', '', str(v))
+                    b = re.sub(r'[^0-9]', '', str(pid))
+                    return a == b and a != ""
+                total = len(series)
+                count = sum(same_pid(v) for v in series)
+                pct = (count / total * 100) if total > 0 else 0
+                row.append(pct)
+                hover_row.append(
+                    f"<b>{task} — {label_for(pid)}</b><br>Models: {count}/{total}<br>Percentage: {pct:.1f}%"
+                )
+            else:
+                row.append(0); hover_row.append(f"<b>{task} — {label_for(pid)}</b><br>No data")
+        matrix.append(row)
+        hovers.append(hover_row)
+
+    fig = go.Figure(data=go.Heatmap(
+        z=matrix, x=tasks, y=y_tick_labels,
+        colorscale=[[0,'#f7fbff'],[0.2,'#deebf7'],[0.4,'#9ecae1'],[0.6,'#4292c6'],[0.8,'#2171b5'],[1,'#08519c']],
+        text=[[f"{val:.0f}%" if val is not None else "" for val in row] for row in matrix],
+        texttemplate="%{text}",
+        textfont={"size": 11, "family": "Arial"},
+        hovertemplate='%{customdata}<extra></extra>',
+        customdata=hovers,
+        colorbar=dict(title="% Models", ticksuffix="%"),
+        zmin=0, zmax=100
+    ))
+    fig.update_layout(
+        title={'text': "Most Effective Prompts Across Models (NER/REL)",
+               'font': {'family': 'Arial', 'size': 14, 'color': '#2c3e50'}},
+        xaxis_title="Task", yaxis_title="Prompt",
+        font=dict(family="Arial", size=11),
+        margin=dict(b=120), template="plotly_white", dragmode=False, height=420
+    )
+
+    # PSI (optional info line)
+    psi, mean_cv, _ = calculate_prompt_sensitivity(
+        dataframe, tasks, prompt_ids_raw
+    )
+    fig.add_annotation(
+        text=f"Prompt Sensitivity (mean CV): {mean_cv:.2f}",
+        xref="paper", yref="paper", x=0.3, y=1.12, showarrow=False,
+        font=dict(size=11, color="#2c3e50", family="Arial")
+    )
+
+    fig.update_xaxes(fixedrange=True); fig.update_yaxes(fixedrange=True)
+    return fig
+
+
+
+
+
+
+
+def mean_of_max_per_field(df):
+    """
+    Calcola il massimo per ciascun campo e poi la media dei massimi.
+
+    Args:
+        df (pd.DataFrame): DataFrame con colonne TE, SA, HS, AT, WIC, FAQ, LS, SU, NER, REL, RML, DIA, HIS
+
+    Returns:
+        float: media dei valori massimi dei campi
+    """
+    #fields = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+    fields = ["NER", "REL", "RML", "DIA", "HIS"]
+    #print(df.columns)
+
+    # Controlla che tutte le colonne esistano nel DataFrame
+    missing = [f for f in fields if f not in df.columns]
+    if missing:
+        raise ValueError(f"Le seguenti colonne mancano nel DataFrame: {missing}")
+
+    # Calcola il massimo per ciascun campo
+    max_values = df[fields].apply(pd.to_numeric, errors='coerce').max(skipna=True)
+
+    # Calcola la media dei massimi
+    mean_max = max_values.mean()
+
+    return mean_max
+
+
+def barplot_mean_few_minus_zero_shot(dataframe, tasks=None):
+    if tasks is None:
+        tasks = [ "NER", "REL", "RML", "DIA", "HIS"]
+ 
+    task_means = {}
+
+    for task in tasks:
+        if task not in dataframe.columns:
+            continue
+
+        # Separa few-shot e zero-shot
+        few_shot = dataframe[dataframe['IS_FS'] == True][["Model", task]]
+        zero_shot = dataframe[dataframe['IS_FS'] == False][["Model", task]]
+
+        # Allinea i modelli
+        merged = pd.merge(few_shot, zero_shot, on="Model", suffixes=("_few", "_zero"))
+
+        # Rimuovi righe con valori mancanti
+        merged = merged.dropna(subset=[f"{task}_few", f"{task}_zero"])
+
+        if merged.empty:
+            continue
+
+        # Calcola differenza few - zero
+        diff = merged[f"{task}_few"] - merged[f"{task}_zero"]
+
+        # Calcola la media
+        task_means[task] = diff.mean()
+
+    # Crea barplot
+    fig = go.Figure([go.Bar(
+        x=list(task_means.keys()),
+        y=list(task_means.values()),
+        marker_color="#ff7f0e",
+        text=[f"{v:.2f}" for v in task_means.values()],
+        textposition="outside",
+        hovertemplate="<b>%{x}</b><br>Mean Delta Accuracy: %{y:.2f}%<extra></extra>"
+    )])
+
+    # Linea di riferimento a 0
+    '''
+    fig.add_shape(
+        type="line",
+        x0=-0.5, x1=len(task_means) - 0.5,
+        y0=0, y1=0,
+        line=dict(color="black", width=2, dash="dash"),
+        xref="x", yref="y"
+    )
+    '''
+
+    fig.update_layout(
+        title="Mean Accuracy Difference (Few-shot − Zero-shot) per Task",
+        xaxis_title="",
+        yaxis_title="Mean Delta Combined Performance",
+        template="plotly_white",
+        font=dict(family="Arial", size=13),
+        #margin=dict(b=100)
+    )
+
+    fig.add_annotation(
+        text="10-shot learning generally outperforms zero-shot. <br>"
+             "",
+        xref="paper", yref="paper",
+        x=0, y=-0.2,
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="left"
+    )
+
+    return fig
+
+
+def boxplot_per_task(dataframe=None, baselines=None, references=None):
+
+    #print(dataframe.columns)
+
+    #tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+    tasks =["NER", "REL", "RML", "HIS", "DIA"]
+    if dataframe is None:
+        np.random.seed(42)
+        dataframe = pd.DataFrame({
+            task: np.random.uniform(0.4, 0.9, 20) * 100
+            for task in tasks
+        })
+
+    if baselines is None:
+        baselines = {task: np.random.randint(50, 70) for task in tasks}
+
+    colors = ["#1f77b4", "#ff7f0e", "#2ca02c", "#d62728", "#9467bd",
+              "#8c564b", "#e377c2", "#7f7f7f", "#bcbd22", "#17becf"]
+
+    fig = go.Figure()
+
+    for i, task in enumerate(tasks):
+        if task in dataframe.columns:
+            y_data = dataframe[task].dropna().tolist()
+
+            # boxplot
+            fig.add_trace(go.Box(
+                y=y_data,
+                name=task,
+                marker=dict(color=colors[i]),
+                line=dict(color="black", width=2),
+                fillcolor=colors[i],
+                opacity=0.7,
+                hovertemplate="<b>"+task+"</b><br>Accuracy: %{y:.2f}%<extra></extra>",
+                width=0.6,
+                whiskerwidth=0.2,
+                quartilemethod="linear"
+            ))
+
+            # baseline
+            #if task in baselines and baselines[task] is not None:
+                #fig.add_shape(
+                #    type="line",
+                #    x0=i - 0.3, x1=i + 0.3,
+                #    y0=baselines[task], y1=baselines[task],
+                #   line=dict(color="black", width=2, dash="dot"),  # più visibile
+                 #   xref="x", yref="y"
+                #)
+                #'''
+                #fig.add_annotation(
+                    #x=i, y=baselines[task],
+                    #text=f"{baselines[task]}%",
+                    #showarrow=False,
+                    #yshift=10,
+                    #font=dict(size=10, color="black")
+                #)
+                #'''
+
+            # reference GPT-4o
+           # if task in references and references[task] is not None:
+             #   fig.add_shape(
+            #        type="line",
+           #         x0=i - 0.3, x1=i + 0.3,
+           #         y0=references[task], y1=references[task],
+           #         line=dict(color="red", width=2, dash="dashdot"),
+           #         xref="x", yref="y"
+           #     )
+
+    fig.update_layout(
+        title="Distribution of Model Accuracy by Task",
+        xaxis_title="Task",
+        yaxis_title="Combined Performance",
+        template="plotly_white",
+        boxmode="group",
+        dragmode=False,
+        font=dict(family="Arial", size=10),
+        margin=dict(b=80),
+    )
+
+    fig.add_annotation(
+        text=(""
+            #"In tasks like TE and SA, models approach the accuracy of supervised <br>"
+            #"models at EVALITA (dashed black line); in NER and REL they remain lower. <br>"
+           # "Dashed red lines show GPT-4o reference results for generative tasks."
+        ),
+        xref="paper", yref="paper",
+        x=0.5, y=-0.30,
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="left"
+    )
+
+    fig.update_yaxes(range=[0, 100], fixedrange=True)
+
+    return fig
+
+# EVALITA results
+BASELINES = {
+    "TE":71.00, "SA": 66.38, "HS": 80.88, "AT": 82.40, "WIC": 85.00,
+    "LS": 38.82, "SU": 38.91, "NER":88.00, "REL": 62.99
+}
+
+# GPT-4o
+REFERENCES = {
+    "NER": 79.11,
+    "REL": 63.32,
+    "LS": 59.25,
+    "SU": 33.04
+
+}
+
+
+def boxplot_prompts_per_task(dataframe, tasks=None):
+    if tasks is None:
+        tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
+
+    # Lista delle colonne da aggiornare
+    cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"]
+    # Applichiamo la trasformazione
+    for col in cols_to_update:
+        dataframe[col] = dataframe[col].replace({1: 7, 2: 8})
+
+    fig = go.Figure()
+
+    # Liste per creare una sola voce in legenda per Average e Best
+    avg_x, avg_y = [], []
+    best_x, best_y, best_text = [], [], []
+
+    for task in tasks:
+        avg_col = f"{task} Prompt Average"
+        best_col = f"{task} Best Prompt"
+        best_id_col = f"{task} Best Prompt Id"
+
+        if all(col in dataframe.columns for col in [avg_col, best_col, best_id_col]):
+            avg_value = dataframe[avg_col].mean()
+            avg_x.append(task)
+            avg_y.append(avg_value)
+
+            best_value = dataframe[best_col].mean()
+            best_x.append(task)
+            best_y.append(best_value)
+            best_id = dataframe[best_id_col].mode()[0]  # Most frequent best prompt id
+            best_text.append(f"P:{best_id}")
+
+    # Barre Average Accuracy (azzurro)
+    fig.add_trace(go.Bar(
+        x=avg_x,
+        y=avg_y,
+        name="Avg. Accuracy",
+        marker_color="#1f77b4",
+    ))
+
+    # Barre Best Prompt (rosso)
+    fig.add_trace(go.Bar(
+        x=best_x,
+        y=best_y,
+        name="Best Prompt",
+        marker_color="#d62728",
+    ))
+
+    # Testo sopra barre Best Prompt con ID
+    for x, y, text in zip(best_x, best_y, best_text):
+        fig.add_annotation(
+            x=x,
+            y=y + 3,  # leggermente sopra la barra
+            text=text,
+            showarrow=False,
+            font=dict(size=12, color="black")
+        )
+
+    fig.update_layout(
+        title= "Prompt Accuracy: Avg vs Best",
+        xaxis_title="Task",
+        yaxis_title="Combined Performance",
+        barmode='group',
+        template="plotly_white",
+        font=dict(family="Arial", size=10),
+        yaxis=dict(range=[0, 100], fixedrange=True)
+    )
+
+    # caption come annotazione separata
+    fig.add_annotation(
+        text="There is no single prompt that performs best across all tasks.<br>"
+             "Different prompts achieve the highest accuracy on different tasks.",
+        xref="paper", yref="paper",
+        x=0.5, y=-0.3,
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="center",
+        xanchor="center"
+    )
+
+    return fig
+
+
+def line_chart(dataframe):
+
+    # Normalizza le dimensioni per avere marker non troppo piccoli né enormi
+    def scale_sizes(values, min_size=8, max_size=30):
+        vmin, vmax = min(values), max(values)
+        return [
+            min_size + (val - vmin) / (vmax - vmin) * (max_size - min_size) if vmax > vmin else (min_size + max_size) / 2
+            for val in values
+        ]
+
+    # dati in base a IS_FS
+    df_true = dataframe[dataframe['IS_FS'] == True]
+    df_false = dataframe[dataframe['IS_FS'] == False]
+
+    # Estrai valori x, y e labels
+    x_true = df_true['#Params (B)'].tolist()
+    y_true = df_true['Avg. Comb. Perf. ⬆️'].tolist()
+    labels_true = [re.search(r'>([^<]+)<', m).group(1) for m in df_true['Model'].tolist()]
+
+    x_false = df_false['#Params (B)'].tolist()
+    y_false = df_false['Avg. Comb. Perf. ⬆️'].tolist()
+    labels_false = [re.search(r'>([^<]+)<', m).group(1) for m in df_false['Model'].tolist()]
+
+    fig = go.Figure()
+
+    # Punti IS_FS=True
+    fig.add_trace(go.Scatter(
+        x=x_true,
+        y=y_true,
+        mode='markers',
+        name='10-Shot',
+        marker=dict(
+            color='blue',
+            size=scale_sizes(x_true)
+        ),
+        hovertemplate='<b>%{customdata}</b><br>#Params: %{x}<br>Performance: %{y}<extra></extra>',
+        customdata=labels_true
+    ))
+
+    # Punti IS_FS=False
+    fig.add_trace(go.Scatter(
+        x=x_false,
+        y=y_false,
+        mode='markers',
+        name='0-Shot',
+        marker=dict(
+            color='red',
+            size=scale_sizes(x_false)
+        ),
+        hovertemplate='<b>%{customdata}</b><br>#Params: %{x}<br>Performance: %{y}<extra></extra>',
+        customdata=labels_false
+    ))
+
+    # Trova il massimo tra tutti i modelli
+    all_y = y_true + y_false
+    all_x = x_true + x_false
+    all_labels = labels_true + labels_false
+    max_idx = all_y.index(max(all_y))
+    max_x = all_x[max_idx]
+    max_y = all_y[max_idx]
+    max_label = all_labels[max_idx]
+
+    # Aggiungi annotazione visibile per il modello migliore
+    fig.add_annotation(
+        x=max_x,
+        y=max_y,
+        #text=f"Top: {max_label} ({max_y:.1f}%)",
+        text=f"{max_label}",
+        showarrow=True,
+        arrowhead=2,
+        arrowsize=1,
+        arrowwidth=2,
+        arrowcolor="black",
+        font=dict(size=11, color="black"),
+        xshift=10,
+        yshift=10,
+        ax = -30, ay = -20,  # sposta la label a sinistra e sopra il punto
+        xanchor = "right"  # allinea la label a destra rispetto al punto
+    )
+
+    fig.update_layout(
+        title="Avg. Combined Performance vs #Params",
+        xaxis_title="#Params (B)",
+        yaxis_title="Avg. Combined Performance",
+        template="plotly_white",
+        hovermode="closest",
+        font=dict(family="Arial", size=10),
+        dragmode=False,
+        xaxis=dict(
+            tickvals=[0, 25, 50, 75, 100, 125],
+            ticktext=["0", "25", "50", "75", "100"]
+        ),
+        yaxis=dict(
+            tickvals=[0, 20, 40, 60, 80, 100],  # 👈 tick fissi
+            range=[0, 100]  # 👈 range bloccato
+        )
+    )
+
+    # Caption
+    fig.add_annotation(
+        text="Accuracy generally rises with #Params, but smaller models <br>"
+             "with 10-shot can outperform larger zero-shot models.",
+        xref="paper", yref="paper",
+        x=0.5, y=-0.3,  # 👈 centrata
+        showarrow=False,
+        font=dict(size=11, color="gray"),
+        align="center",
+        xanchor="center"  # 👈 ancora centrata rispetto al testo
+    )
+
+    fig.update_xaxes(fixedrange=True, rangeslider_visible=False)
+    fig.update_yaxes(fixedrange=True)
+
+    return fig
+
+
+# Define task metadata (icons, names, descriptions)
+TASK_METADATA_MULTIPLECHOICE = {
+    #"TE": {"icon": "📊", "name": "Textual Entailment", "tooltip": ""},
+    #"SA": {"icon": "😃", "name": "Sentiment Analysis", "tooltip": ""},
+    #"HS": {"icon": "⚠️", "name": "Hate Speech", "tooltip": ""},
+    #"AT": {"icon": "🏥", "name": "Admission Test", "tooltip": ""},
+    #"WIC": {"icon": "🔤", "name": "Word in Context", "tooltip": ""},
+    #"FAQ": {"icon": "❓", "name": "Frequently Asked Questions", "tooltip": ""}
+}
+
+# Define task metadata (icons, names, descriptions)
+TASK_METADATA_GENERATIVE = {
+    
+    "NER": {"icon": "🏷️", "name": "Named Entity Recognition", "tooltip": ""},
+    "REL": {"icon": "🔗", "name": "Relation Extraction", "tooltip": ""},
+    "RML": {"icon": "😃", "name": "CRF RML", "tooltip": "CRF RML"},
+    "DIA": {"icon": "🏥", "name": "CRF Diagnosis", "tooltip": "CRF Diagnosis"},
+    "HIS": {"icon": "📝", "name": "CRF History", "tooltip": "CRF History"},
+}
+
+def restart_space():
+    """Restart the Hugging Face space."""
+    API.restart_space(repo_id=REPO_ID)
+
+
+def init_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """
+    Initialize and return the leaderboard when it is first loaded or when 'benchmark' is selected.
+    The table is sorted based on the "Avg. Combined Performance" field.
+    """
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+
+    #print("????????????????????????????????", mean_of_max_per_field(dataframe))
+
+    sorted_dataframe = dataframe.sort_values(by="Avg. Comb. Perf. ⬆️", ascending=False)
+
+    sorted_dataframe = sorted_dataframe.reset_index(drop=True)
+    sorted_dataframe["Rank"] = sorted_dataframe.index + 1
+
+    # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo
+    large_medal_fs_assigned = False
+    medium_medal_fs_assigned = False
+    small_medal_fs_assigned = False
+
+    large_medal_0shot_assigned = False
+    medium_medal_0shot_assigned = False
+    small_medal_0shot_assigned = False
+
+    # Lista temporanea per salvare i nuovi valori della colonna Model
+    new_model_column = []
+
+    for _, row in sorted_dataframe.iterrows():
+        if row['IS_FS']:  # 10-Few-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆")
+                large_medal_fs_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🏆")
+                medium_medal_fs_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🏆")
+                small_medal_fs_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+        else:  # 0-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️")
+                large_medal_0shot_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🎖️")
+                medium_medal_0shot_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🎖️")
+                small_medal_0shot_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+
+    # Lista delle colonne da aggiornare
+    #cols_to_update = ["REL Best Prompt Id", "NER Best Prompt Id", "SU Best Prompt Id", "LS Best Prompt Id"]
+    # Applichiamo la trasformazione
+    #for col in cols_to_update:
+    #    dataframe[col] = dataframe[col].replace({1: 7, 2: 8})
+
+    # Aggiorna la colonna Model
+    sorted_dataframe["Model"] = new_model_column
+
+    field_list = fields(AutoEvalColumn)
+
+    return Leaderboard(
+        value=sorted_dataframe,
+        datatype=[c.type for c in field_list],
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS): "),
+            ColumnFilter(AutoEvalColumn.LANG.name, type="checkboxgroup", label="Languges: "),
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max = 100, default = [0,100], label="Select the number of parameters (B)"),
+        ],
+        bool_checkboxgroup_label="Evaluation Mode",
+        interactive=False,
+    )
+
+def update_task_leaderboard(dataframe, default_selection=None, hidden_columns=None):
+    """
+    Update and return the leaderboard when a specific task is selected.
+    The table is sorted based on the "Combined Performance" field.
+    """
+    if dataframe is None or dataframe.empty:
+        raise ValueError("Leaderboard DataFrame is empty or None.")
+    #sorted_dataframe = dataframe.sort_values(by="Combined Performance", ascending=False)
+    clean_df = dataframe.assign( **{"Combined Performance": pd.to_numeric(dataframe["Combined Performance"], errors="coerce")}).loc[lambda df: df["Combined Performance"].notna() & (df["Combined Performance"] != 0)]
+  
+    sorted_dataframe = clean_df.sort_values(by="Combined Performance", ascending=False)
+
+    # aggiungo la colonna rank in base alla posizione
+    sorted_dataframe = sorted_dataframe.reset_index(drop=True)
+    sorted_dataframe["Rank"] = sorted_dataframe.index + 1
+
+    # Flag per sapere se la medaglia è già stata assegnata per categoria e tipo
+    large_medal_fs_assigned = False
+    medium_medal_fs_assigned = False
+    small_medal_fs_assigned = False
+
+    large_medal_0shot_assigned = False
+    medium_medal_0shot_assigned = False
+    small_medal_0shot_assigned = False
+
+    # Lista temporanea per salvare i nuovi valori della colonna Model
+    new_model_column = []
+
+    for _, row in sorted_dataframe.iterrows():
+        if row['IS_FS']:  # 5-Few-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🏆")
+                large_medal_fs_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🏆")
+                medium_medal_fs_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_fs_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🏆")
+                small_medal_fs_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+        else:  # 0-Shot
+            if row["Size"] == "🔵🔵🔵" and not large_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🔵🎖️")
+                large_medal_0shot_assigned = True
+            elif row["Size"] == "🔵🔵" and not medium_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🔵🎖️")
+                medium_medal_0shot_assigned = True
+            elif row["Size"] == "🔵" and not small_medal_0shot_assigned:
+                new_model_column.append(f"{row['Model']} 🔵🎖️")
+                small_medal_0shot_assigned = True
+            else:
+                new_model_column.append(row["Model"])
+
+    # Aggiorna la colonna Model
+    sorted_dataframe["Model"] = new_model_column
+
+    pd.set_option('display.max_colwidth', None)
+    #print("========================", dataframe['Model'])
+
+    #print(sorted_dataframe['Combined Performance'])
+
+    field_list = fields(AutoEvalColumn)
+
+    return Leaderboard(
+        value=sorted_dataframe,
+        #datatype=[c.type for c in field_list],
+        datatype=[c.type for c in field_list] + [int],
+        #select_columns=SelectColumns(
+        #    default_selection=default_selection or [c.name for c in field_list if c.displayed_by_default],
+        #    cant_deselect=[c.name for c in field_list if c.never_hidden],
+        #    label="Select Columns to Display:",
+        #),
+        search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
+        hide_columns=hidden_columns or [c.name for c in field_list if c.hidden],
+        filter_columns=[
+            ColumnFilter(AutoEvalColumn.fewshot_symbol.name, type="checkboxgroup", label="N-Shot Learning (FS): "),
+            ColumnFilter(AutoEvalColumn.LANG.name, type="checkboxgroup", label="Languges:  "),
+
+            ColumnFilter(AutoEvalColumn.params.name, type="slider", min=0, max=100, default=[0, 100],
+                         label="Select the number of parameters (B)"),
+        ],
+        bool_checkboxgroup_label="Evaluation Mode",
+        interactive=False
+    )
+
+
+
+def download_snapshot(repo, local_dir):
+    """Try to download a snapshot from Hugging Face Hub."""
+    try:
+        print(f"Downloading from {repo} to {local_dir}...")
+        snapshot_download(repo_id=repo, local_dir=local_dir, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN)
+    except Exception as e:
+        print(f"Error downloading {repo}: {e}")
+        restart_space()
+
+
+# Initialize the app by downloading snapshots
+download_snapshot(QUEUE_REPO, EVAL_REQUESTS_PATH)
+download_snapshot(RESULTS_REPO, EVAL_RESULTS_PATH)
+
+# Load leaderboard data
+LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+finished_eval_queue_df, running_eval_queue_df, pending_eval_queue_df = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
+#print(LEADERBOARD_DF.columns.tolist())
+
+theoretical_max_combined_perf = mean_of_max_per_field(LEADERBOARD_DF)
+
+# Prepare the main interface
+demo = gr.Blocks(css=custom_css)
+with demo:
+    #gr.HTML(TITLE)
+    gr.HTML(
+        """
+        <div style="display: flex; align-items: center; position: relative; width: 100%; height: 60px; padding: 10px 0;">
+            <h1 style="
+                margin: 0 auto; 
+                font-weight: 900; 
+                font-size: 5.5em; 
+                letter-spacing: 2px; 
+                text-transform: uppercase; 
+                color: red;
+                background: linear-gradient(90deg, #1f77b4, #00c6ff); 
+                -webkit-background-clip: text; 
+                -webkit-text-fill-color: transparent; 
+                text-shadow: 2px 2px 8px rgba(0.2,0,0,0);
+            ">
+                ECREAM-LLM Leaderboard
+            </h1>
+        </div>
+        """
+    )
+    gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
+
+    # ⬇️ QUI aggiungiamo i grafici subito sotto la barra del titolo e sopra le tabs
+    with gr.Row():
+        gr.Plot(value=line_chart(LEADERBOARD_DF), elem_id="line-chart")
+        gr.Plot(value=boxplot_per_task(LEADERBOARD_DF, BASELINES, REFERENCES), elem_id="boxplot-task")
+
+    # === NEW: second row with the 2 extra plots (NER/REL + p1..p3) ===
+    #with gr.Row():
+        #gr.Plot(value=create_prompt_heatmap(LEADERBOARD_DF), elem_id="prompt-heatmap")
+        #gr.Plot(value=create_best_model_comparison_table(LEADERBOARD_DF), elem_id="best-model-table")
+    # === NEW: gray background wrapper for combos ===
+    with gr.Row(elem_id="filters-wrap"):
+      lang_dd = gr.Dropdown(
+        choices=["All", "EN", "IT", "SL", "SK", "GR", "PL"],
+        value="All", label="Language: ", scale=1
+      )
+      shot_dd = gr.Dropdown(
+        choices=["All", "0", "10"],
+        value="All", label="N-Shot: ", scale=1
+      )
+
+    with gr.Row():
+      heatmap_plot = gr.Plot(value=create_prompt_heatmap(LEADERBOARD_DF, None, None), elem_id="prompt-heatmap")
+      table_plot = gr.Plot(value=create_best_model_comparison_table(LEADERBOARD_DF, None, None), elem_id="best-model-table")
+
+    def _update_both(lang, shot):
+      return (
+        create_prompt_heatmap(LEADERBOARD_DF, None if lang == "All" else lang, None if shot == "All" else shot),
+        create_best_model_comparison_table(LEADERBOARD_DF, None if lang == "All" else lang, None if shot == "All" else shot)
+      )
+
+    lang_dd.change(_update_both, inputs=[lang_dd, shot_dd], outputs=[heatmap_plot, table_plot])
+    shot_dd.change(_update_both, inputs=[lang_dd, shot_dd], outputs=[heatmap_plot, table_plot])
+
+
+  
+    with gr.Tabs(elem_classes="tab-buttons") as tabs:
+
+        # Main leaderboard tab
+        with gr.TabItem("🏅 Benchmark"):
+
+            leaderboard = init_leaderboard(
+                LEADERBOARD_DF,
+                default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "RML", "DIA", "HIS"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "RML", "DIA", "HIS"]]
+            )
+
+
+        # About tab
+        with gr.TabItem("📝 About"):
+            gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+
+        # Task-specific leaderboards
+        for task, metadata in TASK_METADATA_MULTIPLECHOICE.items():
+
+            with gr.TabItem(f"{metadata['icon']}{task}"):
+
+                task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
+                gr.Markdown(task_description, elem_classes="markdown-text")
+
+                leaderboard = update_task_leaderboard(
+                    LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average", f"{task} Prompt Std": "Prompt Std", f"{task} Best Prompt": "Best Prompt", f"{task} Best Prompt Id": "Best Prompt Id", task: "Combined Performance"}),
+                    default_selection=['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id'],
+                    hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt', 'Best Prompt Id']]
+                )
+
+        # About tab
+        with gr.TabItem("│", interactive=False):
+            gr.Markdown("", elem_classes="markdown-text")
+
+        # Task-specific leaderboards
+        for task, metadata in TASK_METADATA_GENERATIVE.items():
+            with gr.TabItem(f"{metadata['icon']}{task}"):
+                task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
+                gr.Markdown(task_description, elem_classes="markdown-text1")
+                #print (LEADERBOARD_DF)
+                leaderboard = update_task_leaderboard(
+                    LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average",
+                                                   f"{task} Prompt Std": "Prompt Std",
+                                                   f"{task} Best Prompt": "Best Prompt",
+                                                   f"{task} Best Prompt Id": "Best Prompt Id",
+                                                   task: "Combined Performance"}),
+                    default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std', 'Best Prompt',
+                                       'Best Prompt Id'],
+                    hidden_columns=[col for col in LEADERBOARD_DF.columns if
+                                    col not in ['Rank', 'Size','LANG', 'FS', 'Model', 'Combined Performance', 'Prompt Average', 'Prompt Std',
+                                                'Best Prompt', 'Best Prompt Id']]
+                )
+
+    # Citation section
+    with gr.Accordion("📙 Citation", open=False):
+        gr.Textbox(value=CITATION_BUTTON_TEXT, label=CITATION_BUTTON_LABEL, lines=20, elem_id="citation-button", show_copy_button=True)
+
+    with gr.Accordion("📙 Credits", open=False):
+        gr.Markdown(
+            """
+            ***This project has been funded by the European Union under:
+
+                   Horizon Europe eCREAM Project (Grant Agreement No.101057726)  
+            """
+        )
+
+# Background job to restart space
+scheduler = BackgroundScheduler()
+scheduler.add_job(restart_space, "interval", seconds=1800)
+scheduler.start()
+
+# Launch the app with concurrent queueing
+demo.queue(default_concurrency_limit=40).launch(debug=True,  # Enable Gradio debug mode
+        show_error=True)

.ipynb_checkpoints/preprocess_models_output-checkpoint.py

@@ -0,0 +1,290 @@
+"""
+EVALITA LLM EVALUATION PROCESSOR
+
+Transforms raw model evaluation outputs into structured performance reports for leaderboard integration.
+
+DATA PIPELINE OVERVIEW:
+
+1. Inputs:
+   - Evaluation Results: Raw .out files from lm-eval-harness
+   - Model Metadata: Pre-collected .json files from HuggingFace
+
+2. Output:
+   - Comprehensive evaluation reports in JSON format
+   - Ready for ingestion into the evaluation leaderboard
+
+--------------------------------------------------------------------
+INPUT SPECIFICATION
+
+Evaluation Results (.out format):
+   hf (pretrained=model-org/model-name), num_fewshot: 5, batch_size: 1
+   | Task          | Metric | Value  | Stderr |
+   |---------------|--------|--------|--------|
+   | main-task     | acc    | 0.5605 | 0.0052 |
+   | - sub-task    | acc    | 0.4640 | 0.0088 |
+   |   - prompt-1  | acc    | 0.3720 | 0.0216 |
+
+Model Metadata (.json format):
+   {
+     "model": "model-org/model-name",
+     "base_model": "ModelArchitecture",
+     "revision": "git_commit_hash",
+     "parameters": 8.03,
+     "language": "en_it"
+   }
+
+--------------------------------------------------------------------
+OUTPUT SPECIFICATION
+
+Evaluation Report (.json format):
+   {
+     "summary_metrics": {
+       "average_CPS": 41.74,
+       "num_tasks": 12
+     },
+     "model_config": {
+       "identifier": "model-org/model-name",
+       "architecture": "ModelArchitecture",
+       "parameters": 8.03,
+       "evaluation_settings": {
+         "fewshot": 5,
+         "batch_size": 1
+       }
+     },
+     "task_results": {
+       "task-name": {
+         "average_score": 52.60,
+         "best_prompt": {
+           "id": "prompt-6",
+           "score": 66.57
+         },
+         "prompt_analysis": [
+           {
+             "prompt_id": "prompt-1",
+             "score": 37.20,
+             "stderr": 0.0216
+           }
+         ]
+       }
+     }
+   }
+"""
+
+import json
+import os
+import re
+import statistics
+
+TASKS ={"NER", "RE", "DIA", "RML", "HIS" }
+
+def safe_float(value):
+    """Safely converts a value to float, returning None if the conversion fails."""
+    try:
+        return float(value)
+    except ValueError:
+        return None
+
+
+def calculate_task_metrics(task_info):
+    """Calculates average accuracy, best prompt accuracy, and CPS for a given task."""
+    accuracies = [prompt['value'] for prompt in task_info['prompts'] if prompt['value'] is not None]
+
+    if not accuracies:
+        return None
+
+    task_info['average_accuracy'] = sum(accuracies) / len(accuracies)
+    task_info['std_accuracy'] = statistics.stdev(accuracies) if len(accuracies) > 1 else 0.0
+    best_prompt_data = max(task_info['prompts'], key=lambda x: x['value'])
+    task_info['best_prompt'] = best_prompt_data['value']
+    task_info['prompt_id'] = best_prompt_data['prompt']
+
+    # Calculate CPS
+    avg_acc = task_info['average_accuracy']
+    best_acc = task_info['best_prompt']
+    task_info['CPS'] = (1 - (best_acc - avg_acc) / 100) * best_acc
+
+
+def extract_data_from_file(file_path):
+    """Extracts task and prompt data from a specified file."""
+    
+
+    LANG=""
+    if file_path.find ("__en__")!=-1 : LANG="EN"
+    if file_path.find ("__sl__")!=-1 : LANG="SL"
+    if file_path.find ("__it__")!=-1 : LANG="IT"
+    if file_path.find ("__gr__")!=-1 : LANG="GR"
+    if file_path.find ("__sk__")!=-1 : LANG="SK"
+    if file_path.find ("__pl__")!=-1 : LANG="PL"
+    if LANG=="" : 
+      print ("ERROR: ",file_path) 
+      
+    with open(file_path, 'r') as file:
+        lines = file.readlines()
+
+    tasks_data = {}
+    
+    current_task = None
+
+    for line in lines:
+        line = line.strip()
+
+        # Skips empty lines
+        if not line:
+            continue
+
+        # Skips header lines
+        if line.startswith("|         Tasks")  or line.startswith("   | Task"):
+            continue
+
+        # Extracts model configuration details
+        if line.startswith("hf (pretrained=") or line.startswith("hf(pretrained="):
+            start = line.find("pretrained=") + len("pretrained=")
+            end = line.find(" )", start)
+            pretrained_model = line[start:end]
+
+            num_fewshot_match = re.search(r"num_fewshot:\s*([\w\d]+)", line)
+            num_fewshot = num_fewshot_match.group(1) if num_fewshot_match else None
+
+            batch_size_match = re.search(r"batch_size:\s*(\d+)", line)
+            batch_size = int(batch_size_match.group(1)) if batch_size_match else None
+
+            continue
+
+        columns = line.split('|')
+        if len(columns) != 11:
+            continue
+        print (columns)
+        task_name = columns[1]
+        metric = columns[5].strip()
+        value = safe_float(columns[7])
+        stderr = safe_float(columns[9])
+        print (value)
+        # Skips normalized accuracy metrics
+        if metric == "acc_norm":
+            continue
+
+        # Identifies task and prompt sections in the file
+        if task_name.startswith(" - "):
+            task_name = task_name[3:].strip()
+            current_task = task_name
+            tasks_data.setdefault(current_task,
+                                  {'prompts': [], 'average_accuracy': 0, 'best_prompt': None, 'prompt_id': None,
+                                   'CPS': None, 'is_dummy': False  })
+
+        elif task_name.startswith("   - ") and current_task:
+            prompt_name = task_name[4:].strip()
+            prompt_data = {'prompt': prompt_name, 'metric': metric, 'value': value * 100,
+                           'stderr': stderr}
+            tasks_data[current_task]['prompts'].append(prompt_data)
+
+    # Special handling for evalita NER task to calculate weighted prompt averages
+    if "evalita NER" in tasks_data:
+        task_info = tasks_data["evalita NER"]
+        weight_map = {"ADG prompt-1": 521, "ADG prompt-2": 521, "FIC prompt-1": 1517, "FIC prompt-2": 1517,
+                      "WN prompt-1": 2088, "WN prompt-2": 2088}
+
+        weighted_values = {"prompt-1": 0, "prompt-2": 0}
+        total_weights = sum(weight_map.values())
+
+        for prompt in task_info['prompts']:
+            if prompt['prompt'] in weight_map:
+                if "prompt-1" in prompt['prompt']:
+                    weighted_values["prompt-1"] += weight_map[prompt['prompt']] * prompt['value']
+                elif "prompt-2" in prompt['prompt']:
+                    weighted_values["prompt-2"] += weight_map[prompt['prompt']] * prompt['value']
+
+        task_info['prompts'] = [
+            {"prompt": "prompt-1", "metric": "acc", "value": weighted_values["prompt-1"] / total_weights,
+             'stderr': None},
+            {"prompt": "prompt-2", "metric": "acc", "value": weighted_values["prompt-2"] / total_weights,
+             'stderr': None}]
+
+    # Inject dummy entries for any TASKS not present in the file
+    for must_have in TASKS:
+      if must_have not in tasks_data:
+        tasks_data[must_have] = {
+            'prompts': [
+                {'prompt': 'p1', 'metric': 'f1', 'value': 0.0, 'stderr': 0.0}
+            ],
+            'average_accuracy': None,
+            'std_accuracy': None,
+            'best_prompt': None,
+            'prompt_id': None,
+            'CPS': None,
+            'is_dummy': True  # <--- mark as dummy
+        }
+
+    # Calculates task metrics for each task
+    for task_info in tasks_data.values():
+        if task_info.get('is_dummy'):
+          # leave metrics as None for dummy tasks
+          continue
+        calculate_task_metrics(task_info)
+
+    # Calculates the average CPS across all tasks
+    tasks_with_cps = [
+      t['CPS'] for t in tasks_data.values() if not t.get('is_dummy') and t.get('CPS') is not None ]
+    average_CPS = (sum(tasks_with_cps) / len(tasks_with_cps)) if tasks_with_cps else 0
+
+    #tasks_with_cps = [task['CPS'] for task in tasks_data.values() if task['CPS'] is not None]
+    #average_CPS = sum(tasks_with_cps) / len(tasks_with_cps) if tasks_with_cps else 0
+
+    config = {
+        "model_name": pretrained_model,
+        "num_fewshot": num_fewshot,
+        "batch_size": batch_size,
+        "LANG": LANG
+    }
+
+    return {'average_CPS': average_CPS, 'config': config, 'tasks': tasks_data}
+
+
+"""
+MAIN PROCESSING PIPELINE
+
+This script executes the complete evaluation data processing workflow:
+
+1. Input Sources:
+   - Raw evaluation results (.out files) from: ../evalita_llm_models_output/
+   - Model metadata JSON files from: ../evalita_llm_requests/
+
+2. Processing Steps:
+   - Parses evaluation metrics from .out files
+   - Combines with model metadata
+   - Calculates aggregated performance statistics
+
+3. Output:
+   - Structured JSON results saved to: ../evalita_llm_results/
+   - Organized by model organization/name
+   - Contains complete evaluation results with metadata
+"""
+directory_in_path = '/home/sfarzi/leaderboard/trail_leaderboard/csv_new/output/'
+directory_in_requests_path = '/home/sfarzi/leaderboard/trail_leaderboard/e3c_llm_requests/'
+directory_out_results_path = '/home/sfarzi/leaderboard/trail_leaderboard/e3c_llm_results/'
+
+for filename in os.listdir(directory_in_path):
+    if filename.endswith('.txt'):
+        file_path = os.path.join(directory_in_path, filename)
+        json_output = extract_data_from_file(file_path)
+
+        model_org_name, model_name = json_output['config']['model_name'].split('/')
+
+         
+        config_file_path = os.path.join(directory_in_requests_path, model_org_name, f"{model_name}.json")
+
+        if os.path.exists(config_file_path):
+            with open(config_file_path, 'r', encoding='utf-8') as config_file:
+                additional_config = json.load(config_file)
+            json_output['config'].update(additional_config)
+
+
+        org_folder_path = os.path.join(directory_out_results_path, model_org_name)
+        os.makedirs(org_folder_path, exist_ok=True)
+
+        file_suffix = f"{json_output['config']['num_fewshot']}" +"_"+ f"{json_output['config']['LANG']}"
+        output_file_path = os.path.join(org_folder_path, f"{model_name}_{file_suffix}.json")
+
+        with open(output_file_path, 'w', newline="\n") as outfile:
+            json.dump(json_output, outfile, indent=4)
+
+        print(f"File {filename} processed and saved to {output_file_path}")

app.py

@@ -52,7 +52,7 @@ def create_best_model_comparison_table(dataframe, lang: str | None = None, shot:
       - lang in {EN, IT, SL, SK, GR, PL} or None/"All"
       - shot in {"0","10"} or None/"All" (mapped to IS_FS False/True)
     """
-    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+    tasks = ["NER", "REL", "RML-CRF", "HIS-CRF", "DIA-CRF", "NER-PHARMAER"]
     df = dataframe.copy()
     
     if lang and lang != "All" and "LANG" in df.columns:
@@ -141,7 +141,7 @@ def create_best_model_comparison_table_without_lang(dataframe):
     Table with the best overall model per task (NER, REL,) and the model that
     achieves the best score with its own best prompt.
     """
-    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+    tasks = ["NER", "REL", "RML-CRF", "HIS-CRF", "DIA-CRF", "NER-PHARMAER"]
     table_data = {'Task': [], 'Best Overall Model': [], 'CPS': [], 'Best Prompt Model': [], 'Acc.': []}
 
     for task in tasks:
@@ -216,7 +216,7 @@ def create_prompt_heatmap(dataframe, lang: str | None = None, shot: str | None =
       - lang: None or one of EN/IT/SL/SK/GR/PL (None means All)
       - shot: None or "0"/"10" (None means All) mapped to IS_FS False/True
     """
-    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+    tasks = ["NER", "REL", "RML-CRF", "HIS-CRF", "DIA-CRF", "NER-PHARMAER"]
 
     df = dataframe.copy()
     # Language filter
@@ -236,6 +236,8 @@ def create_prompt_heatmap(dataframe, lang: str | None = None, shot: str | None =
     for task in tasks:
         col = f"{task} Best Prompt Id"
         if col in df.columns:
+            #print (col)
+            #print(df[col])
             all_ids.update(df[col].dropna().unique())
     prompt_ids_raw = sorted(list(all_ids), key=lambda x: int(re.sub(r'[^0-9]', '', str(x)) or 0))
     prompt_ids_raw = [pid for pid in prompt_ids_raw if label_for(pid) in {"p1", "p2", "p3"}] or [1, 2, 3]
@@ -294,7 +296,7 @@ def create_prompt_heatmap_without_lang(dataframe):
     for tasks NER and REL, with exactly 3 prompts (p1, p2, p3). It supports columns storing
     ids as integers (1/2/3) or strings ('p1'/'p2'/'p3').
     """
-    tasks = ["NER", "REL", "RML", "HIS", "DIA"]
+    tasks = ["NER", "REL", "RML-CRF", "HIS-CRF", "DIA-CRF", "NER-PHARMAER"]
 
     # Collect unique prompt ids as they appear (int or 'pX'); restrict to 3 prompts
     all_ids = set()
@@ -395,10 +397,11 @@ def mean_of_max_per_field(df):
         float: media dei valori massimi dei campi
     """
     #fields = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
-    fields = ["NER", "REL", "RML", "DIA", "HIS"]
+    fields = ["NER", "REL", "RML-CRF", "DIA-CRF", "HIS-CRF", "NER-PHARMAER"]
     #print(df.columns)
 
     # Controlla che tutte le colonne esistano nel DataFrame
+    print(df.columns)
     missing = [f for f in fields if f not in df.columns]
     if missing:
         raise ValueError(f"Le seguenti colonne mancano nel DataFrame: {missing}")
@@ -414,7 +417,7 @@ def mean_of_max_per_field(df):
 
 def barplot_mean_few_minus_zero_shot(dataframe, tasks=None):
     if tasks is None:
-        tasks = [ "NER", "REL", "RML", "DIA", "HIS"]
+        tasks = [ "NER", "REL", "RML-CRF", "DIA-CRF", "HIS-CRF","NER-PHARMAER"]
  
     task_means = {}
 
@@ -489,7 +492,7 @@ def boxplot_per_task(dataframe=None, baselines=None, references=None):
     #print(dataframe.columns)
 
     #tasks = ["TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL"]
-    tasks =["NER", "REL", "RML", "HIS", "DIA"]
+    tasks =["NER", "REL", "RML-CRF", "HIS-CRF", "DIA-CRF" , "NER-PHARMAER"]
     if dataframe is None:
         np.random.seed(42)
         dataframe = pd.DataFrame({
@@ -810,9 +813,11 @@ TASK_METADATA_GENERATIVE = {
     
     "NER": {"icon": "🏷️", "name": "Named Entity Recognition", "tooltip": ""},
     "REL": {"icon": "🔗", "name": "Relation Extraction", "tooltip": ""},
-    "RML": {"icon": "😃", "name": "CRF RML", "tooltip": "CRF RML"},
-    "DIA": {"icon": "🏥", "name": "CRF Diagnosis", "tooltip": "CRF Diagnosis"},
-    "HIS": {"icon": "📝", "name": "CRF History", "tooltip": "CRF History"},
+    "RML-CRF": {"icon": "😃", "name": "CRF RML", "tooltip": "CRF RML"},
+    "DIA-CRF": {"icon": "🏥", "name": "CRF Diagnosis", "tooltip": "CRF Diagnosis"},
+    "HIS-CRF": {"icon": "📝", "name": "CRF History", "tooltip": "CRF History"},
+    "NER-PHARMAER": {"icon": "🏷️", "name": "Named Entity Recognition over PharmaER.It Datasets", "tooltip": ""},
+
 }
 
 def restart_space():
@@ -1075,8 +1080,8 @@ with demo:
 
             leaderboard = init_leaderboard(
                 LEADERBOARD_DF,
-                default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "RML", "DIA", "HIS"],
-                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "RML", "DIA", "HIS"]]
+                default_selection=['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "RML-CRF", "DIA-CRF", "HIS-CRF", "NER-PHARMAER"],
+                hidden_columns=[col for col in LEADERBOARD_DF.columns if col not in ['Rank', 'Size', 'LANG', 'FS', 'Model', "Avg. Comb. Perf. ⬆️", "TE", "SA", "HS", "AT", "WIC", "FAQ", "LS", "SU", "NER", "REL", "RML-CRF", "DIA-CRF", "HIS-CRF", "NER-PHARMAER"]]
             )
 
 
@@ -1107,7 +1112,7 @@ with demo:
             with gr.TabItem(f"{metadata['icon']}{task}"):
                 task_description = TASK_DESCRIPTIONS.get(task, "Description not available.")
                 gr.Markdown(task_description, elem_classes="markdown-text1")
-                #print (LEADERBOARD_DF)
+                print (task)
                 leaderboard = update_task_leaderboard(
                     LEADERBOARD_DF.rename(columns={f"{task} Prompt Average": "Prompt Average",
                                                    f"{task} Prompt Std": "Prompt Std",

csv_new2/output/.ipynb_checkpoints/deepseek-ai__DeepSeek-R1-Distill-Qwen-32B__en__0shot-checkpoint.txt

@@ -0,0 +1,23 @@
+hf (pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-32B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.2877 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1963 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3459 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3208 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4430 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4487 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4492 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.4311 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/.ipynb_checkpoints/deepseek-ai__DeepSeek-R1-Distill-Qwen-32B__en__10shot-checkpoint.txt

@@ -0,0 +1,23 @@
+hf (pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-32B ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.5963 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6024 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5929 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.5935 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.5221 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5191 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5199 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.5273 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.1768 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1169 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1503 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.2633 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.2339 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3117 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.2416 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1483 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.4828 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3922 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5191 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.5371 |   | 0 |

csv_new2/output/.ipynb_checkpoints/deepseek-ai__DeepSeek-R1-Distill-Qwen-32B__it__0shot-checkpoint.txt

@@ -0,0 +1,27 @@
+hf (pretrained=deepseek-ai/DeepSeek-R1-Distill-Qwen-32B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.3220 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.2678 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3568 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3414 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4452 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4519 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4611 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.4227 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - NER-PHARMAER        |       |      |      |f1    |   | 0.4608 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5012 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4493 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.4319 |   | 0 |

csv_new2/output/.ipynb_checkpoints/epfl-llm__meditron-7b__en__0shot-checkpoint.txt

@@ -0,0 +1,23 @@
+hf (pretrained=epfl-llm/meditron-7b ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0612 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0578 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0410 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0848 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0313 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0442 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0497 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__en__0shot.txt

@@ -0,0 +1,23 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0918 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0629 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1041 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1083 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.2604 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1287 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3394 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3131 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__en__10shot.txt

@@ -0,0 +1,23 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.2142 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.2189 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.2243 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1994 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.1681 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1189 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1668 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.2185 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.1779 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1825 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1612 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1900 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.1500 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.2415 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1416 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0668 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0147 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0178 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0068 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0194 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__gr__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0611 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0620 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0592 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0620 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0863 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1017 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0506 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1065 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__gr__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1474 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1667 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1089 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1667 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0970 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0821 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1053 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1036 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__it__0shot.txt

@@ -0,0 +1,27 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0416 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0435 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0429 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0384 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.1413 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0672 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.2266 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1300 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - NER-PHARMAER        |       |      |      |f1    |   | 0.1786 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1702 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.2001 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1657 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__it__10shot.txt

@@ -0,0 +1,27 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.3753 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3299 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4023 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3938 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.1331 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0977 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1226 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1789 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.1044 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0821 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1119 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1190 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0007 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0010 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0002 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0008 |   | 0 |
+| - NER-PHARMAER        |       |      |      |f1    |   | 0.3769 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3616 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3881 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3812 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__pl__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0379 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0379 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0378 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0379 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0891 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0602 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1293 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0778 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__pl__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.3966 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3992 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3916 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3992 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.1003 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0998 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1055 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0956 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__sk__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0385 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0387 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0380 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0387 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0174 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0121 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0280 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0121 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__sk__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.3507 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3444 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3632 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3444 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0884 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0734 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1045 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0875 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__sl__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0438 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0429 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0456 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0429 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.1278 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0967 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1900 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0967 |   | 0 |

csv_new2/output/Henrychur__MMed-Llama-3-8B__sl__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Henrychur/MMed-Llama-3-8B ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.3720 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3558 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4045 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3558 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0762 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0787 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0781 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0719 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__en__0shot.txt

@@ -0,0 +1,23 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0578 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0940 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0331 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0464 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__en__10shot.txt

@@ -0,0 +1,23 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1317 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1215 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1415 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1322 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0031 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0028 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0016 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0049 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__gr__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0769 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0859 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0591 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0859 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__gr__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1448 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1455 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1434 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1455 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0010 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0024 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0007 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__it__0shot.txt

@@ -0,0 +1,26 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0812 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0770 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0920 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0747 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - NER-PHARMAER        |       |      |      |f1    |   | 0.0429 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0762 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0270 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0255 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__it__10shot.txt

@@ -0,0 +1,27 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1694 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1616 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1774 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1690 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0048 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0035 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0064 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0046 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - NER-PHARMAER        |       |      |      |f1    |   | 0.1917 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1647 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.2071 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.2032 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__pl__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0308 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0244 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0436 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0244 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__pl__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1516 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1500 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1548 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1500 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0032 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0040 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0023 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0034 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__sk__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0712 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0880 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0375 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0880 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__sk__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1444 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1485 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1360 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1485 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0027 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0038 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0024 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0020 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__sl__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0711 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0777 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0579 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0777 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/HiTZ__Medical-mT5-large__sl__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=HiTZ/Medical-mT5-large ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1422 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1470 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1325 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1470 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.0080 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0073 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0074 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0093 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__en__0shot.txt

@@ -0,0 +1,23 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.2500 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3425 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1181 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.2893 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4075 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4135 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3917 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.4172 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0001 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0002 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__en__10shot.txt

@@ -0,0 +1,23 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.5993 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6091 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5646 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6243 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.6164 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6332 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.6025 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6133 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.2843 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.2129 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3222 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3178 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.1658 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3073 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1137 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0764 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.2370 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1244 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4429 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1437 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__gr__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.1290 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1339 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1191 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1339 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.3957 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3796 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4266 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3810 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__gr__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.6028 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6119 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5847 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6119 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.6056 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5962 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.6024 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6183 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__it__0shot.txt

@@ -0,0 +1,27 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.2137 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.2467 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1709 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.2234 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4016 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4173 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3770 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.4106 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0002 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0007 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - NER-PHARMAER        |       |      |      |f1    |   | 0.1067 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0844 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0724 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1635 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__it__10shot.txt

@@ -0,0 +1,27 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.6569 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6719 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.6327 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6661 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.5952 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5767 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5998 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6093 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.1557 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1111 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1599 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1960 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.2496 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4407 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1328 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1753 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.2339 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0817 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5103 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1096 |   | 0 |
+| - NER-PHARMAER        |       |      |      |f1    |   | 0.6282 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6312 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.6199 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6337 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__pl__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0586 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0697 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0364 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0697 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4022 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3803 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4464 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3800 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__pl__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.6092 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6226 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5824 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6226 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.5944 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5991 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5466 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6375 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__sk__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.0955 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1220 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0426 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.1220 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4116 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4027 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4294 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.4027 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__sk__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.6419 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6386 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.6486 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6386 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.5899 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5894 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5845 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.5959 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__sl__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.3398 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3910 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.2375 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3910 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.3777 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3775 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3783 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3775 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-14B-Instruct-1M__sl__10shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-14B-Instruct-1M ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.6371 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6467 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.6178 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6467 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.5837 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5949 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5782 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.5781 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-32B-Instruct__en__0shot.txt

@@ -0,0 +1,25 @@
+hf (pretrained=Qwen/Qwen2.5-32B-Instruct ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.3279 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.3804 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.3068 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.2964 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4658 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4734 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4649 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.4591 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0015 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0005 |   | 0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0057 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0002 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0006 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0000 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0000 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-32B-Instruct__en__10shot.txt

@@ -0,0 +1,24 @@
+hf (pretrained=Qwen/Qwen2.5-32B-Instruct ), num_fewshot: 10, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.5895 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5970 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.5602 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6113 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.6440 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.6482 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.6469 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.6370 |   | 0 |
+| - RML-CRF        |       |      |      |f1    |   | 0.0931 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.1501 |   | 0 |
+|   - p1  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1383 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0839 |   | 0 |
+| - DIA-CRF        |       |      |      |f1    |   | 0.0286 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0311 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.0000 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0546 |   | 0 |
+| - HIS-CRF        |       |      |      |f1    |   | 0.0659 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.0247 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1557 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.0174 |   | 0 |

csv_new2/output/Qwen__Qwen2.5-32B-Instruct__gr__0shot.txt

@@ -0,0 +1,11 @@
+hf (pretrained=Qwen/Qwen2.5-32B-Instruct ), num_fewshot: 0, batch_size: 1
+|Tasks  |Version|Filter|n-shot|Metric|    |Value |   |Stderr|
+|-------|-------|------|------|------|----|------|---|------|
+| - NER        |       |      |      |f1    |   | 0.4506 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.5976 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.1568 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.5976 |   | 0 |
+| - RE        |       |      |      |f1    |   | 0.4104 |   |0 |
+|   - p1  |       |      |      |f1    |   | 0.4393 |   | 0 |
+|   - p2  |       |      |      |f1    |   | 0.4083 |   | 0 |
+|   - p3  |       |      |      |f1    |   | 0.3834 |   | 0 |