a

app.py

@@ -20,7 +20,7 @@ from src.display.utils import (
     ModelType,
     fields,
 )
-from src.envs import API, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
+from src.envs import API, EVAL_RESULTS_PATH, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_leaderboard_df
 from src.submission.submit import add_new_eval
 

requirements.txt

@@ -9,8 +9,4 @@ huggingface-hub>=0.18.0
 matplotlib
 numpy
 pandas
-python-dateutil
-tqdm
-transformers
-tokenizers>=0.15.0
-sentencepiece
+python-dateutil

src/display/utils.py

@@ -1,8 +1,6 @@
-from dataclasses import dataclass, make_dataclass
+from dataclasses import dataclass, field, make_dataclass
 from enum import Enum
 
-import pandas as pd
-
 from src.about import Tasks
 
 def fields(raw_class):
@@ -19,15 +17,33 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
 auto_eval_column_dict = []
-auto_eval_column_dict.append(["model_name", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
+# model_name
+auto_eval_column_dict.append((
+    "model_name", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("Model", "markdown", True, never_hidden=True))
+))
 # Scores
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append((
+        task.name, 
+        ColumnContent, 
+        field(default_factory=lambda: ColumnContent(task.value.col_name, "number", True))
+    ))
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
+auto_eval_column_dict.append((
+    "model_type", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("Type", "str", False))
+))
+auto_eval_column_dict.append((
+    "params", 
+    ColumnContent, 
+    field(default_factory=lambda: ColumnContent("#Params (B)", "number", False))
+))
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)

src/leaderboard/read_evals.py

@@ -63,8 +63,8 @@ class EvalResult:
         """Converts the Eval Result to a dict compatible with our dataframe display"""
         data_dict = {
             "Model": self.model_name,
-            AutoEvalColumn.model_type.name: self.model_type.value.name,
-            AutoEvalColumn.params.name: self.num_params
+            "Type": self.model_type.value.name,
+            "#Params (B)": self.num_params
         }
 
         for task in Tasks:

src/populate.py

@@ -1,8 +1,6 @@
-import json
-import os
-
 import pandas as pd
 
+from src.about import Tasks
 from src.display.formatting import has_no_nan_values
 from src.display.utils import AutoEvalColumn
 from src.leaderboard.read_evals import get_raw_eval_results
@@ -14,10 +12,13 @@ def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> p
     all_data_json = [v.to_dict() for v in raw_data]
 
     df = pd.DataFrame.from_records(all_data_json)
-    df = df.sort_values(by=[AutoEvalColumn.task0.name], ascending=True)
+    # print(df)
+    # df = df.sort_values(by=[df.columns.values[3]], ascending=True)
 
-    df = df[cols].round(decimals=2)
+    # df = df[cols].round(decimals=2)
+    # print(cols)
+    # print(df)
 
     # filter out if any of the benchmarks have not been produced
-    df = df[has_no_nan_values(df, benchmark_cols)]
+    # df = df[has_no_nan_values(df, benchmark_cols)]
     return df