Add ability to incorporate external submissions (#7)

app.py

@@ -1,10 +1,13 @@
+import logging
 import os
 import re
+import sys
 
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
 from datasets import load_dataset
+from datasets.data_files import EmptyDatasetError
 from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
 from huggingface_hub import HfApi
 
@@ -13,6 +16,13 @@ from src.display.css_html_js import custom_css
 from src.plots import plot_cost_efficiency, plot_parameter_efficiency
 from src.schema import AutoEvalColumn, EvalResult, fields
 
+logging.basicConfig(
+    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    handlers=[logging.StreamHandler(sys.stdout)],
+    level=logging.INFO,
+)
+
 # 1. Initialization
 _hf_token = os.environ.get("HF_TOKEN")
 if not _hf_token:
@@ -22,6 +32,7 @@ api = HfApi(token=_hf_token)
 
 REPO_ID = "UD-Filipino/filbench-leaderboard"
 REPO_RESULTS = "UD-Filipino/filbench-results"
+SUBMISSION_RESULTS = "UD-Filipino/filbench-results-submission"
 
 
 def restart_space():
@@ -29,11 +40,44 @@ def restart_space():
 
 
 # 2. Load and populate leaderboard data
-def get_results(source: str, aggregate: bool = False) -> tuple[pd.DataFrame, list]:
+def get_results(
+    source: str, aggregate: bool = False, submissions: str = None
+) -> tuple[pd.DataFrame, list]:
+    """Load results from a given source and return a DataFrame with the relevant columns.
+
+    If `aggregate` is True, it returns the aggregated results.
+
+    source (str): The source dataset to load results from.
+    aggregate (bool): Whether to return aggregated results or not.
+    submissions (str, optional): The submissions dataset to load results from.
+    RETURNS (tuple[pd.DataFrame, list]): A tuple containing the DataFrame with results and a list of master columns.
+    """
     results = load_dataset(source, split="train").to_pandas().to_dict(orient="records")
     raw_data = [EvalResult.init_from_dict(result) for result in results]
-    all_data_json = [v.to_dict() for v in raw_data]
 
+    if submissions:
+        try:
+            submission_results = (
+                load_dataset(
+                    submissions, split="train", download_mode="force_redownload"
+                )
+                .to_pandas()
+                .to_dict(orient="records")
+            )
+        except EmptyDatasetError:
+            logging.info("Empty dataset for submissions, skipping...")
+            submission_results = []
+        if len(submission_results) == 0:
+            logging.info("No external submissions found!")
+        else:
+            logging.info(f"Found {len(submission_results)} submission/s!")
+
+        raw_data += [
+            EvalResult.init_from_dict(result, is_submission=True)
+            for result in submission_results
+        ]
+
+    all_data_json = [v.to_dict() for v in raw_data]
     df = pd.DataFrame.from_records(all_data_json)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
     df["Incomplete"] = ~df.isna().any(axis=1)
@@ -58,8 +102,12 @@ def get_results(source: str, aggregate: bool = False) -> tuple[pd.DataFrame, lis
     return df, master_columns
 
 
-def init_leaderboard(source: str, aggregate: bool = False) -> Leaderboard:
-    df, master_columns = get_results(source=source, aggregate=aggregate)
+def init_leaderboard(
+    source: str, aggregate: bool = False, submissions: str = None
+) -> Leaderboard:
+    df, master_columns = get_results(
+        source=source, aggregate=aggregate, submissions=submissions
+    )
 
     return Leaderboard(
         value=df,
@@ -80,6 +128,7 @@ def init_leaderboard(source: str, aggregate: bool = False) -> Leaderboard:
         filter_columns=[
             # fmt: off
             ColumnFilter("Incomplete", type="boolean", label="Hide incomplete evaluations", default=True),
+            ColumnFilter("Submission", type="boolean", label="Show only submitted results", default=False),
             # ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
             ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model type"),
             ColumnFilter(AutoEvalColumn.multilingual.name, type="checkboxgroup", label="Multilinguality"),
@@ -97,8 +146,12 @@ def init_leaderboard(source: str, aggregate: bool = False) -> Leaderboard:
 
 
 def get_clean_df() -> pd.DataFrame:
-    df, _ = get_results(source=REPO_RESULTS, aggregate=False)
-    df_agg, _ = get_results(source=REPO_RESULTS, aggregate=True)
+    df, _ = get_results(
+        source=REPO_RESULTS, aggregate=False, submissions=SUBMISSION_RESULTS
+    )
+    df_agg, _ = get_results(
+        source=REPO_RESULTS, aggregate=True, submissions=SUBMISSION_RESULTS
+    )
 
     # Cleanup
     def extract_names(html_string):
@@ -165,19 +218,25 @@ def download_results():
 demo = gr.Blocks(css=custom_css)
 with demo:
     with gr.Column(scale=6):
-        num_models = len(get_results(REPO_RESULTS, aggregate=True)[0])
+        num_models = len(
+            get_results(REPO_RESULTS, aggregate=True, submissions=SUBMISSION_RESULTS)[0]
+        )
         gr.Markdown(about.TOP_TEXT.format(str(num_models)))
 
     with gr.Tabs(elem_classes="tab-buttons") as tabs:
         with gr.TabItem(
             "🏅 FilBench Leaderboard", elem_id="llm-benchmark-tab-table", id=0
         ):
-            leaderboard = init_leaderboard(REPO_RESULTS, aggregate=True)
+            leaderboard = init_leaderboard(
+                REPO_RESULTS, aggregate=True, submissions=SUBMISSION_RESULTS
+            )
 
         with gr.TabItem(
             "🔍 FilBench - Detailed", elem_id="llm-benchmark-tab-table", id=1
         ):
-            leaderboard = init_leaderboard(REPO_RESULTS, aggregate=False)
+            leaderboard = init_leaderboard(
+                REPO_RESULTS, aggregate=False, submissions=SUBMISSION_RESULTS
+            )
 
         with gr.TabItem("📊 Analysis", id=2):
             df = get_clean_df()

src/about.py

@@ -11,7 +11,8 @@ current_time = datetime.now(pacific_tz).strftime("%H:%M %Z, %d %b %Y")
 TOP_TEXT = f"""
 # FilBench: An Open LLM Leaderboard for Filipino
 
-[Code](https://github.com/filbench/filbench) | [Runner](https://github.com/filbench/lighteval) | [Paper (<i>Coming soon!</i>)]() | Total Models: {{}} | Last restart (PHT): {current_time}
+[Code](https://github.com/filbench/filbench-eval) | [Paper (<i>Coming soon!</i>)]() | Total Models: {{}} | Last restart (PHT): {current_time}
+📥: Indicates model submissions from the community. If you wish to submit your model evaluations, then please check our instructions on [GitHub](https://github.com/filbench/filbench-eval).
 """
 
 # Leaderboard reproducibility
@@ -27,7 +28,6 @@ We average four core sections (weighted by the number of instances):
 3. **Reading Comprehension:** Contains more focused natural language understanding (NLU) tasks and questions from readability benchmarks.
 4. **Generation:** Contains instances for natural language generation (NLG), more focused on translation.
 
-
 ## Evaluation Runner
 
 We use our own fork of [lighteval](https://github.com/filbench/lighteval) to perform evaluations.
@@ -35,30 +35,17 @@ We highly recommend using the vLLM backend for faster inference.
 Sequentially, evaluating on FilBench can take 4.93 hours on 2 NVIDIA H100 GPUs.
 However, the evaluation suite can be parallelized per benchmark, where the longest-running task can take approximately 1 hour and 28 minutes, and the shortest task takes only 5.86 minutes.
 
-To evaluate your model on FilBench and for it to appear in the leaderboard, please follow these steps:
-
-1. First clone the FilBench's lighteval repository and install all dependencies:
-
-```sh
-git clone https://github.com/filbench/lighteval.git
-python3 -m venv venv
-pip install -e .[dev,vllm]
-```
-
-2. Run the evaluation runner via vLLM
+To evaluate your model on FilBench and for it to appear in the leaderboard, please follow the steps in our [Github repository](https://github.com/filbench/filbench-eval).
 
-```sh
-python3 -m lighteval vllm ${MODEL_NAME} ${TASK_NAME} \\
-    --push-to-hub \\
-    --results-org UD-Filipino \\
-    --custom-tasks community_tasks/filbench_evals.py
-```
+## Contact
 
-You can find the list of all FilBench tasks [in this file](https://github.com/filbench/lighteval/blob/main/examples/tasks/all_filbench_tasks.txt).
+This work was done by Lj V. Miranda ([@ljvmiranda921](https://github.com/ljvmiranda921)), Elyanah Aco ([@elyanah-aco](https://github.com/elyanah-aco)), Conner Manuel ([@connermanuel](https://github.com/connermanuel)), Blaise Cruz ([@jcblaisecruz02](https://github.com/jcblaisecruz02)), and Joseph Imperial ([@imperialite](https://github.com/imperialite)).
+For any questions, please reach out to us via filbench-eval@googlegroups.com or through our [GitHub Issues](https://github.com/filbench/filbench-eval/issues).
 
 ## Acknowledgements
 
-The authors would like to thank Cohere Labs for the Cohere Research Grant that includes credits for running the Command models.
+We would like to thank [Cohere Labs](https://cohere.com/research) for providing credits through the [Cohere Research Grant](https://cohere.com/research/grants) to run the Aya model series, and [Together AI](https://together.ai) for additional computational credits for running several open models.
+We also acknowledge the Hugging Face team, particularly the OpenEvals team (Clémentine Fourrier [@clefourrier](https://github.com/clefourrier) and Nathan Habib [@NathanHB](https://github.com/NathanHB)) and Daniel van Strien [@davanstrien](https://github.com/davanstrien), for their support in publishing the FilBench blog post.
 """
 
 # Citation information

src/schema.py

@@ -1,9 +1,18 @@
+import logging
+import sys
 from dataclasses import dataclass, make_dataclass
 from enum import Enum
 
 import numpy as np
 
-from src.display.formatting import make_clickable_model
+from src.display.formatting import make_clickable_model, model_hyperlink
+
+logging.basicConfig(
+    format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+    handlers=[logging.StreamHandler(sys.stdout)],
+    level=logging.INFO,
+)
 
 
 def fields(raw_class):
@@ -147,6 +156,8 @@ auto_eval_cols = [
     ["param_size", ColumnContent, ColumnContent("# Parameters", "number", False, meta=True)],
     ["multilingual", ColumnContent, ColumnContent("Multilingual", "markdown", False, meta=True)],
     ["model_type", ColumnContent, ColumnContent("Model Type", "markdown", False, meta=True)],
+    ["is_submission", ColumnContent, ColumnContent("Submission", "boolean", False, meta=True)],
+    ["submission_date", ColumnContent, ColumnContent("Submission Date", "str", False, meta=True)],
     # fmt: on
 ]
 for task in Tasks:
@@ -199,16 +210,28 @@ class EvalResult:
     average: float
     aggregate_results: dict
     precision: Precision = Precision.Unknown
+    # Submission metadata
+    is_submission: bool = False
+    param_size: float = -1
+    model_type: str = ModelType.UNKNOWN.value
+    multilingual: str = Multilingual.UNKNOWN.value
+    submission_date: str = ""
+    model_url: str = "https://huggingface.co/spaces/UD-Filipino/filbench-leaderboard"
 
     @classmethod
-    def init_from_dict(self, data: dict):
+    def init_from_dict(self, data: dict, is_submission: bool = False) -> "EvalResult":
         """Populate results from a dictionary"""
 
-        config = data.get("config")
-
-        # Get model details
+        # For model details, use user-provided metadata if it's a submission
+        config_key = "display_metadata" if is_submission else "config"
+        config = data.get(config_key)
         precision = Precision.from_str(config.get("model_dtype"))
-        org_and_model = config.get("model_name", config.get("model_args", None))
+
+        org_and_model = (
+            config.get("hf_id")
+            if is_submission
+            else config.get("model_name", config.get("model_args", None))
+        )
         org_and_model = org_and_model.split("/", 1)
 
         if len(org_and_model) == 1:
@@ -221,35 +244,105 @@ class EvalResult:
             result_key = f"{org}_{model}_{precision.value.name}"
         full_model = "/".join(org_and_model)
 
+        results = EvalResult.compute_scores_per_benchmark(data.get("results"))
+        aggregate_results = EvalResult.compute_aggregate_results(results)
+        filbench_score = np.mean(list(aggregate_results.values()))
+
         # Format all results
-        scores = data.get("results")
-        results = {}
+        if is_submission:
+            # Use pre-computed scores and check if they match our computed scores
+            category_scores = data.get("category_scores")
+            aggregate_results_precomputed = {
+                TaskCategory.CULTURAL_KNOWLEDGE.value: category_scores.get(
+                    "CULTURAL_KNOWLEDGE"
+                ),
+                TaskCategory.CLASSICAL_NLP.value: category_scores.get("CLASSICAL_NLP"),
+                TaskCategory.READING_COMPREHENSION.value: category_scores.get(
+                    "READING_COMPREHENSION"
+                ),
+                TaskCategory.TRANSLATION.value: category_scores.get("GENERATION"),
+            }
+            is_similar = EvalResult.compare_category_scores(
+                precomputed=aggregate_results_precomputed,
+                computed=aggregate_results,
+            )
+            if not is_similar:
+                logging.warning("Precomputed and computed category scores differ.")
+                logging.info("Will use computed scores for display.")
+            else:
+                logging.info("Precomputed and computed category scores are similar.")
+                aggregate_results = aggregate_results_precomputed
+
+            # Do the same comparison for FilBench score
+            filbench_score_precomputed = data.get("filbench_score")
+            is_filbench_score_similar = (
+                abs(filbench_score_precomputed - filbench_score) < 1e-2
+            )
+            if not is_filbench_score_similar:
+                logging.warning(
+                    f"Precomputed filbench_score ({filbench_score_precomputed}) and"
+                    f" official FilBench score ({filbench_score}) differ."
+                )
+            average = (
+                filbench_score_precomputed
+                if is_filbench_score_similar
+                else filbench_score
+            )
+            display_metadata = data.get("display_metadata")
+
+            return EvalResult(
+                eval_name=result_key,
+                full_model=full_model,
+                org=org,
+                model=model,
+                precision=precision,
+                results=results,
+                aggregate_results=aggregate_results,
+                average=average,
+                # Display Metadata
+                is_submission=True,
+                submission_date=display_metadata.get("submission_date", ""),
+                param_size=display_metadata.get("num_params", -1),
+                model_type=display_metadata.get("model_type", ModelType.UNKNOWN.value),
+                multilingual=display_metadata.get(
+                    "multilinguality", Multilingual.UNKNOWN.value
+                ),
+                model_url=display_metadata.get(
+                    "url",
+                    "https://huggingface.co/spaces/UD-Filipino/filbench-leaderboard",
+                ),
+            )
+        else:
+            return self(
+                eval_name=result_key,
+                full_model=full_model,
+                org=org,
+                model=model,
+                precision=precision,
+                results=results,
+                aggregate_results=aggregate_results,
+                is_submission=False,
+                average=filbench_score,
+            )
+
+    @classmethod
+    def compute_scores_per_benchmark(cls, results: dict) -> dict[str, float]:
+        scores_per_benchmark = {}
         for task in Tasks:
             task = task.value
-            if scores.get(task.benchmark):
-                score = scores.get(task.benchmark).get(task.metric)
+            if results.get(task.benchmark):
+                score = results.get(task.benchmark).get(task.metric)
                 if "acc_" in task.metric:
                     score = score * 100.0
                 if "rougeL" in task.metric:
                     score = score * 100.0
-                results[task.benchmark] = score
+                scores_per_benchmark[task.benchmark] = score
             else:
-                results[task.benchmark] = None
-
-        # Compute weighted average
-        # weighted_total = 0
-        # for task in Tasks:
-        #     task = task.value
-        #     if results[task.benchmark]:
-        #         score = results[task.benchmark]
-        #     else:
-        #         score = 0
-        #     weighted_total += score * task.num_samples
-
-        # total = sum([task.value.num_samples for task in Tasks])
-        # average = weighted_total / total
-
-        # Compute weighted average for each category
+                scores_per_benchmark[task.benchmark] = None
+        return scores_per_benchmark
+
+    @classmethod
+    def compute_aggregate_results(cls, results: dict) -> dict[str, float]:
         aggregate_results = {}
         for task_category in TaskCategory:
             tasks = [
@@ -266,41 +359,59 @@ class EvalResult:
             aggregate_results[task_category.value] = (
                 weighted_total_category / total_category
             )
+        return aggregate_results
 
-        # Correct average computation
-        average = np.mean(list(aggregate_results.values()))
-
-        return self(
-            eval_name=result_key,
-            full_model=full_model,
-            org=org,
-            model=model,
-            precision=precision,
-            results=results,
-            aggregate_results=aggregate_results,
-            average=average,
-        )
+    @classmethod
+    def compare_category_scores(
+        cls, precomputed: dict, computed: dict, threshold: float = 1e-2
+    ) -> bool:
+        """Compares precomputed and computed category scores."""
+        is_similar = True
+        for key, precomputed_value in precomputed.items():
+            computed_value = computed.get(key)
+            if precomputed_value is not None and computed_value is not None:
+                if abs(precomputed_value - computed_value) > threshold:
+                    logging.warning(
+                        f"Aggregate result for '{key}' differs"
+                        f" (precomputed={precomputed_value}, computed={computed_value})"
+                    )
+                    is_similar = False
+        return is_similar
 
     def to_dict(self):
         """Converts the EvalResult to a dict compatible with our dataframe display"""
 
-        model_details = model_registry.get(
-            self.full_model,
-            ModelSUT(
-                param_size=-1,
-                model_type=ModelType.UNKNOWN.value,
-                multilingual=Multilingual.UNKNOWN.value,
-            ),
-        )
+        if not self.is_submission:
+            model_details = model_registry.get(
+                self.full_model,
+                ModelSUT(
+                    param_size=-1,
+                    model_type=ModelType.UNKNOWN.value,
+                    multilingual=Multilingual.UNKNOWN.value,
+                ),
+            )
+        else:
+            model_details = ModelSUT(
+                param_size=self.param_size,
+                model_type=self.model_type,
+                multilingual=self.multilingual,
+            )
 
+        model_name_with_url = (
+            make_clickable_model(self.full_model)
+            if not self.is_submission
+            else f"📥 {model_hyperlink(self.model_url, self.full_model)}"
+        )
         data_dict = {
             "eval_name": self.eval_name,  # not a column, just a save name
             AutoEvalColumn.precision.name: self.precision.value.name,
-            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.model.name: model_name_with_url,
             AutoEvalColumn.average.name: self.average,
             AutoEvalColumn.param_size.name: model_details.param_size,
             AutoEvalColumn.model_type.name: model_details.model_type,
             AutoEvalColumn.multilingual.name: model_details.multilingual,
+            AutoEvalColumn.is_submission.name: self.is_submission,
+            AutoEvalColumn.submission_date.name: self.submission_date,
         }
 
         for task in Tasks: