Apply formatter

app.py

@@ -1,7 +1,7 @@
 import gradio as gr
-from gradio_leaderboard import Leaderboard, ColumnFilter, SelectColumns
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
+from gradio_leaderboard import ColumnFilter, Leaderboard, SelectColumns
 from huggingface_hub import snapshot_download
 
 from src.about import (
@@ -20,11 +20,19 @@ from src.display.utils import (
     EVAL_TYPES,
     AutoEvalColumn,
     ModelType,
-    fields,
+    Precision,
     WeightType,
-    Precision
+    fields,
+)
+from src.envs import (
+    API,
+    EVAL_REQUESTS_PATH,
+    EVAL_RESULTS_PATH,
+    QUEUE_REPO,
+    REPO_ID,
+    RESULTS_REPO,
+    TOKEN,
 )
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
 from src.populate import get_evaluation_queue_df, get_leaderboard_df
 from src.submission.submit import add_new_eval
 
@@ -32,24 +40,37 @@ from src.submission.submit import add_new_eval
 def restart_space():
     API.restart_space(repo_id=REPO_ID)
 
+
 ### Space initialisation
 try:
     print(EVAL_REQUESTS_PATH)
     snapshot_download(
-        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+        repo_id=QUEUE_REPO,
+        local_dir=EVAL_REQUESTS_PATH,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+        token=TOKEN,
     )
 except Exception:
     restart_space()
 try:
     print(EVAL_RESULTS_PATH)
     snapshot_download(
-        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
+        repo_id=RESULTS_REPO,
+        local_dir=EVAL_RESULTS_PATH,
+        repo_type="dataset",
+        tqdm_class=None,
+        etag_timeout=30,
+        token=TOKEN,
     )
 except Exception:
     restart_space()
 
 
-LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
+LEADERBOARD_DF = get_leaderboard_df(
+    EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS
+)
 
 (
     finished_eval_queue_df,
@@ -57,6 +78,7 @@ LEADERBOARD_DF = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS,
     pending_eval_queue_df,
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
+
 def init_leaderboard(dataframe):
     if dataframe is None or dataframe.empty:
         raise ValueError("Leaderboard DataFrame is empty or None.")
@@ -64,15 +86,23 @@ def init_leaderboard(dataframe):
         value=dataframe,
         datatype=[c.type for c in fields(AutoEvalColumn)],
         select_columns=SelectColumns(
-            default_selection=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default],
+            default_selection=[
+                c.name for c in fields(AutoEvalColumn) if c.displayed_by_default
+            ],
             cant_deselect=[c.name for c in fields(AutoEvalColumn) if c.never_hidden],
             label="Select Columns to Display:",
         ),
         search_columns=[AutoEvalColumn.model.name, AutoEvalColumn.license.name],
         hide_columns=[c.name for c in fields(AutoEvalColumn) if c.hidden],
         filter_columns=[
-            ColumnFilter(AutoEvalColumn.model_type.name, type="checkboxgroup", label="Model types"),
-            ColumnFilter(AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"),
+            ColumnFilter(
+                AutoEvalColumn.model_type.name,
+                type="checkboxgroup",
+                label="Model types",
+            ),
+            ColumnFilter(
+                AutoEvalColumn.precision.name, type="checkboxgroup", label="Precision"
+            ),
             ColumnFilter(
                 AutoEvalColumn.params.name,
                 type="slider",
@@ -81,7 +111,10 @@ def init_leaderboard(dataframe):
                 label="Select the number of parameters (B)",
             ),
             ColumnFilter(
-                AutoEvalColumn.still_on_hub.name, type="boolean", label="Deleted/incomplete", default=True
+                AutoEvalColumn.still_on_hub.name,
+                type="boolean",
+                label="Deleted/incomplete",
+                default=True,
             ),
         ],
         bool_checkboxgroup_label="Hide models",
@@ -142,14 +175,20 @@ with demo:
                                 row_count=5,
                             )
             with gr.Row():
-                gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+                gr.Markdown(
+                    "# ✉️✨ Submit your model here!", elem_classes="markdown-text"
+                )
 
             with gr.Row():
                 with gr.Column():
                     model_name_textbox = gr.Textbox(label="Model name")
-                    revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
+                    revision_name_textbox = gr.Textbox(
+                        label="Revision commit", placeholder="main"
+                    )
                     model_type = gr.Dropdown(
-                        choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
+                        choices=[
+                            t.to_str(" : ") for t in ModelType if t != ModelType.Unknown
+                        ],
                         label="Model type",
                         multiselect=False,
                         value=None,
@@ -158,7 +197,9 @@ with demo:
 
                 with gr.Column():
                     precision = gr.Dropdown(
-                        choices=[i.value.name for i in Precision if i != Precision.Unknown],
+                        choices=[
+                            i.value.name for i in Precision if i != Precision.Unknown
+                        ],
                         label="Precision",
                         multiselect=False,
                         value="float16",
@@ -171,7 +212,9 @@ with demo:
                         value="Original",
                         interactive=True,
                     )
-                    base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
+                    base_model_name_textbox = gr.Textbox(
+                        label="Base model (for delta or adapter weights)"
+                    )
 
             submit_button = gr.Button("Submit Eval")
             submission_result = gr.Markdown()
@@ -201,4 +244,4 @@ with demo:
 scheduler = BackgroundScheduler()
 scheduler.add_job(restart_space, "interval", seconds=1800)
 scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+demo.queue(default_concurrency_limit=40).launch()

pyproject.toml

@@ -2,12 +2,13 @@
 # Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
 select = ["E", "F"]
 ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
+line-length = 88
 fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
 
 [tool.isort]
 profile = "black"
-line_length = 119
+line_length = 88
+multi_line_output = 9
 
 [tool.black]
-line-length = 119
+line-length = 88

src/about.py

@@ -1,6 +1,7 @@
 from dataclasses import dataclass
 from enum import Enum
 
+
 @dataclass
 class Task:
     benchmark: str
@@ -11,13 +12,13 @@ class Task:
 # Select your tasks here
 # ---------------------------------------------------
 class Tasks(Enum):
-    # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
+    # task_key in the json file, metric_key in the json file, name to display in the leaderboard
     task0 = Task("anli_r1", "acc", "ANLI")
     task1 = Task("logiqa", "acc_norm", "LogiQA")
 
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
 
+NUM_FEWSHOT = 0  # Change with your few shot
+# ---------------------------------------------------
 
 
 # Your leaderboard name

src/display/formatting.py

@@ -16,7 +16,9 @@ def styled_warning(warn):
 
 
 def styled_message(message):
-    return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+    return (
+        f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
+    )
 
 
 def has_no_nan_values(df, columns):

src/display/utils.py

@@ -5,8 +5,11 @@ import pandas as pd
 
 from src.about import Tasks
 
+
 def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+    return [
+        v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"
+    ]
 
 
 # These classes are for user facing column names,
@@ -20,29 +23,69 @@ class ColumnContent:
     hidden: bool = False
     never_hidden: bool = False
 
+
 ## Leaderboard columns
 auto_eval_column_dict = []
 # Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
+auto_eval_column_dict.append(
+    [
+        "model_type_symbol",
+        ColumnContent,
+        ColumnContent("T", "str", True, never_hidden=True),
+    ]
+)
+auto_eval_column_dict.append(
+    [
+        "model",
+        ColumnContent,
+        ColumnContent("Model", "markdown", True, never_hidden=True),
+    ]
+)
+# Scores
+auto_eval_column_dict.append(
+    ["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)]
+)
 for task in Tasks:
-    auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
+    auto_eval_column_dict.append(
+        [task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)]
+    )
 # Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
+auto_eval_column_dict.append(
+    ["model_type", ColumnContent, ColumnContent("Type", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["architecture", ColumnContent, ColumnContent("Architecture", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)]
+)
+auto_eval_column_dict.append(
+    ["precision", ColumnContent, ColumnContent("Precision", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["license", ColumnContent, ColumnContent("Hub License", "str", False)]
+)
+auto_eval_column_dict.append(
+    ["params", ColumnContent, ColumnContent("#Params (B)", "number", False)]
+)
+auto_eval_column_dict.append(
+    ["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)]
+)
+auto_eval_column_dict.append(
+    [
+        "still_on_hub",
+        ColumnContent,
+        ColumnContent("Available on the hub", "bool", False),
+    ]
+)
+auto_eval_column_dict.append(
+    ["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)]
+)
 
 # We use make dataclass to dynamically fill the scores from Tasks
 AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
 
+
 ## For the queue columns in the submission tab
 @dataclass(frozen=True)
 class EvalQueueColumn:  # Queue column
@@ -53,12 +96,13 @@ class EvalQueueColumn:  # Queue column
     weight_type = ColumnContent("weight_type", "str", "Original")
     status = ColumnContent("status", "str", True)
 
+
 ## All the model information that we might need
 @dataclass
 class ModelDetails:
     name: str
     display_name: str = ""
-    symbol: str = "" # emoji
+    symbol: str = ""  # emoji
 
 
 class ModelType(Enum):
@@ -83,11 +127,13 @@ class ModelType(Enum):
             return ModelType.IFT
         return ModelType.Unknown
 
+
 class WeightType(Enum):
     Adapter = ModelDetails("Adapter")
     Original = ModelDetails("Original")
     Delta = ModelDetails("Delta")
 
+
 class Precision(Enum):
     float16 = ModelDetails("float16")
     bfloat16 = ModelDetails("bfloat16")
@@ -100,6 +146,7 @@ class Precision(Enum):
             return Precision.bfloat16
         return Precision.Unknown
 
+
 # Column selection
 COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
 
@@ -107,4 +154,3 @@ EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
 EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
 
 BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-

src/envs.py

@@ -4,9 +4,9 @@ from huggingface_hub import HfApi
 
 # Info to change for your repository
 # ----------------------------------
-TOKEN = os.environ.get("HF_TOKEN") # A read/write token for your org
+TOKEN = os.environ.get("HF_TOKEN")  # A read/write token for your org
 
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
+OWNER = "demo-leaderboard-backend"  # Change to your org - don't forget to create a results and request dataset, with the correct format!
 # ----------------------------------
 
 REPO_ID = f"{OWNER}/leaderboard"
@@ -14,7 +14,7 @@ QUEUE_REPO = f"{OWNER}/requests"
 RESULTS_REPO = f"{OWNER}/results"
 
 # If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
+CACHE_PATH = os.getenv("HF_HOME", ".")
 
 # Local caches
 EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")

src/leaderboard/read_evals.py

@@ -8,28 +8,28 @@ import dateutil
 import numpy as np
 
 from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
+from src.display.utils import AutoEvalColumn, ModelType, Precision, Tasks, WeightType
 from src.submission.check_validity import is_model_on_hub
 
 
 @dataclass
 class EvalResult:
-    """Represents one full evaluation. Built from a combination of the result and request file for a given run.
-    """
-    eval_name: str # org_model_precision (uid)
-    full_model: str # org/model (path on hub)
-    org: str 
+    """Represents one full evaluation. Built from a combination of the result and request file for a given run."""
+
+    eval_name: str  # org_model_precision (uid)
+    full_model: str  # org/model (path on hub)
+    org: str
     model: str
-    revision: str # commit hash, "" if main
+    revision: str  # commit hash, "" if main
     results: dict
     precision: Precision = Precision.Unknown
-    model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
-    weight_type: WeightType = WeightType.Original # Original or Adapter
-    architecture: str = "Unknown" 
+    model_type: ModelType = ModelType.Unknown  # Pretrained, fine tuned, ...
+    weight_type: WeightType = WeightType.Original  # Original or Adapter
+    architecture: str = "Unknown"
     license: str = "?"
     likes: int = 0
     num_params: int = 0
-    date: str = "" # submission date of request file
+    date: str = ""  # submission date of request file
     still_on_hub: bool = False
 
     @classmethod
@@ -58,7 +58,10 @@ class EvalResult:
         full_model = "/".join(org_and_model)
 
         still_on_hub, _, model_config = is_model_on_hub(
-            full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
+            full_model,
+            config.get("model_sha", "main"),
+            trust_remote_code=True,
+            test_tokenizer=False,
         )
         architecture = "?"
         if model_config is not None:
@@ -72,7 +75,13 @@ class EvalResult:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            accs = np.array(
+                [
+                    v.get(task.metric, None)
+                    for k, v in data["results"].items()
+                    if task.benchmark == k
+                ]
+            )
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue
 
@@ -85,15 +94,17 @@ class EvalResult:
             org=org,
             model=model,
             results=results,
-            precision=precision,  
-            revision= config.get("model_sha", ""),
+            precision=precision,
+            revision=config.get("model_sha", ""),
             still_on_hub=still_on_hub,
-            architecture=architecture
+            architecture=architecture,
         )
 
     def update_with_request_file(self, requests_path):
         """Finds the relevant request file for the current model and updates info with it"""
-        request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
+        request_file = get_request_file_for_model(
+            requests_path, self.full_model, self.precision.value.name
+        )
 
         try:
             with open(request_file, "r") as f:
@@ -105,7 +116,9 @@ class EvalResult:
             self.num_params = request.get("params", 0)
             self.date = request.get("submitted_time", "")
         except Exception:
-            print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
+            print(
+                f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}"
+            )
 
     def to_dict(self):
         """Converts the Eval Result to a dict compatible with our dataframe display"""
@@ -165,7 +178,9 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
 
         # Sort the files by date
         try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+            files.sort(
+                key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7]
+            )
         except dateutil.parser._parser.ParserError:
             files = [files[-1]]
 
@@ -181,14 +196,16 @@ def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResu
         # Store results of same eval together
         eval_name = eval_result.eval_name
         if eval_name in eval_results.keys():
-            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+            eval_results[eval_name].results.update(
+                {k: v for k, v in eval_result.results.items() if v is not None}
+            )
         else:
             eval_results[eval_name] = eval_result
 
     results = []
     for v in eval_results.values():
         try:
-            v.to_dict() # we test if the dict version is complete
+            v.to_dict()  # we test if the dict version is complete
             results.append(v)
         except KeyError:  # not all eval values present
             continue

src/populate.py

@@ -8,7 +8,9 @@ from src.display.utils import AutoEvalColumn, EvalQueueColumn
 from src.leaderboard.read_evals import get_raw_eval_results
 
 
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
+def get_leaderboard_df(
+    results_path: str, requests_path: str, cols: list, benchmark_cols: list
+) -> pd.DataFrame:
     """Creates a dataframe from all the individual experiment results"""
     raw_data = get_raw_eval_results(results_path, requests_path)
     all_data_json = [v.to_dict() for v in raw_data]
@@ -39,7 +41,11 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
             all_evals.append(data)
         elif ".md" not in entry:
             # this is a folder
-            sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if os.path.isfile(e) and not e.startswith(".")]
+            sub_entries = [
+                e
+                for e in os.listdir(f"{save_path}/{entry}")
+                if os.path.isfile(e) and not e.startswith(".")
+            ]
             for sub_entry in sub_entries:
                 file_path = os.path.join(save_path, entry, sub_entry)
                 with open(file_path) as fp:
@@ -51,7 +57,11 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
 
     pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
     running_list = [e for e in all_evals if e["status"] == "RUNNING"]
-    finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
+    finished_list = [
+        e
+        for e in all_evals
+        if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"
+    ]
     df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)

src/submission/check_validity.py

@@ -10,12 +10,16 @@ from huggingface_hub.hf_api import ModelInfo
 from transformers import AutoConfig
 from transformers.models.auto.tokenization_auto import AutoTokenizer
 
+
 def check_model_card(repo_id: str) -> tuple[bool, str]:
     """Checks if the model card and license exist and have been filled"""
     try:
         card = ModelCard.load(repo_id)
     except huggingface_hub.utils.EntryNotFoundError:
-        return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
+        return (
+            False,
+            "Please add a model card to your model to explain how you trained/fine-tuned it.",
+        )
 
     # Enforce license metadata
     if card.data.license is None:
@@ -31,28 +35,49 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
 
     return True, ""
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
+
+def is_model_on_hub(
+    model_name: str,
+    revision: str,
+    token: str = None,
+    trust_remote_code=False,
+    test_tokenizer=False,
+) -> tuple[bool, str]:
     """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
     try:
-        config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+        config = AutoConfig.from_pretrained(
+            model_name,
+            revision=revision,
+            trust_remote_code=trust_remote_code,
+            token=token,
+        )
         if test_tokenizer:
             try:
-                tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
+                tk = AutoTokenizer.from_pretrained(
+                    model_name,
+                    revision=revision,
+                    trust_remote_code=trust_remote_code,
+                    token=token,
+                )
             except ValueError as e:
                 return (
                     False,
                     f"uses a tokenizer which is not in a transformers release: {e}",
-                    None
+                    None,
                 )
             except Exception as e:
-                return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
+                return (
+                    False,
+                    "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?",
+                    None,
+                )
         return True, None, config
 
     except ValueError:
         return (
             False,
             "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
-            None
+            None,
         )
 
     except Exception as e:
@@ -64,16 +89,22 @@ def get_model_size(model_info: ModelInfo, precision: str):
     try:
         model_size = round(model_info.safetensors["total"] / 1e9, 3)
     except (AttributeError, TypeError):
-        return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+        return (
+            0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+        )
 
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    size_factor = (
+        8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    )
     model_size = size_factor * model_size
     return model_size
 
+
 def get_model_arch(model_info: ModelInfo):
     """Gets the model architecture from the configuration"""
     return model_info.config.get("architectures", "Unknown")
 
+
 def already_submitted_models(requested_models_dir: str) -> set[str]:
     """Gather a list of already submitted models to avoid duplicates"""
     depth = 1
@@ -88,12 +119,16 @@ def already_submitted_models(requested_models_dir: str) -> set[str]:
                     continue
                 with open(os.path.join(root, file), "r") as f:
                     info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+                    file_names.append(
+                        f"{info['model']}_{info['revision']}_{info['precision']}"
+                    )
 
                     # Select organisation
                     if info["model"].count("/") == 0 or "submitted_time" not in info:
                         continue
                     organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
+                    users_to_submission_dates[organisation].append(
+                        info["submitted_time"]
+                    )
 
     return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -3,7 +3,7 @@ import os
 from datetime import datetime, timezone
 
 from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
+from src.envs import API, EVAL_REQUESTS_PATH, QUEUE_REPO, TOKEN
 from src.submission.check_validity import (
     already_submitted_models,
     check_model_card,
@@ -14,6 +14,7 @@ from src.submission.check_validity import (
 REQUESTED_MODELS = None
 USERS_TO_SUBMISSION_DATES = None
 
+
 def add_new_eval(
     model: str,
     base_model: str,
@@ -25,7 +26,9 @@ def add_new_eval(
     global REQUESTED_MODELS
     global USERS_TO_SUBMISSION_DATES
     if not REQUESTED_MODELS:
-        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
+        REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(
+            EVAL_REQUESTS_PATH
+        )
 
     user_name = ""
     model_path = model
@@ -45,12 +48,16 @@ def add_new_eval(
 
     # Is the model on the hub?
     if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
+        base_model_on_hub, error, _ = is_model_on_hub(
+            model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True
+        )
         if not base_model_on_hub:
             return styled_error(f'Base model "{base_model}" {error}')
 
     if not weight_type == "Adapter":
-        model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
+        model_on_hub, error, _ = is_model_on_hub(
+            model_name=model, revision=revision, token=TOKEN, test_tokenizer=True
+        )
         if not model_on_hub:
             return styled_error(f'Model "{model}" {error}')
 
@@ -58,7 +65,9 @@ def add_new_eval(
     try:
         model_info = API.model_info(repo_id=model, revision=revision)
     except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
+        return styled_error(
+            "Could not get your model information. Please fill it up properly."
+        )
 
     model_size = get_model_size(model_info=model_info, precision=precision)
 
@@ -97,7 +106,9 @@ def add_new_eval(
     print("Creating eval file")
     OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
     os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    out_path = (
+        f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
+    )
 
     with open(out_path, "w") as f:
         f.write(json.dumps(eval_entry))