refacto part 1

app.py

@@ -1,14 +1,24 @@
 import json
 import os
-from datetime import datetime, timezone
 
 import gradio as gr
 import pandas as pd
 from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import HfApi, snapshot_download
-
-from src.assets.css_html_js import custom_css, get_window_url_params
-from src.assets.text_content import (
+from huggingface_hub import snapshot_download
+
+from src.display.utils import (
+    COLS,
+    TYPES,
+    BENCHMARK_COLS,
+    EVAL_COLS,
+    EVAL_TYPES,
+    AutoEvalColumn,
+    ModelType,
+    NUMERIC_INTERVALS,
+    fields,
+)
+from src.display.css_html_js import custom_css, get_window_url_params
+from src.display.about import (
     CITATION_BUTTON_LABEL,
     CITATION_BUTTON_TEXT,
     EVALUATION_QUEUE_TEXT,
@@ -16,102 +26,44 @@ from src.assets.text_content import (
     LLM_BENCHMARKS_TEXT,
     TITLE,
 )
-from src.plots.plot_results import (
+from src.tools.plots import (
     create_metric_plot_obj,
     create_scores_df,
     create_plot_df,
     join_model_info_with_results,
     HUMAN_BASELINES,
 )
-from src.get_model_info.apply_metadata_to_df import DO_NOT_SUBMIT_MODELS, ModelType
-from src.get_model_info.get_metadata_from_hub import get_model_size
-from src.filters import check_model_card
-from src.get_model_info.utils import (
-    AutoEvalColumn,
-    EvalQueueColumn,
-    fields,
-    styled_error,
-    styled_message,
-    styled_warning,
-)
-from src.manage_collections import update_collections
-from src.load_from_hub import get_all_requested_models, get_evaluation_queue_df, get_leaderboard_df
-from src.filters import is_model_on_hub, user_submission_permission
-
-pd.set_option("display.precision", 1)
-
-# clone / pull the lmeh eval data
-H4_TOKEN = os.environ.get("H4_TOKEN", None)
-
-QUEUE_REPO = "open-llm-leaderboard/requests"
-RESULTS_REPO = "open-llm-leaderboard/results"
-
-PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"
-PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"
-
-IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
-
-EVAL_REQUESTS_PATH = "eval-queue"
-EVAL_RESULTS_PATH = "eval-results"
-
-EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
-EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
-
-api = HfApi(token=H4_TOKEN)
+from src.tools.collections import update_collections
+from src.populate import get_evaluation_queue_df, get_leaderboard_df
+from src.envs import H4_TOKEN, QUEUE_REPO, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, RESULTS_REPO, API, REPO_ID, IS_PUBLIC
+from src.submission.submit import add_new_eval
 
 
 def restart_space():
-    api.restart_space(repo_id="HuggingFaceH4/open_llm_leaderboard", token=H4_TOKEN)
-
-
-# Rate limit variables
-RATE_LIMIT_PERIOD = 7
-RATE_LIMIT_QUOTA = 5
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-
-if not IS_PUBLIC:
-    COLS.insert(2, AutoEvalColumn.precision.name)
-    TYPES.insert(2, AutoEvalColumn.precision.type)
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [
-    c.name
-    for c in [
-        AutoEvalColumn.arc,
-        AutoEvalColumn.hellaswag,
-        AutoEvalColumn.mmlu,
-        AutoEvalColumn.truthfulqa,
-        AutoEvalColumn.winogrande,
-        AutoEvalColumn.gsm8k,
-        AutoEvalColumn.drop
-    ]
-]
+    API.restart_space(repo_id=REPO_ID, token=H4_TOKEN)
+
 
 try:
-    snapshot_download(repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
+    snapshot_download(
+        repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    )
 except Exception:
     restart_space()
 try:
-    snapshot_download(repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30)
+    snapshot_download(
+        repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30
+    )
 except Exception:
     restart_space()
 
-requested_models, users_to_submission_dates = get_all_requested_models(EVAL_REQUESTS_PATH)
 
 original_df = get_leaderboard_df(EVAL_RESULTS_PATH, COLS, BENCHMARK_COLS)
-update_collections(original_df.copy())
+#update_collections(original_df.copy())
 leaderboard_df = original_df.copy()
 
-models = original_df["model_name_for_query"].tolist()  # needed for model backlinks in their to the leaderboard
-#plot_df = create_plot_df(create_scores_df(join_model_info_with_results(original_df)))
-to_be_dumped = f"models = {repr(models)}\n"
+#models = original_df["model_name_for_query"].tolist()  # needed for model backlinks in their to the leaderboard
+# plot_df = create_plot_df(create_scores_df(join_model_info_with_results(original_df)))
+#to_be_dumped = f"models = {repr(models)}\n"
 
 (
     finished_eval_queue_df,
@@ -120,115 +72,6 @@ to_be_dumped = f"models = {repr(models)}\n"
 ) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
 
 
-## INTERACTION FUNCTIONS
-def add_new_eval(
-    model: str,
-    base_model: str,
-    revision: str,
-    precision: str,
-    private: bool,
-    weight_type: str,
-    model_type: str,
-):
-    precision = precision.split(" ")[0]
-    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
-    if model_type is None or model_type == "":
-        return styled_error("Please select a model type.")
-
-    # Is the user rate limited?
-    user_can_submit, error_msg = user_submission_permission(model, users_to_submission_dates, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA)
-    if not user_can_submit:
-        return styled_error(error_msg)
-
-    # Did the model authors forbid its submission to the leaderboard?
-    if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
-        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
-
-    # Does the model actually exist?
-    if revision == "":
-        revision = "main"
-
-    if weight_type in ["Delta", "Adapter"]:
-        base_model_on_hub, error = is_model_on_hub(base_model, revision, H4_TOKEN)
-        if not base_model_on_hub:
-            return styled_error(f'Base model "{base_model}" {error}')
-
-    if not weight_type == "Adapter":
-        model_on_hub, error = is_model_on_hub(model, revision)
-        if not model_on_hub:
-            return styled_error(f'Model "{model}" {error}')
-
-    try:
-        model_info = api.model_info(repo_id=model, revision=revision)
-    except Exception:
-        return styled_error("Could not get your model information. Please fill it up properly.")
-
-    model_size = get_model_size(model_info=model_info , precision= precision)
-
-    # Were the model card and license filled?
-    try:
-        license = model_info.cardData["license"]
-    except Exception:
-        return styled_error("Please select a license for your model")
-
-    modelcard_OK, error_msg = check_model_card(model)
-    if not modelcard_OK:
-        return styled_error(error_msg)
-
-    # Seems good, creating the eval
-    print("Adding new eval")
-
-    eval_entry = {
-        "model": model,
-        "base_model": base_model,
-        "revision": revision,
-        "private": private,
-        "precision": precision,
-        "weight_type": weight_type,
-        "status": "PENDING",
-        "submitted_time": current_time,
-        "model_type": model_type,
-        "likes": model_info.likes,
-        "params": model_size,
-        "license": license,
-    }
-
-    user_name = ""
-    model_path = model
-    if "/" in model:
-        user_name = model.split("/")[0]
-        model_path = model.split("/")[1]
-
-    print("Creating eval file")
-    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
-    os.makedirs(OUT_DIR, exist_ok=True)
-    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
-
-    # Check for duplicate submission
-    if f"{model}_{revision}_{precision}" in requested_models:
-        return styled_warning("This model has been already submitted.")
-
-    with open(out_path, "w") as f:
-        f.write(json.dumps(eval_entry))
-
-    print("Uploading eval file")
-    api.upload_file(
-        path_or_fileobj=out_path,
-        path_in_repo=out_path.split("eval-queue/")[1],
-        repo_id=QUEUE_REPO,
-        repo_type="dataset",
-        commit_message=f"Add {model} to eval queue",
-    )
-
-    # Remove the local file
-    os.remove(out_path)
-
-    return styled_message(
-        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
-    )
-
-
 # Basics
 def change_tab(query_param: str):
     query_param = query_param.replace("'", '"')
@@ -272,18 +115,6 @@ def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
     return filtered_df
 
 
-NUMERIC_INTERVALS = {
-    "?": pd.Interval(-1, 0, closed="right"),
-    "~1.5": pd.Interval(0, 2, closed="right"),
-    "~3": pd.Interval(2, 4, closed="right"),
-    "~7": pd.Interval(4, 9, closed="right"),
-    "~13": pd.Interval(9, 20, closed="right"),
-    "~35": pd.Interval(20, 45, closed="right"),
-    "~60": pd.Interval(45, 70, closed="right"),
-    "70+": pd.Interval(70, 10000, closed="right"),
-}
-
-
 def filter_queries(query: str, filtered_df: pd.DataFrame):
     """Added by Abishek"""
     final_df = []
@@ -311,7 +142,7 @@ def filter_models(
     if show_deleted:
         filtered_df = df
     else:  # Show only still on the hub models
-        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] is True]
+        filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
 
     type_emoji = [t[0] for t in type_query]
     filtered_df = filtered_df[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
@@ -342,54 +173,22 @@ with demo:
                         )
                     with gr.Row():
                         shown_columns = gr.CheckboxGroup(
-                            choices=[
-                                c
-                                for c in COLS
-                                if c
-                                not in [
-                                    AutoEvalColumn.dummy.name,
-                                    AutoEvalColumn.model.name,
-                                    AutoEvalColumn.model_type_symbol.name,
-                                    AutoEvalColumn.still_on_hub.name,
-                                ]
-                            ],
-                            value=[
-                                c
-                                for c in COLS_LITE
-                                if c
-                                not in [
-                                    AutoEvalColumn.dummy.name,
-                                    AutoEvalColumn.model.name,
-                                    AutoEvalColumn.model_type_symbol.name,
-                                    AutoEvalColumn.still_on_hub.name,
-                                ]
-                            ],
+                            choices=[c.name for c in fields(AutoEvalColumn) if not c.hidden and not c.never_hidden and not c.dummy],
+                            value=[c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden and not c.never_hidden],
                             label="Select columns to show",
                             elem_id="column-select",
                             interactive=True,
                         )
                     with gr.Row():
                         deleted_models_visibility = gr.Checkbox(
-                            value=True, label="Show gated/private/deleted models", interactive=True
+                            value=False, label="Show gated/private/deleted models", interactive=True
                         )
                 with gr.Column(min_width=320):
                     with gr.Box(elem_id="box-filter"):
                         filter_columns_type = gr.CheckboxGroup(
                             label="Model types",
-                            choices=[
-                                ModelType.PT.to_str(),
-                                ModelType.FT.to_str(),
-                                ModelType.IFT.to_str(),
-                                ModelType.RL.to_str(),
-                                ModelType.Unknown.to_str(),
-                            ],
-                            value=[
-                                ModelType.PT.to_str(),
-                                ModelType.FT.to_str(),
-                                ModelType.IFT.to_str(),
-                                ModelType.RL.to_str(),
-                                ModelType.Unknown.to_str(),
-                            ],
+                            choices=[t.to_str() for t in ModelType],
+                            value=[t.to_str() for t in ModelType],
                             interactive=True,
                             elem_id="filter-columns-type",
                         )
@@ -410,16 +209,11 @@ with demo:
 
             leaderboard_table = gr.components.Dataframe(
                 value=leaderboard_df[
-                    [AutoEvalColumn.model_type_symbol.name, AutoEvalColumn.model.name]
+                    [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
                     + shown_columns.value
                     + [AutoEvalColumn.dummy.name]
                 ],
-                headers=[
-                    AutoEvalColumn.model_type_symbol.name,
-                    AutoEvalColumn.model.name,
-                ]
-                + shown_columns.value
-                + [AutoEvalColumn.dummy.name],
+                headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
                 datatype=TYPES,
                 max_rows=None,
                 elem_id="leaderboard-table",
@@ -429,7 +223,7 @@ with demo:
 
             # Dummy leaderboard for handling the case when the user uses backspace key
             hidden_leaderboard_table_for_search = gr.components.Dataframe(
-                value=original_df,
+                value=original_df[COLS],
                 headers=COLS,
                 datatype=TYPES,
                 max_rows=None,
@@ -519,7 +313,8 @@ with demo:
                 queue=True,
             )
 
-        # with gr.TabItem("📈 Metrics evolution through time", elem_id="llm-benchmark-tab-table", id=4):
+        # with gr.TabItem("📈
+        #  evolution through time", elem_id="llm-benchmark-tab-table", id=4):
         #     with gr.Row():
         #         with gr.Column():
         #             chart = create_metric_plot_obj(
@@ -589,12 +384,7 @@ with demo:
                     revision_name_textbox = gr.Textbox(label="revision", placeholder="main")
                     private = gr.Checkbox(False, label="Private", visible=not IS_PUBLIC)
                     model_type = gr.Dropdown(
-                        choices=[
-                            ModelType.PT.to_str(" : "),
-                            ModelType.FT.to_str(" : "),
-                            ModelType.IFT.to_str(" : "),
-                            ModelType.RL.to_str(" : "),
-                        ],
+                        choices=[t.to_str(" : ") for t in ModelType],
                         label="Model type",
                         multiselect=False,
                         value=None,

model_info_cache.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:15ee9a3cdd3ffdfa4d46497b829fbb43ea5a66222a17d34dfef5ad1111a8eb18
-size 3789941

model_size_cache.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ace7167a258f711fa7ffeaadddc6ebef8ccb92651dce8b805228c2f18c988958
-size 75324

scripts/create_request_file.py

@@ -10,10 +10,11 @@ import pprint
 EVAL_REQUESTS_PATH = "eval-queue"
 QUEUE_REPO = "open-llm-leaderboard/requests"
 
-precisions =("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
+precisions = ("float16", "bfloat16", "8bit (LLM.int8)", "4bit (QLoRA / FP4)", "GPTQ")
 model_types = ("pretrained", "fine-tuned", "RL-tuned", "instruction-tuned")
 weight_types = ("Original", "Delta", "Adapter")
 
+
 def get_model_size(model_info, precision: str):
     size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
     try:
@@ -24,12 +25,13 @@ def get_model_size(model_info, precision: str):
             model_size = size_match.group(0)
             model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
         except AttributeError:
-            return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
 
     size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
     model_size = size_factor * model_size
     return model_size
 
+
 def main():
     api = HfApi()
     current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
@@ -49,7 +51,7 @@ def main():
         print(f"{Fore.RED}Could not find model info for {model_name} on the Hub\n{e}{Fore.RESET}")
         return 1
 
-    model_size = get_model_size(model_info=model_info , precision=precision) 
+    model_size = get_model_size(model_info=model_info, precision=precision)
 
     try:
         license = model_info.cardData["license"]
@@ -98,7 +100,7 @@ def main():
         )
     else:
         click.echo("aborting...")
-    
 
-if __name__ == '__main__':
-    main()
+
+if __name__ == "__main__":
+    main()

src/assets/hardcoded_evals.py

@@ -1,43 +0,0 @@
-from src.get_model_info.utils import AutoEvalColumn, model_hyperlink
-
-gpt4_values = {
-    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt4"),
-    AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 84.3,
-    AutoEvalColumn.arc.name: 96.3,
-    AutoEvalColumn.hellaswag.name: 95.3,
-    AutoEvalColumn.mmlu.name: 86.4,
-    AutoEvalColumn.truthfulqa.name: 59.0,
-    AutoEvalColumn.dummy.name: "GPT-4",
-    AutoEvalColumn.model_type.name: "",
-}
-
-gpt35_values = {
-    AutoEvalColumn.model.name: model_hyperlink("https://arxiv.org/abs/2303.08774", "gpt3.5"),
-    AutoEvalColumn.revision.name: "tech report",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 71.9,
-    AutoEvalColumn.arc.name: 85.2,
-    AutoEvalColumn.hellaswag.name: 85.5,
-    AutoEvalColumn.mmlu.name: 70.0,
-    AutoEvalColumn.truthfulqa.name: 47.0,
-    AutoEvalColumn.dummy.name: "GPT-3.5",
-    AutoEvalColumn.model_type.name: "",
-}
-
-baseline = {
-    AutoEvalColumn.model.name: "<p>Baseline</p>",
-    AutoEvalColumn.revision.name: "N/A",
-    AutoEvalColumn.precision.name: None,
-    AutoEvalColumn.average.name: 25.0,
-    AutoEvalColumn.arc.name: 25.0,
-    AutoEvalColumn.hellaswag.name: 25.0,
-    AutoEvalColumn.mmlu.name: 25.0,
-    AutoEvalColumn.truthfulqa.name: 25.0,
-    AutoEvalColumn.winogrande.name: 50.0,
-    AutoEvalColumn.gsm8k.name: 0.21,
-    AutoEvalColumn.drop.name: 0.47,
-    AutoEvalColumn.dummy.name: "baseline",
-    AutoEvalColumn.model_type.name: "",
-}

src/{assets/text_content.py → display/about.py}

@@ -1,4 +1,4 @@
-from src.get_model_info.hardocded_metadata.types import ModelType
+from src.display.utils import ModelType
 
 TITLE = """<h1 align="center" id="space-title">🤗 Open LLM Leaderboard</h1>"""
 
@@ -42,7 +42,7 @@ We chose these benchmarks as they test a variety of reasoning and general knowle
 ## Details and logs
 You can find:
 - detailed numerical results in the `results` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/results
-- details on the input/outputs for the models in the `details` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/details
+- details on the input/outputs for the models in the `details` of each model, that you can access by clicking the 📄 emoji after the model name
 - community queries and running status in the `requests` Hugging Face dataset: https://huggingface.co/datasets/open-llm-leaderboard/requests
 
 ## Reproducibility

src/{get_model_info/utils.py → display/formatting.py}

@@ -1,68 +1,8 @@
 import os
-from dataclasses import dataclass
-
 from huggingface_hub import HfApi
 
 API = HfApi()
 
-
-# These classes are for user facing column names, to avoid having to change them
-# all around the code when a modif is needed
-@dataclass
-class ColumnContent:
-    name: str
-    type: str
-    displayed_by_default: bool
-    hidden: bool = False
-
-
-def fields(raw_class):
-    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-@dataclass(frozen=True)
-class AutoEvalColumn:  # Auto evals column
-    model_type_symbol = ColumnContent("T", "str", True)
-    model = ColumnContent("Model", "markdown", True)
-    average = ColumnContent("Average ⬆️", "number", True)
-    arc = ColumnContent("ARC", "number", True)
-    hellaswag = ColumnContent("HellaSwag", "number", True)
-    mmlu = ColumnContent("MMLU", "number", True)
-    truthfulqa = ColumnContent("TruthfulQA", "number", True)
-    winogrande = ColumnContent("Winogrande", "number", True)
-    gsm8k = ColumnContent("GSM8K", "number", True)
-    drop = ColumnContent("DROP", "number", True)
-    model_type = ColumnContent("Type", "str", False)
-    precision = ColumnContent("Precision", "str", False)  # , True)
-    license = ColumnContent("Hub License", "str", False)
-    params = ColumnContent("#Params (B)", "number", False)
-    likes = ColumnContent("Hub ❤️", "number", False)
-    still_on_hub = ColumnContent("Available on the hub", "bool", False)
-    revision = ColumnContent("Model sha", "str", False, False)
-    dummy = ColumnContent(
-        "model_name_for_query", "str", True
-    )  # dummy col to implement search bar (hidden by custom CSS)
-
-
-@dataclass(frozen=True)
-class EloEvalColumn:  # Elo evals column
-    model = ColumnContent("Model", "markdown", True)
-    gpt4 = ColumnContent("GPT-4 (all)", "number", True)
-    human_all = ColumnContent("Human (all)", "number", True)
-    human_instruct = ColumnContent("Human (instruct)", "number", True)
-    human_code_instruct = ColumnContent("Human (code-instruct)", "number", True)
-
-
-@dataclass(frozen=True)
-class EvalQueueColumn:  # Queue column
-    model = ColumnContent("model", "markdown", True)
-    revision = ColumnContent("revision", "str", True)
-    private = ColumnContent("private", "bool", True)
-    precision = ColumnContent("precision", "str", True)
-    weight_type = ColumnContent("weight_type", "str", "Original")
-    status = ColumnContent("status", "str", True)
-
-
 LLAMAS = [
     "huggingface/llama-7b",
     "huggingface/llama-13b",
@@ -70,7 +10,6 @@ LLAMAS = [
     "huggingface/llama-65b",
 ]
 
-
 KOALA_LINK = "https://huggingface.co/TheBloke/koala-13B-HF"
 VICUNA_LINK = "https://huggingface.co/lmsys/vicuna-13b-delta-v1.1"
 OASST_LINK = "https://huggingface.co/OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"

src/display/utils.py

@@ -0,0 +1,141 @@
+from dataclasses import dataclass
+import pandas as pd
+from enum import Enum
+
+
+# These classes are for user facing column names,
+# to avoid having to change them all around the code
+# when a modif is needed
+@dataclass
+class ColumnContent:
+    name: str
+    type: str
+    displayed_by_default: bool
+    hidden: bool = False
+    never_hidden: bool = False
+    dummy: bool = False
+
+
+def fields(raw_class):
+    return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
+
+
+@dataclass(frozen=True)
+class AutoEvalColumn:  # Auto evals column
+    model_type_symbol = ColumnContent("T", "str", True, never_hidden=True)
+    model = ColumnContent("Model", "markdown", True, never_hidden=True)
+    average = ColumnContent("Average ⬆️", "number", True)
+    arc = ColumnContent("ARC", "number", True)
+    hellaswag = ColumnContent("HellaSwag", "number", True)
+    mmlu = ColumnContent("MMLU", "number", True)
+    truthfulqa = ColumnContent("TruthfulQA", "number", True)
+    winogrande = ColumnContent("Winogrande", "number", True)
+    gsm8k = ColumnContent("GSM8K", "number", True)
+    drop = ColumnContent("DROP", "number", True)
+    model_type = ColumnContent("Type", "str", False)
+    weight_type = ColumnContent("Weight type", "str", False, True)
+    precision = ColumnContent("Precision", "str", False)  # , True)
+    license = ColumnContent("Hub License", "str", False)
+    params = ColumnContent("#Params (B)", "number", False)
+    likes = ColumnContent("Hub ❤️", "number", False)
+    still_on_hub = ColumnContent("Available on the hub", "bool", False)
+    revision = ColumnContent("Model sha", "str", False, False)
+    dummy = ColumnContent(
+        "model_name_for_query", "str", False, dummy=True
+    )  # dummy col to implement search bar (hidden by custom CSS)
+
+
+@dataclass(frozen=True)
+class EvalQueueColumn:  # Queue column
+    model = ColumnContent("model", "markdown", True)
+    revision = ColumnContent("revision", "str", True)
+    private = ColumnContent("private", "bool", True)
+    precision = ColumnContent("precision", "str", True)
+    weight_type = ColumnContent("weight_type", "str", "Original")
+    status = ColumnContent("status", "str", True)
+
+
+baseline_row = {
+    AutoEvalColumn.model.name: "<p>Baseline</p>",
+    AutoEvalColumn.revision.name: "N/A",
+    AutoEvalColumn.precision.name: None,
+    AutoEvalColumn.average.name: 25.0,
+    AutoEvalColumn.arc.name: 25.0,
+    AutoEvalColumn.hellaswag.name: 25.0,
+    AutoEvalColumn.mmlu.name: 25.0,
+    AutoEvalColumn.truthfulqa.name: 25.0,
+    AutoEvalColumn.winogrande.name: 50.0,
+    AutoEvalColumn.gsm8k.name: 0.21,
+    AutoEvalColumn.drop.name: 0.47,
+    AutoEvalColumn.dummy.name: "baseline",
+    AutoEvalColumn.model_type.name: "",
+}
+
+
+@dataclass
+class ModelInfo:
+    name: str
+    symbol: str  # emoji
+
+
+class ModelType(Enum):
+    PT = ModelInfo(name="pretrained", symbol="🟢")
+    FT = ModelInfo(name="fine-tuned", symbol="🔶")
+    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
+    RL = ModelInfo(name="RL-tuned", symbol="🟦")
+    Unknown = ModelInfo(name="", symbol="?")
+
+    def to_str(self, separator=" "):
+        return f"{self.value.symbol}{separator}{self.value.name}"
+
+    @staticmethod
+    def from_str(type):
+        if "fine-tuned" in type or "🔶" in type:
+            return ModelType.FT
+        if "pretrained" in type or "🟢" in type:
+            return ModelType.PT
+        if "RL-tuned" in type or "🟦" in type:
+            return ModelType.RL
+        if "instruction-tuned" in type or "⭕" in type:
+            return ModelType.IFT
+        return ModelType.Unknown
+
+
+@dataclass
+class Task:
+    benchmark: str
+    metric: str
+    col_name: str
+
+
+class Tasks(Enum):
+    arc = Task("arc:challenge", "acc_norm", AutoEvalColumn.arc.name)
+    hellaswag = Task("hellaswag", "acc_norm", AutoEvalColumn.hellaswag.name)
+    mmlu = Task("hendrycksTest", "acc", AutoEvalColumn.mmlu.name)
+    truthfulqa = Task("truthfulqa:mc", "mc2", AutoEvalColumn.truthfulqa.name)
+    winogrande = Task("winogrande", "acc", AutoEvalColumn.winogrande.name)
+    gsm8k = Task("gsm8k", "acc", AutoEvalColumn.gsm8k.name)
+    drop = Task("drop", "f1", AutoEvalColumn.drop.name)
+
+
+# Column selection
+COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
+TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
+COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
+
+EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
+EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
+
+BENCHMARK_COLS = [t.value.col_name for t in Tasks if t.value.col_name in fields(AutoEvalColumn)]
+
+NUMERIC_INTERVALS = {
+    "?": pd.Interval(-1, 0, closed="right"),
+    "~1.5": pd.Interval(0, 2, closed="right"),
+    "~3": pd.Interval(2, 4, closed="right"),
+    "~7": pd.Interval(4, 9, closed="right"),
+    "~13": pd.Interval(9, 20, closed="right"),
+    "~35": pd.Interval(20, 45, closed="right"),
+    "~60": pd.Interval(45, 70, closed="right"),
+    "70+": pd.Interval(70, 10000, closed="right"),
+}

src/envs.py

@@ -0,0 +1,28 @@
+import os
+from huggingface_hub import HfApi
+
+# clone / pull the lmeh eval data
+H4_TOKEN = os.environ.get("H4_TOKEN", None)
+
+REPO_ID = "HuggingFaceH4/open_llm_leaderboard"
+QUEUE_REPO = "open-llm-leaderboard/requests"
+RESULTS_REPO = "open-llm-leaderboard/results"
+
+PRIVATE_QUEUE_REPO = "open-llm-leaderboard/private-requests"
+PRIVATE_RESULTS_REPO = "open-llm-leaderboard/private-results"
+
+IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
+
+EVAL_REQUESTS_PATH = "eval-queue"
+EVAL_RESULTS_PATH = "eval-results"
+
+EVAL_REQUESTS_PATH_PRIVATE = "eval-queue-private"
+EVAL_RESULTS_PATH_PRIVATE = "eval-results-private"
+
+PATH_TO_COLLECTION = "open-llm-leaderboard/llm-leaderboard-best-models-652d6c7965a4619fb5c27a03"
+
+# Rate limit variables
+RATE_LIMIT_PERIOD = 7
+RATE_LIMIT_QUOTA = 5
+
+API = HfApi(token=H4_TOKEN)

src/get_model_info/apply_metadata_to_df.py

@@ -1,95 +0,0 @@
-import glob
-import json
-import os
-from typing import List
-
-from huggingface_hub import HfApi
-from tqdm import tqdm
-
-from src.get_model_info.hardocded_metadata.flags import DO_NOT_SUBMIT_MODELS, FLAGGED_MODELS
-from src.get_model_info.hardocded_metadata.types import MODEL_TYPE_METADATA, ModelType, model_type_from_str
-from src.get_model_info.utils import AutoEvalColumn, model_hyperlink
-
-api = HfApi(token=os.environ.get("H4_TOKEN", None))
-
-
-def get_model_metadata(leaderboard_data: List[dict]):
-    for model_data in tqdm(leaderboard_data):
-        request_files = os.path.join(
-            "eval-queue",
-            model_data["model_name_for_query"] + "_eval_request_*" + ".json",
-        )
-        request_files = glob.glob(request_files)
-
-        # Select correct request file (precision)
-        request_file = ""
-        if len(request_files) == 1:
-            request_file = request_files[0]
-        elif len(request_files) > 1:
-            request_files = sorted(request_files, reverse=True)
-            for tmp_request_file in request_files:
-                with open(tmp_request_file, "r") as f:
-                    req_content = json.load(f)
-                    if (
-                        req_content["status"] in ["FINISHED", "PENDING_NEW_EVAL"]
-                        and req_content["precision"] == model_data["Precision"].split(".")[-1]
-                    ):
-                        request_file = tmp_request_file
-
-        try:
-            with open(request_file, "r") as f:
-                request = json.load(f)
-            model_type = model_type_from_str(request.get("model_type", ""))
-            model_data[AutoEvalColumn.model_type.name] = model_type.value.name
-            model_data[AutoEvalColumn.model_type_symbol.name] = model_type.value.symbol  # + ("🔺" if is_delta else "")
-            model_data[AutoEvalColumn.license.name] = request.get("license", "?")
-            model_data[AutoEvalColumn.likes.name] = request.get("likes", 0)
-            model_data[AutoEvalColumn.params.name] = request.get("params", 0)
-        except Exception:
-            print(f"Could not find request file for {model_data['model_name_for_query']}")
-
-            if model_data["model_name_for_query"] in MODEL_TYPE_METADATA:
-                model_data[AutoEvalColumn.model_type.name] = MODEL_TYPE_METADATA[
-                    model_data["model_name_for_query"]
-                ].value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = MODEL_TYPE_METADATA[
-                    model_data["model_name_for_query"]
-                ].value.symbol  # + ("🔺" if is_delta else "")
-            else:
-                model_data[AutoEvalColumn.model_type.name] = ModelType.Unknown.value.name
-                model_data[AutoEvalColumn.model_type_symbol.name] = ModelType.Unknown.value.symbol
-
-            # if we cannot find a request file, set license and likes to unknown
-            model_data[AutoEvalColumn.license.name] =  "?"
-            model_data[AutoEvalColumn.likes.name] = 0
-            model_data[AutoEvalColumn.params.name] =  0
-
-
-def flag_models(leaderboard_data: List[dict]):
-    for model_data in leaderboard_data:
-        if model_data["model_name_for_query"] in FLAGGED_MODELS:
-            issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
-            issue_link = model_hyperlink(
-                FLAGGED_MODELS[model_data["model_name_for_query"]],
-                f"See discussion #{issue_num}",
-            )
-            model_data[
-                AutoEvalColumn.model.name
-            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
-
-
-def remove_forbidden_models(leaderboard_data: List[dict]):
-    indices_to_remove = []
-    for ix, model in enumerate(leaderboard_data):
-        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
-            indices_to_remove.append(ix)
-
-    for ix in reversed(indices_to_remove):
-        leaderboard_data.pop(ix)
-    return leaderboard_data
-
-
-def apply_metadata(leaderboard_data: List[dict]):
-    leaderboard_data = remove_forbidden_models(leaderboard_data)
-    get_model_metadata(leaderboard_data)
-    flag_models(leaderboard_data)

src/get_model_info/get_metadata_from_hub.py

@@ -1,19 +0,0 @@
-import re
-from huggingface_hub.hf_api import ModelInfo
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
-    size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
-    try:
-        model_size = round(model_info.safetensors["total"] / 1e9, 3)
-    except AttributeError:
-        try:
-            size_match = re.search(size_pattern, model_info.modelId.lower())
-            model_size = size_match.group(0)
-            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
-        except AttributeError:
-            return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
-    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
-    model_size = size_factor * model_size
-    return model_size

src/get_model_info/hardocded_metadata/types.py

@@ -1,555 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-from typing import Dict
-
-
-@dataclass
-class ModelInfo:
-    name: str
-    symbol: str  # emoji
-
-
-class ModelType(Enum):
-    PT = ModelInfo(name="pretrained", symbol="🟢")
-    FT = ModelInfo(name="fine-tuned", symbol="🔶")
-    IFT = ModelInfo(name="instruction-tuned", symbol="⭕")
-    RL = ModelInfo(name="RL-tuned", symbol="🟦")
-    Unknown = ModelInfo(name="", symbol="?")
-
-    def to_str(self, separator=" "):
-        return f"{self.value.symbol}{separator}{self.value.name}"
-
-
-MODEL_TYPE_METADATA: Dict[str, ModelType] = {
-    "tiiuae/falcon-180B": ModelType.PT,
-    "tiiuae/falcon-180B-chat": ModelType.RL,
-    "microsoft/phi-1_5": ModelType.PT,
-    "Qwen/Qwen-7B": ModelType.PT,
-    "Qwen/Qwen-7B-Chat": ModelType.RL,
-    "notstoic/PygmalionCoT-7b": ModelType.IFT,
-    "aisquared/dlite-v1-355m": ModelType.IFT,
-    "aisquared/dlite-v1-1_5b": ModelType.IFT,
-    "aisquared/dlite-v1-774m": ModelType.IFT,
-    "aisquared/dlite-v1-124m": ModelType.IFT,
-    "aisquared/chopt-2_7b": ModelType.IFT,
-    "aisquared/dlite-v2-124m": ModelType.IFT,
-    "aisquared/dlite-v2-774m": ModelType.IFT,
-    "aisquared/dlite-v2-1_5b": ModelType.IFT,
-    "aisquared/chopt-1_3b": ModelType.IFT,
-    "aisquared/dlite-v2-355m": ModelType.IFT,
-    "augtoma/qCammel-13": ModelType.IFT,
-    "Aspik101/Llama-2-7b-hf-instruct-pl-lora_unload": ModelType.IFT,
-    "Aspik101/vicuna-7b-v1.3-instruct-pl-lora_unload": ModelType.IFT,
-    "TheBloke/alpaca-lora-65B-HF": ModelType.FT,
-    "TheBloke/tulu-7B-fp16": ModelType.IFT,
-    "TheBloke/guanaco-7B-HF": ModelType.FT,
-    "TheBloke/koala-7B-HF": ModelType.FT,
-    "TheBloke/wizardLM-7B-HF": ModelType.IFT,
-    "TheBloke/airoboros-13B-HF": ModelType.IFT,
-    "TheBloke/koala-13B-HF": ModelType.FT,
-    "TheBloke/Wizard-Vicuna-7B-Uncensored-HF": ModelType.FT,
-    "TheBloke/dromedary-65b-lora-HF": ModelType.IFT,
-    "TheBloke/wizardLM-13B-1.0-fp16": ModelType.IFT,
-    "TheBloke/WizardLM-13B-V1-1-SuperHOT-8K-fp16": ModelType.FT,
-    "TheBloke/Wizard-Vicuna-30B-Uncensored-fp16": ModelType.FT,
-    "TheBloke/wizard-vicuna-13B-HF": ModelType.IFT,
-    "TheBloke/UltraLM-13B-fp16": ModelType.IFT,
-    "TheBloke/OpenAssistant-FT-7-Llama-30B-HF": ModelType.FT,
-    "TheBloke/vicuna-13B-1.1-HF": ModelType.IFT,
-    "TheBloke/guanaco-13B-HF": ModelType.FT,
-    "TheBloke/guanaco-65B-HF": ModelType.FT,
-    "TheBloke/airoboros-7b-gpt4-fp16": ModelType.IFT,
-    "TheBloke/llama-30b-supercot-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/Llama-2-13B-fp16": ModelType.PT,
-    "TheBloke/llama-2-70b-Guanaco-QLoRA-fp16": ModelType.FT,
-    "TheBloke/landmark-attention-llama7b-fp16": ModelType.IFT,
-    "TheBloke/Planner-7B-fp16": ModelType.IFT,
-    "TheBloke/Wizard-Vicuna-13B-Uncensored-HF": ModelType.FT,
-    "TheBloke/gpt4-alpaca-lora-13B-HF": ModelType.IFT,
-    "TheBloke/gpt4-x-vicuna-13B-HF": ModelType.IFT,
-    "TheBloke/gpt4-alpaca-lora_mlp-65B-HF": ModelType.IFT,
-    "TheBloke/tulu-13B-fp16": ModelType.IFT,
-    "TheBloke/VicUnlocked-alpaca-65B-QLoRA-fp16": ModelType.IFT,
-    "TheBloke/Llama-2-70B-fp16": ModelType.IFT,
-    "TheBloke/WizardLM-30B-fp16": ModelType.IFT,
-    "TheBloke/robin-13B-v2-fp16": ModelType.FT,
-    "TheBloke/robin-33B-v2-fp16": ModelType.FT,
-    "TheBloke/Vicuna-13B-CoT-fp16": ModelType.IFT,
-    "TheBloke/Vicuna-33B-1-3-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/Wizard-Vicuna-30B-Superhot-8K-fp16": ModelType.FT,
-    "TheBloke/Nous-Hermes-13B-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/GPlatty-30B-SuperHOT-8K-fp16": ModelType.FT,
-    "TheBloke/CAMEL-33B-Combined-Data-SuperHOT-8K-fp16": ModelType.IFT,
-    "TheBloke/Chinese-Alpaca-33B-SuperHOT-8K-fp16": ModelType.IFT,
-    "jphme/orca_mini_v2_ger_7b": ModelType.IFT,
-    "Ejafa/vicuna_7B_vanilla_1.1": ModelType.FT,
-    "kevinpro/Vicuna-13B-CoT": ModelType.IFT,
-    "AlekseyKorshuk/pygmalion-6b-vicuna-chatml": ModelType.FT,
-    "AlekseyKorshuk/chatml-pyg-v1": ModelType.FT,
-    "concedo/Vicuzard-30B-Uncensored": ModelType.FT,
-    "concedo/OPT-19M-ChatSalad": ModelType.FT,
-    "concedo/Pythia-70M-ChatSalad": ModelType.FT,
-    "digitous/13B-HyperMantis": ModelType.IFT,
-    "digitous/Adventien-GPTJ": ModelType.FT,
-    "digitous/Alpacino13b": ModelType.IFT,
-    "digitous/GPT-R": ModelType.IFT,
-    "digitous/Javelin-R": ModelType.IFT,
-    "digitous/Javalion-GPTJ": ModelType.IFT,
-    "digitous/Javalion-R": ModelType.IFT,
-    "digitous/Skegma-GPTJ": ModelType.FT,
-    "digitous/Alpacino30b": ModelType.IFT,
-    "digitous/Janin-GPTJ": ModelType.FT,
-    "digitous/Janin-R": ModelType.FT,
-    "digitous/Javelin-GPTJ": ModelType.FT,
-    "SaylorTwift/gpt2_test": ModelType.PT,
-    "anton-l/gpt-j-tiny-random": ModelType.FT,
-    "Andron00e/YetAnother_Open-Llama-3B-LoRA-OpenOrca": ModelType.FT,
-    "Lazycuber/pyg-instruct-wizardlm": ModelType.FT,
-    "Lazycuber/Janemalion-6B": ModelType.FT,
-    "IDEA-CCNL/Ziya-LLaMA-13B-Pretrain-v1": ModelType.FT,
-    "IDEA-CCNL/Ziya-LLaMA-13B-v1": ModelType.IFT,
-    "dsvv-cair/alpaca-cleaned-llama-30b-bf16": ModelType.FT,
-    "gpt2-medium": ModelType.PT,
-    "camel-ai/CAMEL-13B-Combined-Data": ModelType.IFT,
-    "camel-ai/CAMEL-13B-Role-Playing-Data": ModelType.FT,
-    "camel-ai/CAMEL-33B-Combined-Data": ModelType.IFT,
-    "PygmalionAI/pygmalion-6b": ModelType.FT,
-    "PygmalionAI/metharme-1.3b": ModelType.IFT,
-    "PygmalionAI/pygmalion-1.3b": ModelType.FT,
-    "PygmalionAI/pygmalion-350m": ModelType.FT,
-    "PygmalionAI/pygmalion-2.7b": ModelType.FT,
-    "medalpaca/medalpaca-7b": ModelType.FT,
-    "lilloukas/Platypus-30B": ModelType.IFT,
-    "lilloukas/GPlatty-30B": ModelType.FT,
-    "mncai/chatdoctor": ModelType.FT,
-    "chaoyi-wu/MedLLaMA_13B": ModelType.FT,
-    "LoupGarou/WizardCoder-Guanaco-15B-V1.0": ModelType.IFT,
-    "LoupGarou/WizardCoder-Guanaco-15B-V1.1": ModelType.FT,
-    "hakurei/instruct-12b": ModelType.IFT,
-    "hakurei/lotus-12B": ModelType.FT,
-    "shibing624/chinese-llama-plus-13b-hf": ModelType.IFT,
-    "shibing624/chinese-alpaca-plus-7b-hf": ModelType.IFT,
-    "shibing624/chinese-alpaca-plus-13b-hf": ModelType.IFT,
-    "mosaicml/mpt-7b-instruct": ModelType.IFT,
-    "mosaicml/mpt-30b-chat": ModelType.IFT,
-    "mosaicml/mpt-7b-storywriter": ModelType.FT,
-    "mosaicml/mpt-30b-instruct": ModelType.IFT,
-    "mosaicml/mpt-7b-chat": ModelType.IFT,
-    "mosaicml/mpt-30b": ModelType.PT,
-    "Corianas/111m": ModelType.IFT,
-    "Corianas/Quokka_1.3b": ModelType.IFT,
-    "Corianas/256_5epoch": ModelType.FT,
-    "Corianas/Quokka_256m": ModelType.IFT,
-    "Corianas/Quokka_590m": ModelType.IFT,
-    "Corianas/gpt-j-6B-Dolly": ModelType.FT,
-    "Corianas/Quokka_2.7b": ModelType.IFT,
-    "cyberagent/open-calm-7b": ModelType.FT,
-    "Aspik101/Nous-Hermes-13b-pl-lora_unload": ModelType.IFT,
-    "THUDM/chatglm2-6b": ModelType.IFT,
-    "MetaIX/GPT4-X-Alpasta-30b": ModelType.IFT,
-    "NYTK/PULI-GPTrio": ModelType.PT,
-    "EleutherAI/pythia-1.3b": ModelType.PT,
-    "EleutherAI/pythia-2.8b-deduped": ModelType.PT,
-    "EleutherAI/gpt-neo-125m": ModelType.PT,
-    "EleutherAI/pythia-160m": ModelType.PT,
-    "EleutherAI/gpt-neo-2.7B": ModelType.PT,
-    "EleutherAI/pythia-1b-deduped": ModelType.PT,
-    "EleutherAI/pythia-6.7b": ModelType.PT,
-    "EleutherAI/pythia-70m-deduped": ModelType.PT,
-    "EleutherAI/gpt-neox-20b": ModelType.PT,
-    "EleutherAI/pythia-1.4b-deduped": ModelType.PT,
-    "EleutherAI/pythia-2.7b": ModelType.PT,
-    "EleutherAI/pythia-6.9b-deduped": ModelType.PT,
-    "EleutherAI/pythia-70m": ModelType.PT,
-    "EleutherAI/gpt-j-6b": ModelType.PT,
-    "EleutherAI/pythia-12b-deduped": ModelType.PT,
-    "EleutherAI/gpt-neo-1.3B": ModelType.PT,
-    "EleutherAI/pythia-410m-deduped": ModelType.PT,
-    "EleutherAI/pythia-160m-deduped": ModelType.PT,
-    "EleutherAI/polyglot-ko-12.8b": ModelType.PT,
-    "EleutherAI/pythia-12b": ModelType.PT,
-    "roneneldan/TinyStories-33M": ModelType.PT,
-    "roneneldan/TinyStories-28M": ModelType.PT,
-    "roneneldan/TinyStories-1M": ModelType.PT,
-    "roneneldan/TinyStories-8M": ModelType.PT,
-    "roneneldan/TinyStories-3M": ModelType.PT,
-    "jerryjalapeno/nart-100k-7b": ModelType.FT,
-    "lmsys/vicuna-13b-v1.3": ModelType.IFT,
-    "lmsys/vicuna-7b-v1.3": ModelType.IFT,
-    "lmsys/vicuna-13b-v1.1": ModelType.IFT,
-    "lmsys/vicuna-13b-delta-v1.1": ModelType.IFT,
-    "lmsys/vicuna-7b-delta-v1.1": ModelType.IFT,
-    "abhiramtirumala/DialoGPT-sarcastic-medium": ModelType.FT,
-    "haonan-li/bactrian-x-llama-13b-merged": ModelType.IFT,
-    "Gryphe/MythoLogic-13b": ModelType.IFT,
-    "Gryphe/MythoBoros-13b": ModelType.IFT,
-    "pillowtalks-ai/delta13b": ModelType.FT,
-    "wannaphong/openthaigpt-0.1.0-beta-full-model_for_open_llm_leaderboard": ModelType.FT,
-    "bigscience/bloom-7b1": ModelType.PT,
-    "bigcode/tiny_starcoder_py": ModelType.PT,
-    "bigcode/starcoderplus": ModelType.FT,
-    "bigcode/gpt_bigcode-santacoder": ModelType.PT,
-    "bigcode/starcoder": ModelType.PT,
-    "Open-Orca/OpenOrca-Preview1-13B": ModelType.IFT,
-    "microsoft/DialoGPT-large": ModelType.FT,
-    "microsoft/DialoGPT-small": ModelType.FT,
-    "microsoft/DialoGPT-medium": ModelType.FT,
-    "microsoft/CodeGPT-small-py": ModelType.FT,
-    "Tincando/fiction_story_generator": ModelType.FT,
-    "Pirr/pythia-13b-deduped-green_devil": ModelType.FT,
-    "Aeala/GPT4-x-AlpacaDente2-30b": ModelType.FT,
-    "Aeala/GPT4-x-AlpacaDente-30b": ModelType.FT,
-    "Aeala/GPT4-x-Alpasta-13b": ModelType.FT,
-    "Aeala/VicUnlocked-alpaca-30b": ModelType.IFT,
-    "Tap-M/Luna-AI-Llama2-Uncensored": ModelType.FT,
-    "illuin/test-custom-llama": ModelType.FT,
-    "dvruette/oasst-llama-13b-2-epochs": ModelType.FT,
-    "dvruette/oasst-gpt-neox-20b-1000-steps": ModelType.FT,
-    "dvruette/llama-13b-pretrained-dropout": ModelType.PT,
-    "dvruette/llama-13b-pretrained": ModelType.PT,
-    "dvruette/llama-13b-pretrained-sft-epoch-1": ModelType.FT,
-    "dvruette/llama-13b-pretrained-sft-do2": ModelType.FT,
-    "dvruette/oasst-gpt-neox-20b-3000-steps": ModelType.FT,
-    "dvruette/oasst-pythia-12b-pretrained-sft": ModelType.FT,
-    "dvruette/oasst-pythia-6.9b-4000-steps": ModelType.FT,
-    "dvruette/gpt-neox-20b-full-precision": ModelType.FT,
-    "dvruette/oasst-llama-13b-1000-steps": ModelType.FT,
-    "openlm-research/open_llama_7b_700bt_preview": ModelType.PT,
-    "openlm-research/open_llama_7b": ModelType.PT,
-    "openlm-research/open_llama_7b_v2": ModelType.PT,
-    "openlm-research/open_llama_3b": ModelType.PT,
-    "openlm-research/open_llama_13b": ModelType.PT,
-    "openlm-research/open_llama_3b_v2": ModelType.PT,
-    "PocketDoc/Dans-PileOfSets-Mk1-llama-13b-merged": ModelType.IFT,
-    "GeorgiaTechResearchInstitute/galpaca-30b": ModelType.IFT,
-    "GeorgiaTechResearchInstitute/starcoder-gpteacher-code-instruct": ModelType.IFT,
-    "databricks/dolly-v2-7b": ModelType.IFT,
-    "databricks/dolly-v2-3b": ModelType.IFT,
-    "databricks/dolly-v2-12b": ModelType.IFT,
-    "Rachneet/gpt2-xl-alpaca": ModelType.FT,
-    "Locutusque/gpt2-conversational-or-qa": ModelType.FT,
-    "psyche/kogpt": ModelType.FT,
-    "NbAiLab/nb-gpt-j-6B-alpaca": ModelType.IFT,
-    "Mikael110/llama-2-7b-guanaco-fp16": ModelType.FT,
-    "Mikael110/llama-2-13b-guanaco-fp16": ModelType.FT,
-    "Fredithefish/CrimsonPajama": ModelType.IFT,
-    "Fredithefish/RedPajama-INCITE-Chat-3B-ShareGPT-11K": ModelType.FT,
-    "Fredithefish/ScarletPajama-3B-HF": ModelType.FT,
-    "Fredithefish/RedPajama-INCITE-Chat-3B-Instruction-Tuning-with-GPT-4": ModelType.IFT,
-    "acrastt/RedPajama-INCITE-Chat-Instruct-3B-V1": ModelType.IFT,
-    "eachadea/vicuna-13b-1.1": ModelType.FT,
-    "eachadea/vicuna-7b-1.1": ModelType.FT,
-    "eachadea/vicuna-13b": ModelType.FT,
-    "openaccess-ai-collective/wizard-mega-13b": ModelType.IFT,
-    "openaccess-ai-collective/manticore-13b": ModelType.IFT,
-    "openaccess-ai-collective/manticore-30b-chat-pyg-alpha": ModelType.IFT,
-    "openaccess-ai-collective/minotaur-13b": ModelType.IFT,
-    "openaccess-ai-collective/minotaur-13b-fixed": ModelType.IFT,
-    "openaccess-ai-collective/hippogriff-30b-chat": ModelType.IFT,
-    "openaccess-ai-collective/manticore-13b-chat-pyg": ModelType.IFT,
-    "pythainlp/wangchanglm-7.5B-sft-enth": ModelType.IFT,
-    "pythainlp/wangchanglm-7.5B-sft-en-sharded": ModelType.IFT,
-    "euclaise/gpt-neox-122m-minipile-digits": ModelType.FT,
-    "stabilityai/StableBeluga1-Delta": ModelType.IFT,
-    "stabilityai/stablelm-tuned-alpha-7b": ModelType.IFT,
-    "stabilityai/StableBeluga2": ModelType.IFT,
-    "stabilityai/StableBeluga-13B": ModelType.IFT,
-    "stabilityai/StableBeluga-7B": ModelType.IFT,
-    "stabilityai/stablelm-base-alpha-7b": ModelType.PT,
-    "stabilityai/stablelm-base-alpha-3b": ModelType.PT,
-    "stabilityai/stablelm-tuned-alpha-3b": ModelType.IFT,
-    "alibidaran/medical_transcription_generator": ModelType.FT,
-    "CalderaAI/30B-Lazarus": ModelType.IFT,
-    "CalderaAI/13B-BlueMethod": ModelType.IFT,
-    "CalderaAI/13B-Ouroboros": ModelType.IFT,
-    "KoboldAI/OPT-13B-Erebus": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Janeway": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Shinen": ModelType.FT,
-    "KoboldAI/fairseq-dense-2.7B": ModelType.PT,
-    "KoboldAI/OPT-6B-nerys-v2": ModelType.FT,
-    "KoboldAI/GPT-NeoX-20B-Skein": ModelType.FT,
-    "KoboldAI/PPO_Pygway-6b-Mix": ModelType.FT,
-    "KoboldAI/fairseq-dense-6.7B": ModelType.PT,
-    "KoboldAI/fairseq-dense-125M": ModelType.PT,
-    "KoboldAI/OPT-13B-Nerybus-Mix": ModelType.FT,
-    "KoboldAI/OPT-2.7B-Erebus": ModelType.FT,
-    "KoboldAI/OPT-350M-Nerys-v2": ModelType.FT,
-    "KoboldAI/OPT-2.7B-Nerys-v2": ModelType.FT,
-    "KoboldAI/OPT-2.7B-Nerybus-Mix": ModelType.FT,
-    "KoboldAI/OPT-13B-Nerys-v2": ModelType.FT,
-    "KoboldAI/GPT-NeoX-20B-Erebus": ModelType.FT,
-    "KoboldAI/OPT-6.7B-Erebus": ModelType.FT,
-    "KoboldAI/fairseq-dense-355M": ModelType.PT,
-    "KoboldAI/OPT-6.7B-Nerybus-Mix": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Adventure": ModelType.FT,
-    "KoboldAI/OPT-350M-Erebus": ModelType.FT,
-    "KoboldAI/GPT-J-6B-Skein": ModelType.FT,
-    "KoboldAI/OPT-30B-Erebus": ModelType.FT,
-    "klosax/pythia-160m-deduped-step92k-193bt": ModelType.PT,
-    "klosax/open_llama_3b_350bt_preview": ModelType.PT,
-    "klosax/openllama-3b-350bt": ModelType.PT,
-    "klosax/pythia-70m-deduped-step44k-92bt": ModelType.PT,
-    "klosax/open_llama_13b_600bt_preview": ModelType.PT,
-    "klosax/open_llama_7b_400bt_preview": ModelType.PT,
-    "kfkas/Llama-2-ko-7b-Chat": ModelType.IFT,
-    "WeOpenML/Alpaca-7B-v1": ModelType.IFT,
-    "WeOpenML/PandaLM-Alpaca-7B-v1": ModelType.IFT,
-    "TFLai/gpt2-turkish-uncased": ModelType.FT,
-    "ehartford/WizardLM-13B-Uncensored": ModelType.IFT,
-    "ehartford/dolphin-llama-13b": ModelType.IFT,
-    "ehartford/Wizard-Vicuna-30B-Uncensored": ModelType.FT,
-    "ehartford/WizardLM-30B-Uncensored": ModelType.IFT,
-    "ehartford/Wizard-Vicuna-13B-Uncensored": ModelType.FT,
-    "ehartford/WizardLM-7B-Uncensored": ModelType.IFT,
-    "ehartford/based-30b": ModelType.FT,
-    "ehartford/Wizard-Vicuna-7B-Uncensored": ModelType.FT,
-    "wahaha1987/llama_7b_sharegpt94k_fastchat": ModelType.FT,
-    "wahaha1987/llama_13b_sharegpt94k_fastchat": ModelType.FT,
-    "OpenAssistant/oasst-sft-1-pythia-12b": ModelType.FT,
-    "OpenAssistant/stablelm-7b-sft-v7-epoch-3": ModelType.IFT,
-    "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5": ModelType.FT,
-    "OpenAssistant/pythia-12b-sft-v8-2.5k-steps": ModelType.IFT,
-    "OpenAssistant/pythia-12b-sft-v8-7k-steps": ModelType.IFT,
-    "OpenAssistant/pythia-12b-pre-v8-12.5k-steps": ModelType.IFT,
-    "OpenAssistant/llama2-13b-orca-8k-3319": ModelType.IFT,
-    "junelee/wizard-vicuna-13b": ModelType.FT,
-    "BreadAi/gpt-YA-1-1_160M": ModelType.PT,
-    "BreadAi/MuseCan": ModelType.PT,
-    "BreadAi/MusePy-1-2": ModelType.PT,
-    "BreadAi/DiscordPy": ModelType.PT,
-    "BreadAi/PM_modelV2": ModelType.PT,
-    "BreadAi/gpt-Youtube": ModelType.PT,
-    "BreadAi/StoryPy": ModelType.FT,
-    "julianweng/Llama-2-7b-chat-orcah": ModelType.FT,
-    "AGI-inc/lora_moe_7b_baseline": ModelType.FT,
-    "AGI-inc/lora_moe_7b": ModelType.FT,
-    "togethercomputer/GPT-NeoXT-Chat-Base-20B": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Chat-7B-v0.1": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Instruct-7B-v0.1": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-7B-Base": ModelType.PT,
-    "togethercomputer/RedPajama-INCITE-7B-Instruct": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Base-3B-v1": ModelType.PT,
-    "togethercomputer/Pythia-Chat-Base-7B": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Base-7B-v0.1": ModelType.PT,
-    "togethercomputer/GPT-JT-6B-v1": ModelType.IFT,
-    "togethercomputer/GPT-JT-6B-v0": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Chat-3B-v1": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-7B-Chat": ModelType.IFT,
-    "togethercomputer/RedPajama-INCITE-Instruct-3B-v1": ModelType.IFT,
-    "Writer/camel-5b-hf": ModelType.IFT,
-    "Writer/palmyra-base": ModelType.PT,
-    "MBZUAI/LaMini-GPT-1.5B": ModelType.IFT,
-    "MBZUAI/lamini-cerebras-111m": ModelType.IFT,
-    "MBZUAI/lamini-neo-1.3b": ModelType.IFT,
-    "MBZUAI/lamini-cerebras-1.3b": ModelType.IFT,
-    "MBZUAI/lamini-cerebras-256m": ModelType.IFT,
-    "MBZUAI/LaMini-GPT-124M": ModelType.IFT,
-    "MBZUAI/lamini-neo-125m": ModelType.IFT,
-    "TehVenom/DiffMerge-DollyGPT-Pygmalion": ModelType.FT,
-    "TehVenom/PPO_Shygmalion-6b": ModelType.FT,
-    "TehVenom/Dolly_Shygmalion-6b-Dev_V8P2": ModelType.FT,
-    "TehVenom/Pygmalion_AlpacaLora-7b": ModelType.FT,
-    "TehVenom/PPO_Pygway-V8p4_Dev-6b": ModelType.FT,
-    "TehVenom/Dolly_Malion-6b": ModelType.FT,
-    "TehVenom/PPO_Shygmalion-V8p4_Dev-6b": ModelType.FT,
-    "TehVenom/ChanMalion": ModelType.FT,
-    "TehVenom/GPT-J-Pyg_PPO-6B": ModelType.IFT,
-    "TehVenom/Pygmalion-13b-Merged": ModelType.FT,
-    "TehVenom/Metharme-13b-Merged": ModelType.IFT,
-    "TehVenom/Dolly_Shygmalion-6b": ModelType.FT,
-    "TehVenom/GPT-J-Pyg_PPO-6B-Dev-V8p4": ModelType.IFT,
-    "georgesung/llama2_7b_chat_uncensored": ModelType.FT,
-    "vicgalle/gpt2-alpaca": ModelType.IFT,
-    "vicgalle/alpaca-7b": ModelType.FT,
-    "vicgalle/gpt2-alpaca-gpt4": ModelType.IFT,
-    "facebook/opt-350m": ModelType.PT,
-    "facebook/opt-125m": ModelType.PT,
-    "facebook/xglm-4.5B": ModelType.PT,
-    "facebook/opt-2.7b": ModelType.PT,
-    "facebook/opt-6.7b": ModelType.PT,
-    "facebook/galactica-30b": ModelType.PT,
-    "facebook/opt-13b": ModelType.PT,
-    "facebook/opt-66b": ModelType.PT,
-    "facebook/xglm-7.5B": ModelType.PT,
-    "facebook/xglm-564M": ModelType.PT,
-    "facebook/opt-30b": ModelType.PT,
-    "golaxy/gogpt-7b": ModelType.FT,
-    "golaxy/gogpt2-7b": ModelType.FT,
-    "golaxy/gogpt-7b-bloom": ModelType.FT,
-    "golaxy/gogpt-3b-bloom": ModelType.FT,
-    "psmathur/orca_mini_v2_7b": ModelType.IFT,
-    "psmathur/orca_mini_7b": ModelType.IFT,
-    "psmathur/orca_mini_3b": ModelType.IFT,
-    "psmathur/orca_mini_v2_13b": ModelType.IFT,
-    "gpt2-xl": ModelType.PT,
-    "lxe/Cerebras-GPT-2.7B-Alpaca-SP": ModelType.FT,
-    "Monero/Manticore-13b-Chat-Pyg-Guanaco": ModelType.FT,
-    "Monero/WizardLM-Uncensored-SuperCOT-StoryTelling-30b": ModelType.IFT,
-    "Monero/WizardLM-13b-OpenAssistant-Uncensored": ModelType.IFT,
-    "Monero/WizardLM-30B-Uncensored-Guanaco-SuperCOT-30b": ModelType.IFT,
-    "jzjiao/opt-1.3b-rlhf": ModelType.FT,
-    "HuggingFaceH4/starchat-beta": ModelType.IFT,
-    "KnutJaegersberg/gpt-2-xl-EvolInstruct": ModelType.IFT,
-    "KnutJaegersberg/megatron-GPT-2-345m-EvolInstruct": ModelType.IFT,
-    "KnutJaegersberg/galactica-orca-wizardlm-1.3b": ModelType.IFT,
-    "openchat/openchat_8192": ModelType.IFT,
-    "openchat/openchat_v2": ModelType.IFT,
-    "openchat/openchat_v2_w": ModelType.IFT,
-    "ausboss/llama-13b-supercot": ModelType.IFT,
-    "ausboss/llama-30b-supercot": ModelType.IFT,
-    "Neko-Institute-of-Science/metharme-7b": ModelType.IFT,
-    "Neko-Institute-of-Science/pygmalion-7b": ModelType.FT,
-    "SebastianSchramm/Cerebras-GPT-111M-instruction": ModelType.IFT,
-    "victor123/WizardLM-13B-1.0": ModelType.IFT,
-    "OpenBuddy/openbuddy-openllama-13b-v7-fp16": ModelType.FT,
-    "OpenBuddy/openbuddy-llama2-13b-v8.1-fp16": ModelType.FT,
-    "OpenBuddyEA/openbuddy-llama-30b-v7.1-bf16": ModelType.FT,
-    "baichuan-inc/Baichuan-7B": ModelType.PT,
-    "tiiuae/falcon-40b-instruct": ModelType.IFT,
-    "tiiuae/falcon-40b": ModelType.PT,
-    "tiiuae/falcon-7b": ModelType.PT,
-    "YeungNLP/firefly-llama-13b": ModelType.FT,
-    "YeungNLP/firefly-llama-13b-v1.2": ModelType.FT,
-    "YeungNLP/firefly-llama2-13b": ModelType.FT,
-    "YeungNLP/firefly-ziya-13b": ModelType.FT,
-    "shaohang/Sparse0.5_OPT-1.3": ModelType.FT,
-    "xzuyn/Alpacino-SuperCOT-13B": ModelType.IFT,
-    "xzuyn/MedicWizard-7B": ModelType.FT,
-    "xDAN-AI/xDAN_13b_l2_lora": ModelType.FT,
-    "beomi/KoAlpaca-Polyglot-5.8B": ModelType.FT,
-    "beomi/llama-2-ko-7b": ModelType.IFT,
-    "Salesforce/codegen-6B-multi": ModelType.PT,
-    "Salesforce/codegen-16B-nl": ModelType.PT,
-    "Salesforce/codegen-6B-nl": ModelType.PT,
-    "ai-forever/rugpt3large_based_on_gpt2": ModelType.FT,
-    "gpt2-large": ModelType.PT,
-    "frank098/orca_mini_3b_juniper": ModelType.FT,
-    "frank098/WizardLM_13B_juniper": ModelType.FT,
-    "FPHam/Free_Sydney_13b_HF": ModelType.FT,
-    "huggingface/llama-13b": ModelType.PT,
-    "huggingface/llama-7b": ModelType.PT,
-    "huggingface/llama-65b": ModelType.PT,
-    "huggingface/llama-30b": ModelType.PT,
-    "Henk717/chronoboros-33B": ModelType.IFT,
-    "jondurbin/airoboros-13b-gpt4-1.4": ModelType.IFT,
-    "jondurbin/airoboros-7b": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.1": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.2": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.3": ModelType.IFT,
-    "jondurbin/airoboros-7b-gpt4-1.4": ModelType.IFT,
-    "jondurbin/airoboros-l2-7b-gpt4-1.4.1": ModelType.IFT,
-    "jondurbin/airoboros-l2-13b-gpt4-1.4.1": ModelType.IFT,
-    "jondurbin/airoboros-l2-70b-gpt4-1.4.1": ModelType.IFT,
-    "jondurbin/airoboros-13b": ModelType.IFT,
-    "jondurbin/airoboros-33b-gpt4-1.4": ModelType.IFT,
-    "jondurbin/airoboros-33b-gpt4-1.2": ModelType.IFT,
-    "jondurbin/airoboros-65b-gpt4-1.2": ModelType.IFT,
-    "ariellee/SuperPlatty-30B": ModelType.IFT,
-    "danielhanchen/open_llama_3b_600bt_preview": ModelType.FT,
-    "cerebras/Cerebras-GPT-256M": ModelType.PT,
-    "cerebras/Cerebras-GPT-1.3B": ModelType.PT,
-    "cerebras/Cerebras-GPT-13B": ModelType.PT,
-    "cerebras/Cerebras-GPT-2.7B": ModelType.PT,
-    "cerebras/Cerebras-GPT-111M": ModelType.PT,
-    "cerebras/Cerebras-GPT-6.7B": ModelType.PT,
-    "Yhyu13/oasst-rlhf-2-llama-30b-7k-steps-hf": ModelType.RL,
-    "Yhyu13/llama-30B-hf-openassitant": ModelType.FT,
-    "NousResearch/Nous-Hermes-Llama2-13b": ModelType.IFT,
-    "NousResearch/Nous-Hermes-llama-2-7b": ModelType.IFT,
-    "NousResearch/Redmond-Puffin-13B": ModelType.IFT,
-    "NousResearch/Nous-Hermes-13b": ModelType.IFT,
-    "project-baize/baize-v2-7b": ModelType.IFT,
-    "project-baize/baize-v2-13b": ModelType.IFT,
-    "LLMs/WizardLM-13B-V1.0": ModelType.FT,
-    "LLMs/AlpacaGPT4-7B-elina": ModelType.FT,
-    "wenge-research/yayi-7b": ModelType.FT,
-    "wenge-research/yayi-7b-llama2": ModelType.FT,
-    "wenge-research/yayi-13b-llama2": ModelType.FT,
-    "yhyhy3/open_llama_7b_v2_med_instruct": ModelType.IFT,
-    "llama-anon/instruct-13b": ModelType.IFT,
-    "huggingtweets/jerma985": ModelType.FT,
-    "huggingtweets/gladosystem": ModelType.FT,
-    "huggingtweets/bladeecity-jerma985": ModelType.FT,
-    "huggyllama/llama-13b": ModelType.PT,
-    "huggyllama/llama-65b": ModelType.PT,
-    "FabbriSimo01/Facebook_opt_1.3b_Quantized": ModelType.PT,
-    "upstage/Llama-2-70b-instruct": ModelType.IFT,
-    "upstage/Llama-2-70b-instruct-1024": ModelType.IFT,
-    "upstage/llama-65b-instruct": ModelType.IFT,
-    "upstage/llama-30b-instruct-2048": ModelType.IFT,
-    "upstage/llama-30b-instruct": ModelType.IFT,
-    "WizardLM/WizardLM-13B-1.0": ModelType.IFT,
-    "WizardLM/WizardLM-13B-V1.1": ModelType.IFT,
-    "WizardLM/WizardLM-13B-V1.2": ModelType.IFT,
-    "WizardLM/WizardLM-30B-V1.0": ModelType.IFT,
-    "WizardLM/WizardCoder-15B-V1.0": ModelType.IFT,
-    "gpt2": ModelType.PT,
-    "keyfan/vicuna-chinese-replication-v1.1": ModelType.IFT,
-    "nthngdy/pythia-owt2-70m-100k": ModelType.FT,
-    "nthngdy/pythia-owt2-70m-50k": ModelType.FT,
-    "quantumaikr/KoreanLM-hf": ModelType.FT,
-    "quantumaikr/open_llama_7b_hf": ModelType.FT,
-    "quantumaikr/QuantumLM-70B-hf": ModelType.IFT,
-    "MayaPH/FinOPT-Lincoln": ModelType.FT,
-    "MayaPH/FinOPT-Franklin": ModelType.FT,
-    "MayaPH/GodziLLa-30B": ModelType.IFT,
-    "MayaPH/GodziLLa-30B-plus": ModelType.IFT,
-    "MayaPH/FinOPT-Washington": ModelType.FT,
-    "ogimgio/gpt-neo-125m-neurallinguisticpioneers": ModelType.FT,
-    "layoric/llama-2-13b-code-alpaca": ModelType.FT,
-    "CobraMamba/mamba-gpt-3b": ModelType.FT,
-    "CobraMamba/mamba-gpt-3b-v2": ModelType.FT,
-    "CobraMamba/mamba-gpt-3b-v3": ModelType.FT,
-    "timdettmers/guanaco-33b-merged": ModelType.FT,
-    "elinas/chronos-33b": ModelType.IFT,
-    "heegyu/RedTulu-Uncensored-3B-0719": ModelType.IFT,
-    "heegyu/WizardVicuna-Uncensored-3B-0719": ModelType.IFT,
-    "heegyu/WizardVicuna-3B-0719": ModelType.IFT,
-    "meta-llama/Llama-2-7b-chat-hf": ModelType.RL,
-    "meta-llama/Llama-2-7b-hf": ModelType.PT,
-    "meta-llama/Llama-2-13b-chat-hf": ModelType.RL,
-    "meta-llama/Llama-2-13b-hf": ModelType.PT,
-    "meta-llama/Llama-2-70b-chat-hf": ModelType.RL,
-    "meta-llama/Llama-2-70b-hf": ModelType.PT,
-    "xhyi/PT_GPTNEO350_ATG": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-en-1024-20b": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-en-1024-open-llama-7b-preview-400bt": ModelType.FT,
-    "h2oai/h2ogpt-oig-oasst1-512-6_9b": ModelType.IFT,
-    "h2oai/h2ogpt-oasst1-512-12b": ModelType.IFT,
-    "h2oai/h2ogpt-oig-oasst1-256-6_9b": ModelType.IFT,
-    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt": ModelType.FT,
-    "h2oai/h2ogpt-oasst1-512-20b": ModelType.IFT,
-    "h2oai/h2ogpt-gm-oasst1-en-2048-open-llama-7b-preview-300bt-v2": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-en-1024-12b": ModelType.FT,
-    "h2oai/h2ogpt-gm-oasst1-multilang-1024-20b": ModelType.FT,
-    "bofenghuang/vigogne-13b-instruct": ModelType.IFT,
-    "bofenghuang/vigogne-13b-chat": ModelType.FT,
-    "bofenghuang/vigogne-2-7b-instruct": ModelType.IFT,
-    "bofenghuang/vigogne-7b-instruct": ModelType.IFT,
-    "bofenghuang/vigogne-7b-chat": ModelType.FT,
-    "Vmware/open-llama-7b-v2-open-instruct": ModelType.IFT,
-    "VMware/open-llama-0.7T-7B-open-instruct-v1.1": ModelType.IFT,
-    "ewof/koishi-instruct-3b": ModelType.IFT,
-    "gywy/llama2-13b-chinese-v1": ModelType.FT,
-    "GOAT-AI/GOAT-7B-Community": ModelType.FT,
-    "psyche/kollama2-7b": ModelType.FT,
-    "TheTravellingEngineer/llama2-7b-hf-guanaco": ModelType.FT,
-    "beaugogh/pythia-1.4b-deduped-sharegpt": ModelType.FT,
-    "augtoma/qCammel-70-x": ModelType.IFT,
-    "Lajonbot/Llama-2-7b-chat-hf-instruct-pl-lora_unload": ModelType.IFT,
-    "anhnv125/pygmalion-6b-roleplay": ModelType.FT,
-    "64bits/LexPodLM-13B": ModelType.FT,
-}
-
-
-def model_type_from_str(type):
-    if "fine-tuned" in type or "🔶" in type:
-        return ModelType.FT
-    if "pretrained" in type or "🟢" in type:
-        return ModelType.PT
-    if "RL-tuned" in type or "🟦" in type:
-        return ModelType.RL
-    if "instruction-tuned" in type or "⭕" in type:
-        return ModelType.IFT
-    return ModelType.Unknown

src/{get_model_info/hardocded_metadata/flags.py → leaderboard/filter_models.py}

@@ -1,3 +1,6 @@
+from src.display.formatting import model_hyperlink
+from src.display.utils import AutoEvalColumn
+
 # Models which have been flagged by users as being problematic for a reason or another
 # (Model name to forum discussion link)
 FLAGGED_MODELS = {
@@ -16,3 +19,32 @@ FLAGGED_MODELS = {
 DO_NOT_SUBMIT_MODELS = [
     "Voicelab/trurl-2-13b",  # trained on MMLU
 ]
+
+
+def flag_models(leaderboard_data: list[dict]):
+    for model_data in leaderboard_data:
+        if model_data["model_name_for_query"] in FLAGGED_MODELS:
+            issue_num = FLAGGED_MODELS[model_data["model_name_for_query"]].split("/")[-1]
+            issue_link = model_hyperlink(
+                FLAGGED_MODELS[model_data["model_name_for_query"]],
+                f"See discussion #{issue_num}",
+            )
+            model_data[
+                AutoEvalColumn.model.name
+            ] = f"{model_data[AutoEvalColumn.model.name]} has been flagged! {issue_link}"
+
+
+def remove_forbidden_models(leaderboard_data: list[dict]):
+    indices_to_remove = []
+    for ix, model in enumerate(leaderboard_data):
+        if model["model_name_for_query"] in DO_NOT_SUBMIT_MODELS:
+            indices_to_remove.append(ix)
+
+    for ix in reversed(indices_to_remove):
+        leaderboard_data.pop(ix)
+    return leaderboard_data
+
+
+def filter_models(leaderboard_data: list[dict]):
+    leaderboard_data = remove_forbidden_models(leaderboard_data)
+    flag_models(leaderboard_data)

src/leaderboard/read_evals.py

@@ -0,0 +1,197 @@
+import json
+import os
+import math
+import glob
+from dataclasses import dataclass
+from typing import Dict, List, Tuple
+
+import dateutil
+import numpy as np
+
+from src.display.utils import AutoEvalColumn, ModelType, Tasks
+from src.display.formatting import make_clickable_model
+from src.submission.check_validity import is_model_on_hub
+
+
+@dataclass
+class EvalResult:
+    eval_name: str
+    full_model: str
+    org: str
+    model: str
+    revision: str
+    results: dict
+    precision: str = ""
+    model_type: ModelType = ModelType.Unknown
+    weight_type: str = "Original"
+    license: str = "?"
+    likes: int = 0
+    num_params: int = 0
+    date: str = ""
+    still_on_hub: bool = False
+
+    @classmethod
+    def init_from_json_file(self, json_filepath):
+        with open(json_filepath) as fp:
+            data = json.load(fp)
+
+        # We manage the legacy config format
+        config = data.get("config", data.get("config_general", None))
+
+        # Precision
+        precision = config.get("model_dtype")
+        if precision == "None":
+            precision = "GPTQ"
+
+        # Get model and org
+        org_and_model = config.get("model_name", config.get("model_args", None))
+        org_and_model = org_and_model.split("/", 1)
+
+        if len(org_and_model) == 1:
+            org = None
+            model = org_and_model[0]
+            result_key = f"{model}_{precision}"
+        else:
+            org = org_and_model[0]
+            model = org_and_model[1]
+            result_key = f"{org}_{model}_{precision}"
+
+        still_on_hub = is_model_on_hub("/".join(org_and_model), config.get("model_sha", "main"), trust_remote_code=True)[0]
+
+        # Extract results available in this file (some results are split in several files)
+        results = {}
+        for task in Tasks:
+            task = task.value
+            # We skip old mmlu entries
+            wrong_mmlu_version = False
+            if task.benchmark == "hendrycksTest":
+                for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
+                    if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
+                        wrong_mmlu_version = True
+
+            if wrong_mmlu_version:
+                continue
+
+            # Some truthfulQA values are NaNs
+            if task.benchmark == "truthfulqa:mc" and task.benchmark in data["results"]:
+                if math.isnan(float(data["results"][task.benchmark][task.metric])):
+                    results[task.benchmark] = 0.0
+                    continue
+
+            # We average all scores of a given metric (mostly for mmlu)
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark in k])
+            if accs.size == 0 or any([acc is None for acc in accs]):
+                continue
+
+            mean_acc = np.mean(accs) * 100.0
+            results[task.benchmark] = mean_acc
+
+        return self(
+            eval_name=result_key,
+            full_model="/".join(org_and_model),
+            org=org,
+            model=model,
+            results=results,
+            precision=precision,  # todo model_type=, weight_type=
+            revision=config.get("model_sha", ""),
+            date=config.get("submission_date", ""),
+            still_on_hub=still_on_hub,
+        )
+
+    def update_with_request_file(self):
+        request_file = get_request_file_for_model(self.full_model, self.precision)
+
+        try:
+            with open(request_file, "r") as f:
+                request = json.load(f)
+            self.model_type = ModelType.from_str(request.get("model_type", ""))
+            self.license = request.get("license", "?")
+            self.likes = request.get("likes", 0)
+            self.num_params = request.get("params", 0)
+        except Exception:
+            print(f"Could not find request file for {self.org}/{self.model}")
+
+    def to_dict(self):
+        average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
+        data_dict = {
+            "eval_name": self.eval_name,  # not a column, just a save name,
+            AutoEvalColumn.precision.name: self.precision,
+            AutoEvalColumn.model_type.name: self.model_type.value.name,
+            AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
+            AutoEvalColumn.weight_type.name: self.weight_type,
+            AutoEvalColumn.model.name: make_clickable_model(self.full_model),
+            AutoEvalColumn.dummy.name: self.full_model,
+            AutoEvalColumn.revision.name: self.revision,
+            AutoEvalColumn.average.name: average,
+            AutoEvalColumn.license.name: self.license,
+            AutoEvalColumn.likes.name: self.likes,
+            AutoEvalColumn.params.name: self.num_params,
+            AutoEvalColumn.still_on_hub.name: self.still_on_hub,
+        }
+
+        for task in Tasks:
+            data_dict[task.value.col_name] = self.results[task.value.benchmark]
+
+        return data_dict
+
+
+def get_request_file_for_model(model_name, precision):
+    request_files = os.path.join(
+        "eval-queue",
+        f"{model_name}_eval_request_*.json",
+    )
+    request_files = glob.glob(request_files)
+
+    # Select correct request file (precision)
+    request_file = ""
+    request_files = sorted(request_files, reverse=True)
+    for tmp_request_file in request_files:
+        with open(tmp_request_file, "r") as f:
+            req_content = json.load(f)
+            if (
+                req_content["status"] in ["FINISHED", "PENDING_NEW_EVAL"]
+                and req_content["precision"] == precision.split(".")[-1]
+            ):
+                request_file = tmp_request_file
+    return request_file
+
+
+def get_eval_results(results_path: str) -> List[EvalResult]:
+    json_filepaths = []
+
+    for root, _, files in os.walk(results_path):
+        # We should only have json files in model results
+        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
+            continue
+
+        # Sort the files by date
+        try:
+            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
+        except dateutil.parser._parser.ParserError:
+            files = [files[-1]]
+
+        # up_to_date = files[-1]
+        for file in files:
+            json_filepaths.append(os.path.join(root, file))
+
+    eval_results = {}
+    for json_filepath in json_filepaths:
+        # Creation of result
+        eval_result = EvalResult.init_from_json_file(json_filepath)
+        eval_result.update_with_request_file()
+
+        # Store results of same eval together
+        eval_name = eval_result.eval_name
+        if eval_name in eval_results.keys():
+            eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
+        else:
+            eval_results[eval_name] = eval_result
+
+    results = []
+    for v in eval_results.values():
+        try:
+            results.append(v.to_dict())
+        except KeyError: # not all eval values present 
+            continue
+
+    return results

src/plots/read_results.py

@@ -1,158 +0,0 @@
-import json
-import os
-from dataclasses import dataclass
-from typing import Dict, List, Tuple
-
-import dateutil
-import numpy as np
-
-from src.get_model_info.utils import AutoEvalColumn, make_clickable_model
-
-METRICS = ["acc_norm", "acc_norm", "acc", "mc2", "acc", "acc", "f1"]
-BENCHMARKS = ["arc:challenge", "hellaswag", "hendrycksTest", "truthfulqa:mc", "winogrande", "gsm8k", "drop"]
-BENCH_TO_NAME = {
-    "arc:challenge": AutoEvalColumn.arc.name,
-    "hellaswag": AutoEvalColumn.hellaswag.name,
-    "hendrycksTest": AutoEvalColumn.mmlu.name,
-    "truthfulqa:mc": AutoEvalColumn.truthfulqa.name,
-    "winogrande": AutoEvalColumn.winogrande.name,
-    "gsm8k": AutoEvalColumn.gsm8k.name,
-    "drop": AutoEvalColumn.drop.name,
-}
-
-
-@dataclass
-class EvalResult:
-    eval_name: str
-    org: str
-    model: str
-    revision: str
-    results: dict
-    precision: str = ""
-    model_type: str = ""
-    weight_type: str = "Original"
-    date: str = ""
-
-    def to_dict(self):
-        from src.filters import is_model_on_hub
-
-        if self.org is not None:
-            base_model = f"{self.org}/{self.model}"
-        else:
-            base_model = f"{self.model}"
-        data_dict = {}
-
-        data_dict["eval_name"] = self.eval_name  # not a column, just a save name
-        data_dict["weight_type"] = self.weight_type  # not a column, just a save name
-        data_dict[AutoEvalColumn.precision.name] = self.precision
-        data_dict[AutoEvalColumn.model_type.name] = self.model_type
-        data_dict[AutoEvalColumn.model.name] = make_clickable_model(base_model)
-        data_dict[AutoEvalColumn.dummy.name] = base_model
-        data_dict[AutoEvalColumn.revision.name] = self.revision
-        data_dict[AutoEvalColumn.average.name] = sum([v for k, v in self.results.items()]) / 7.0
-        data_dict[AutoEvalColumn.still_on_hub.name] = (
-            is_model_on_hub(base_model, self.revision)[0] or base_model == "baseline"
-        )
-
-        for benchmark in BENCHMARKS:
-            if benchmark not in self.results.keys():
-                self.results[benchmark] = None
-
-        for k, v in BENCH_TO_NAME.items():
-            data_dict[v] = self.results[k]
-
-        return data_dict
-
-
-def parse_eval_result(json_filepath: str) -> Tuple[str, list[dict]]:
-    with open(json_filepath) as fp:
-        data = json.load(fp)
-
-    for mmlu_k in ["harness|hendrycksTest-abstract_algebra|5", "hendrycksTest-abstract_algebra"]:
-        if mmlu_k in data["versions"] and data["versions"][mmlu_k] == 0:
-            return None, []  # we skip models with the wrong version
-
-    try:
-        config = data["config"]
-    except KeyError:
-        config = data["config_general"]
-    model = config.get("model_name", None)
-    if model is None:
-        model = config.get("model_args", None)
-
-    model_sha = config.get("model_sha", "")
-    model_split = model.split("/", 1)
-
-    precision = config.get("model_dtype")
-    if precision == "None":
-        precision = "GPTQ"
-
-    model = model_split[-1]
-
-    if len(model_split) == 1:
-        org = None
-        model = model_split[0]
-        result_key = f"{model}_{precision}"
-    else:
-        org = model_split[0]
-        model = model_split[1]
-        result_key = f"{org}_{model}_{precision}"
-
-    eval_results = []
-    for benchmark, metric in zip(BENCHMARKS, METRICS):
-        accs = np.array([v.get(metric, None) for k, v in data["results"].items() if benchmark in k])
-        if accs.size == 0 or any([acc is None for acc in accs]):
-            continue
-        mean_acc = np.mean(accs) * 100.0
-        eval_results.append(
-            EvalResult(
-                eval_name=result_key,
-                org=org,
-                model=model,
-                revision=model_sha,
-                results={benchmark: mean_acc},
-                precision=precision,  # todo model_type=, weight_type=
-                date=config.get("submission_date"),
-            )
-        )
-
-    return result_key, eval_results
-
-
-def get_eval_results(results_path: str) -> List[EvalResult]:
-    json_filepaths = []
-
-    for root, dir, files in os.walk(results_path):
-        # We should only have json files in model results
-        if len(files) == 0 or any([not f.endswith(".json") for f in files]):
-            continue
-
-        # Sort the files by date
-        # store results by precision maybe?
-        try:
-            files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
-        except dateutil.parser._parser.ParserError:
-            files = [files[-1]]
-
-        # up_to_date = files[-1]
-        for file in files:
-            json_filepaths.append(os.path.join(root, file))
-
-    eval_results = {}
-    for json_filepath in json_filepaths:
-        result_key, results = parse_eval_result(json_filepath)
-        for eval_result in results:
-            if result_key in eval_results.keys():
-                eval_results[result_key].results.update(eval_result.results)
-            else:
-                eval_results[result_key] = eval_result
-
-    eval_results = [v for v in eval_results.values()]
-
-    return eval_results
-
-
-def get_eval_results_dicts(results_path: str) -> List[Dict]:
-    eval_results = get_eval_results(results_path)
-
-    return [e.to_dict() for e in eval_results]

src/{load_from_hub.py → populate.py}

@@ -1,50 +1,18 @@
 import json
 import os
-from collections import defaultdict
 
 import pandas as pd
 
-from src.assets.hardcoded_evals import baseline, gpt4_values, gpt35_values
-from src.get_model_info.apply_metadata_to_df import apply_metadata
-from src.plots.read_results import get_eval_results_dicts, make_clickable_model
-from src.get_model_info.utils import AutoEvalColumn, EvalQueueColumn, has_no_nan_values
-
-IS_PUBLIC = bool(os.environ.get("IS_PUBLIC", True))
-
-
-def get_all_requested_models(requested_models_dir: str) -> set[str]:
-    depth = 1
-    file_names = []
-    users_to_submission_dates = defaultdict(list)
-
-    for root, _, files in os.walk(requested_models_dir):
-        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
-        if current_depth == depth:
-            for file in files:
-                if not file.endswith(".json"):
-                    continue
-                with open(os.path.join(root, file), "r") as f:
-                    info = json.load(f)
-                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
-                    # Select organisation
-                    if info["model"].count("/") == 0 or "submitted_time" not in info:
-                        continue
-                    organisation, _ = info["model"].split("/")
-                    users_to_submission_dates[organisation].append(info["submitted_time"])
-
-    return set(file_names), users_to_submission_dates
+from src.leaderboard.filter_models import filter_models
+from src.leaderboard.read_evals import get_eval_results
+from src.display.formatting import make_clickable_model, has_no_nan_values
+from src.display.utils import AutoEvalColumn, EvalQueueColumn, baseline_row
 
 
 def get_leaderboard_df(results_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
-    all_data = get_eval_results_dicts(results_path)
-
-    if not IS_PUBLIC:
-        all_data.append(gpt4_values)
-        all_data.append(gpt35_values)
-
-    all_data.append(baseline)
-    apply_metadata(all_data)  # Populate model type based on known hardcoded values in `metadata.py`
+    all_data = get_eval_results(results_path)
+    all_data.append(baseline_row)
+    filter_models(all_data)
 
     df = pd.DataFrame.from_records(all_data)
     df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
@@ -88,4 +56,3 @@ def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
     df_running = pd.DataFrame.from_records(running_list, columns=cols)
     df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
     return df_finished[cols], df_running[cols], df_pending[cols]
-

src/{filters.py → submission/check_validity.py}

@@ -1,5 +1,9 @@
 import huggingface_hub
 import os
+import json
+import re
+from collections import defaultdict
+from huggingface_hub.hf_api import ModelInfo
 from huggingface_hub import ModelCard
 from transformers import AutoConfig
 
@@ -30,9 +34,9 @@ def check_model_card(repo_id: str) -> tuple[bool, str]:
     return True, ""
 
 
-def is_model_on_hub(model_name: str, revision: str, token: str = None) -> bool:
+def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False) -> tuple[bool, str]:
     try:
-        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=False, token=token)
+        AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
         return True, None
 
     except ValueError:
@@ -45,6 +49,23 @@ def is_model_on_hub(model_name: str, revision: str, token: str = None) -> bool:
         return False, "was not found on hub!"
 
 
+def get_model_size(model_info: ModelInfo, precision: str):
+    size_pattern = size_pattern = re.compile(r"(\d\.)?\d+(b|m)")
+    try:
+        model_size = round(model_info.safetensors["total"] / 1e9, 3)
+    except AttributeError:
+        try:
+            size_match = re.search(size_pattern, model_info.modelId.lower())
+            model_size = size_match.group(0)
+            model_size = round(float(model_size[:-1]) if model_size[-1] == "b" else float(model_size[:-1]) / 1e3, 3)
+        except AttributeError:
+            return 0  # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
+
+    size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
+    model_size = size_factor * model_size
+    return model_size
+
+
 def user_submission_permission(submission_name, users_to_submission_dates, rate_limit_period, rate_limit_quota):
     org_or_user, _ = submission_name.split("/")
     if org_or_user not in users_to_submission_dates:
@@ -65,3 +86,26 @@ def user_submission_permission(submission_name, users_to_submission_dates, rate_
         return False, error_msg
     return True, ""
 
+
+def already_submitted_models(requested_models_dir: str) -> set[str]:
+    depth = 1
+    file_names = []
+    users_to_submission_dates = defaultdict(list)
+
+    for root, _, files in os.walk(requested_models_dir):
+        current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
+        if current_depth == depth:
+            for file in files:
+                if not file.endswith(".json"):
+                    continue
+                with open(os.path.join(root, file), "r") as f:
+                    info = json.load(f)
+                    file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
+
+                    # Select organisation
+                    if info["model"].count("/") == 0 or "submitted_time" not in info:
+                        continue
+                    organisation, _ = info["model"].split("/")
+                    users_to_submission_dates[organisation].append(info["submitted_time"])
+
+    return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -0,0 +1,128 @@
+import os, json
+
+from datetime import datetime, timezone
+
+from src.display.formatting import styled_error, styled_warning, styled_message
+from src.leaderboard.filter_models import DO_NOT_SUBMIT_MODELS
+from src.submission.check_validity import (
+    user_submission_permission,
+    is_model_on_hub,
+    get_model_size,
+    check_model_card,
+    already_submitted_models,
+)
+from src.envs import RATE_LIMIT_QUOTA, RATE_LIMIT_PERIOD, H4_TOKEN, EVAL_REQUESTS_PATH, API, QUEUE_REPO
+
+requested_models, users_to_submission_dates = already_submitted_models(EVAL_REQUESTS_PATH)
+
+
+def add_new_eval(
+    model: str,
+    base_model: str,
+    revision: str,
+    precision: str,
+    private: bool,
+    weight_type: str,
+    model_type: str,
+):
+    precision = precision.split(" ")[0]
+    current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
+
+    if model_type is None or model_type == "":
+        return styled_error("Please select a model type.")
+
+    # Is the user rate limited?
+    user_can_submit, error_msg = user_submission_permission(
+        model, users_to_submission_dates, RATE_LIMIT_PERIOD, RATE_LIMIT_QUOTA
+    )
+    if not user_can_submit:
+        return styled_error(error_msg)
+
+    # Did the model authors forbid its submission to the leaderboard?
+    if model in DO_NOT_SUBMIT_MODELS or base_model in DO_NOT_SUBMIT_MODELS:
+        return styled_warning("Model authors have requested that their model be not submitted on the leaderboard.")
+
+    # Does the model actually exist?
+    if revision == "":
+        revision = "main"
+
+    # Is the model on the hub?
+    if weight_type in ["Delta", "Adapter"]:
+        base_model_on_hub, error = is_model_on_hub(base_model, revision, H4_TOKEN)
+        if not base_model_on_hub:
+            return styled_error(f'Base model "{base_model}" {error}')
+
+    if not weight_type == "Adapter":
+        model_on_hub, error = is_model_on_hub(model, revision)
+        if not model_on_hub:
+            return styled_error(f'Model "{model}" {error}')
+
+    # Is the model info correctly filled?
+    try:
+        model_info = API.model_info(repo_id=model, revision=revision)
+    except Exception:
+        return styled_error("Could not get your model information. Please fill it up properly.")
+
+    model_size = get_model_size(model_info=model_info, precision=precision)
+
+    # Were the model card and license filled?
+    try:
+        license = model_info.cardData["license"]
+    except Exception:
+        return styled_error("Please select a license for your model")
+
+    modelcard_OK, error_msg = check_model_card(model)
+    if not modelcard_OK:
+        return styled_error(error_msg)
+
+    # Seems good, creating the eval
+    print("Adding new eval")
+
+    eval_entry = {
+        "model": model,
+        "base_model": base_model,
+        "revision": revision,
+        "private": private,
+        "precision": precision,
+        "weight_type": weight_type,
+        "status": "PENDING",
+        "submitted_time": current_time,
+        "model_type": model_type,
+        "likes": model_info.likes,
+        "params": model_size,
+        "license": license,
+    }
+
+    user_name = ""
+    model_path = model
+    if "/" in model:
+        user_name = model.split("/")[0]
+        model_path = model.split("/")[1]
+
+    print("Creating eval file")
+    OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
+    os.makedirs(OUT_DIR, exist_ok=True)
+    out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{precision}_{weight_type}.json"
+
+    # Check for duplicate submission
+    if f"{model}_{revision}_{precision}" in requested_models:
+        return styled_warning("This model has been already submitted.")
+
+    with open(out_path, "w") as f:
+        f.write(json.dumps(eval_entry))
+
+    print("Uploading eval file")
+    API.upload_file(
+        path_or_fileobj=out_path,
+        path_in_repo=out_path.split("eval-queue/")[1],
+        repo_id=QUEUE_REPO,
+        repo_type="dataset",
+        commit_message=f"Add {model} to eval queue",
+    )
+
+    # Remove the local file
+    os.remove(out_path)
+
+    return styled_message(
+        "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
+    )

src/{manage_collections.py → tools/collections.py}

@@ -4,33 +4,34 @@ from pandas import DataFrame
 from huggingface_hub import get_collection, add_collection_item, update_collection_item, delete_collection_item
 from huggingface_hub.utils._errors import HfHubHTTPError
 
-from src.get_model_info.hardocded_metadata.types import ModelType
-from src.get_model_info.utils import AutoEvalColumn
+from src.display.utils import AutoEvalColumn, ModelType
 
-H4_TOKEN = os.environ.get("H4_TOKEN", None)
+from src.envs import H4_TOKEN, PATH_TO_COLLECTION
 
-path_to_collection = "open-llm-leaderboard/llm-leaderboard-best-models-652d6c7965a4619fb5c27a03"
+# Specific intervals for the collections
 intervals = {
     "1B": pd.Interval(0, 1.5, closed="right"),
     "3B": pd.Interval(2.5, 3.5, closed="neither"),
     "7B": pd.Interval(6, 8, closed="neither"),
     "13B": pd.Interval(10, 14, closed="neither"),
-    "30B":pd.Interval(25, 35, closed="neither"), 
+    "30B": pd.Interval(25, 35, closed="neither"),
     "65B": pd.Interval(60, 70, closed="neither"),
 }
 
+
 def update_collections(df: DataFrame):
-    """This function updates the Open LLM Leaderboard model collection with the latest best models for 
+    """This function updates the Open LLM Leaderboard model collection with the latest best models for
     each size category and type.
     """
-    collection = get_collection(collection_slug=path_to_collection, token=H4_TOKEN)
+    collection = get_collection(collection_slug=PATH_TO_COLLECTION, token=H4_TOKEN)
     params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
 
     cur_best_models = []
 
     ix = 0
     for type in ModelType:
-        if type.value.name == "": continue
+        if type.value.name == "":
+            continue
         for size in intervals:
             # We filter the df to gather the relevant models
             type_emoji = [t[0] for t in type.value.symbol]
@@ -40,7 +41,9 @@ def update_collections(df: DataFrame):
             mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
             filtered_df = filtered_df.loc[mask]
 
-            best_models = list(filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name])
+            best_models = list(
+                filtered_df.sort_values(AutoEvalColumn.average.name, ascending=False)[AutoEvalColumn.dummy.name]
+            )
             print(type.value.symbol, size, best_models[:10])
 
             # We add them one by one to the leaderboard
@@ -49,27 +52,32 @@ def update_collections(df: DataFrame):
                 cur_len_collection = len(collection.items)
                 try:
                     collection = add_collection_item(
-                        path_to_collection, 
-                        item_id=model, 
-                        item_type="model", 
+                        PATH_TO_COLLECTION,
+                        item_id=model,
+                        item_type="model",
                         exists_ok=True,
-                        note=f"Best {type.to_str(' ')} model of around {size} on the leaderboard today!", 
-                        token=H4_TOKEN
+                        note=f"Best {type.to_str(' ')} model of around {size} on the leaderboard today!",
+                        token=H4_TOKEN,
                     )
-                    if len(collection.items) > cur_len_collection: # we added an item - we make sure its position is correct
-                        item_object_id = collection.items[-1].item_object_id 
-                        update_collection_item(collection_slug=path_to_collection, item_object_id=item_object_id, position=ix)
+                    if (
+                        len(collection.items) > cur_len_collection
+                    ):  # we added an item - we make sure its position is correct
+                        item_object_id = collection.items[-1].item_object_id
+                        update_collection_item(
+                            collection_slug=PATH_TO_COLLECTION, item_object_id=item_object_id, position=ix
+                        )
                         cur_len_collection = len(collection.items)
                     cur_best_models.append(model)
                     break
                 except HfHubHTTPError:
                     continue
 
-    collection = get_collection(path_to_collection, token=H4_TOKEN)
+    collection = get_collection(PATH_TO_COLLECTION, token=H4_TOKEN)
     for item in collection.items:
         if item.item_id not in cur_best_models:
             try:
-                delete_collection_item(collection_slug=path_to_collection, item_object_id=item.item_object_id, token=H4_TOKEN)
+                delete_collection_item(
+                    collection_slug=PATH_TO_COLLECTION, item_object_id=item.item_object_id, token=H4_TOKEN
+                )
             except HfHubHTTPError:
                 continue
-

src/{plots/plot_results.py → tools/plots.py}

@@ -4,7 +4,7 @@ from plotly.graph_objs import Figure
 import pickle
 from datetime import datetime, timezone
 from typing import List, Dict, Tuple, Any
-from src.get_model_info.hardocded_metadata.flags import FLAGGED_MODELS
+from src.leaderboard.filter_models import FLAGGED_MODELS
 
 # Average ⬆️ human baseline is 0.897 (source: averaging human baselines below)
 # ARC human baseline is 0.80 (source: https://lab42.global/arc/)
@@ -220,4 +220,4 @@ def create_metric_plot_obj(
 
 # Example Usage:
 # human_baselines dictionary is defined.
-# chart = create_metric_plot_obj(scores_df, ["ARC", "HellaSwag", "MMLU", "TruthfulQA"], human_baselines, "Graph Title")
+# chart = create_metric_plot_obj(scores_df, ["ARC", "HellaSwag", "MMLU", "TruthfulQA"], human_baselines, "Graph Title")