coda

.gitignore

@@ -1,13 +0,0 @@
-auto_evals/
-venv/
-__pycache__/
-.env
-.ipynb_checkpoints
-*ipynb
-.vscode/
-
-eval-queue/
-eval-results/
-eval-queue-bk/
-eval-results-bk/
-logs/

.pre-commit-config.yaml

@@ -1,53 +0,0 @@
-# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-default_language_version:
- python: python3
-
-ci:
- autofix_prs: true
- autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions'
- autoupdate_schedule: quarterly
-
-repos:
- - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.3.0
- hooks:
- - id: check-yaml
- - id: check-case-conflict
- - id: detect-private-key
- - id: check-added-large-files
- args: ['--maxkb=1000']
- - id: requirements-txt-fixer
- - id: end-of-file-fixer
- - id: trailing-whitespace
-
- - repo: https://github.com/PyCQA/isort
- rev: 5.12.0
- hooks:
- - id: isort
- name: Format imports
-
- - repo: https://github.com/psf/black
- rev: 22.12.0
- hooks:
- - id: black
- name: Format code
- additional_dependencies: ['click==8.0.2']
-
- - repo: https://github.com/charliermarsh/ruff-pre-commit
- # Ruff version.
- rev: 'v0.0.267'
- hooks:
- - id: ruff

Makefile

@@ -1,13 +0,0 @@
-.PHONY: style format
-
-
-style:
- python -m black --line-length 119 .
- python -m isort .
- ruff check --fix .
-
-
-quality:
- python -m black --check --line-length 119 .
- python -m isort --check-only .
- ruff check .

README.md

@@ -7,6 +7,8 @@ sdk: gradio
 app_file: app.py
 pinned: true
 license: apache-2.0
+sdk_version: 4.36.1
+short_description: 'A Benchmark for Metamorphic Evaluation of T2V Generation'
 ---
 
 # Start the configuration

app.py

@@ -1,345 +1,338 @@
-import subprocess
+__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions']
+
 import gradio as gr
 import pandas as pd
-from apscheduler.schedulers.background import BackgroundScheduler
-from huggingface_hub import snapshot_download
-
-from src.about import (
- CITATION_BUTTON_LABEL,
- CITATION_BUTTON_TEXT,
- EVALUATION_QUEUE_TEXT,
- INTRODUCTION_TEXT,
- LLM_BENCHMARKS_TEXT,
- TITLE,
-)
-from src.display.css_html_js import custom_css
-from src.display.utils import (
- BENCHMARK_COLS,
- COLS,
- EVAL_COLS,
- EVAL_TYPES,
- NUMERIC_INTERVALS,
- TYPES,
- AutoEvalColumn,
- ModelType,
- fields,
- WeightType,
- Precision
-)
-from src.envs import API, EVAL_REQUESTS_PATH, EVAL_RESULTS_PATH, QUEUE_REPO, REPO_ID, RESULTS_REPO, TOKEN
-from src.populate import get_evaluation_queue_df, get_leaderboard_df
-from src.submission.submit import add_new_eval
-
-
-def restart_space():
- API.restart_space(repo_id=REPO_ID)
-
-try:
- print(EVAL_REQUESTS_PATH)
- snapshot_download(
- repo_id=QUEUE_REPO, local_dir=EVAL_REQUESTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
- )
-except Exception:
- restart_space()
-try:
- print(EVAL_RESULTS_PATH)
- snapshot_download(
- repo_id=RESULTS_REPO, local_dir=EVAL_RESULTS_PATH, repo_type="dataset", tqdm_class=None, etag_timeout=30, token=TOKEN
- )
-except Exception:
- restart_space()
-
-
-raw_data, original_df = get_leaderboard_df(EVAL_RESULTS_PATH, EVAL_REQUESTS_PATH, COLS, BENCHMARK_COLS)
-leaderboard_df = original_df.copy()
-
-(
- finished_eval_queue_df,
- running_eval_queue_df,
- pending_eval_queue_df,
-) = get_evaluation_queue_df(EVAL_REQUESTS_PATH, EVAL_COLS)
-
-
-# Searching and filtering
-def update_table(
- hidden_df: pd.DataFrame,
- columns: list,
- type_query: list,
- precision_query: str,
- size_query: list,
- show_deleted: bool,
- query: str,
+import json
+import io
+
+from constants import *
+
+global data_component, data_component_150, filter_component
+
+def upload_file(files):
+ file_paths = [file.name for file in files]
+ return file_paths
+
+def compute_scores(input_data):
+ return [None, [
+ input_data["Average_MTScore"],
+ input_data["Average_CHScore"],
+ input_data["Average_GPT4o-MTScore"],
+ input_data["Average_UMT-FVD"],
+ input_data["Average_UMTScore"]
+ ]]
+
+def add_new_eval(
+ input_file,
+ model_name_textbox: str,
+ revision_name_textbox: str,
+ backbone_type_dropdown: str,
+ model_link: str,
+):
+ if input_file is None:
+ return "Error! Empty file!"
+ else:
+ input_json = json.load(io.BytesIO(input_file))
+ 
+ if model_name_textbox not in input_json:
+ return f"Error! Model '{model_name_textbox}' not found in input file!"
+ 
+ selected_model_data = input_json[model_name_textbox]
+
+ scores = compute_scores(selected_model_data)
+ input_data = scores[1]
+ input_data = [float(i) for i in input_data]
+
+ csv_data = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH)
+
+ if revision_name_textbox == '':
+ col = csv_data.shape[0]
+ model_name = model_name_textbox
+ name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in csv_data['Model']]
+ assert model_name not in name_list
+ else:
+ model_name = revision_name_textbox
+ model_name_list = csv_data['Model']
+ name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in model_name_list]
+ if revision_name_textbox not in name_list:
+ col = csv_data.shape[0]
+ else:
+ col = name_list.index(revision_name_textbox)
+
+ if model_link == '':
+ model_name = model_name # no url
+ else:
+ model_name = '[' + model_name + '](' + model_link + ')'
+
+ backbone = backbone_type_dropdown
+
+ new_data = [
+ model_name,
+ backbone,
+ input_data[3],
+ input_data[4],
+ input_data[0],
+ input_data[1],
+ input_data[2],
+ ]
+ csv_data.loc[col] = new_data 
+ csv_data.to_csv(CSV_DIR_CHRONOMAGIC_BENCH, index=False)
+ return "Evaluation successfully submitted!"
+
+def add_new_eval_150(
+ input_file,
+ model_name_textbox: str,
+ revision_name_textbox: str,
+ backbone_type_dropdown: str,
+ model_link: str,
 ):
- filtered_df = filter_models(hidden_df, type_query, size_query, precision_query, show_deleted)
- filtered_df = filter_queries(query, filtered_df)
- df = select_columns(filtered_df, columns)
+ if input_file is None:
+ return "Error! Empty file!"
+ else:
+ input_json = json.load(io.BytesIO(input_file))
+ 
+ if model_name_textbox not in input_json:
+ return f"Error! Model '{model_name_textbox}' not found in input file!"
+ 
+ selected_model_data = input_json[model_name_textbox]
+
+ scores = compute_scores(selected_model_data)
+ input_data = scores[1]
+ input_data = [float(i) for i in input_data]
+
+ csv_data = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150)
+
+ if revision_name_textbox == '':
+ col = csv_data.shape[0]
+ model_name = model_name_textbox
+ name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in csv_data['Model']]
+ assert model_name not in name_list
+ else:
+ model_name = revision_name_textbox
+ model_name_list = csv_data['Model']
+ name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in model_name_list]
+ if revision_name_textbox not in name_list:
+ col = csv_data.shape[0]
+ else:
+ col = name_list.index(revision_name_textbox)
+
+ if model_link == '':
+ model_name = model_name # no url
+ else:
+ model_name = '[' + model_name + '](' + model_link + ')'
+
+ backbone = backbone_type_dropdown
+
+ new_data = [
+ model_name,
+ backbone,
+ input_data[3],
+ input_data[4],
+ input_data[0],
+ input_data[1],
+ input_data[2],
+ ]
+ csv_data.loc[col] = new_data 
+ csv_data.to_csv(CSV_DIR_CHRONOMAGIC_BENCH_150, index=False)
+ return "Evaluation (150) successfully submitted!"
+
+def get_baseline_df():
+ df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH)
+ df = df.sort_values(by="MTScore↑", ascending=False)
+ present_columns = MODEL_INFO + checkbox_group.value
+ df = df[present_columns]
  return df
 
+def get_baseline_df_150():
+ df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150)
+ df = df.sort_values(by="MTScore↑", ascending=False)
+ present_columns = MODEL_INFO + checkbox_group_150.value
+ df = df[present_columns]
+ return df
 
-def search_table(df: pd.DataFrame, query: str) -> pd.DataFrame:
- return df[(df[AutoEvalColumn.model.name].str.contains(query, case=False))]
-
-
-def select_columns(df: pd.DataFrame, columns: list) -> pd.DataFrame:
- always_here_cols = [
- AutoEvalColumn.model_type_symbol.name,
- AutoEvalColumn.model.name,
- ]
- # We use COLS to maintain sorting
- filtered_df = df[
- always_here_cols + [c for c in COLS if c in df.columns and c in columns] 
- ]
- return filtered_df
-
-
-def filter_queries(query: str, filtered_df: pd.DataFrame) -> pd.DataFrame:
- final_df = []
- if query != "":
- queries = [q.strip() for q in query.split(";")]
- for _q in queries:
- _q = _q.strip()
- if _q != "":
- temp_filtered_df = search_table(filtered_df, _q)
- if len(temp_filtered_df) > 0:
- final_df.append(temp_filtered_df)
- if len(final_df) > 0:
- filtered_df = pd.concat(final_df)
- filtered_df = filtered_df.drop_duplicates(
- subset=[AutoEvalColumn.model.name, AutoEvalColumn.precision.name, AutoEvalColumn.revision.name]
- )
-
- return filtered_df
-
-
-def filter_models(
- df: pd.DataFrame, type_query: list, size_query: list, precision_query: list, show_deleted: bool
-) -> pd.DataFrame:
- # Show all models
- if show_deleted:
- filtered_df = df
- else: # Show only still on the hub models
- filtered_df = df[df[AutoEvalColumn.still_on_hub.name] == True]
-
- type_emoji = [t[0] for t in type_query]
- filtered_df = filtered_df.loc[df[AutoEvalColumn.model_type_symbol.name].isin(type_emoji)]
- filtered_df = filtered_df.loc[df[AutoEvalColumn.precision.name].isin(precision_query + ["None"])]
-
- numeric_interval = pd.IntervalIndex(sorted([NUMERIC_INTERVALS[s] for s in size_query]))
- params_column = pd.to_numeric(df[AutoEvalColumn.params.name], errors="coerce")
- mask = params_column.apply(lambda x: any(numeric_interval.contains(x)))
- filtered_df = filtered_df.loc[mask]
+def get_all_df():
+ df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH)
+ df = df.sort_values(by="MTScore↑", ascending=False)
+ return df
 
- return filtered_df
+def get_all_df_150():
+ df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150)
+ df = df.sort_values(by="MTScore↑", ascending=False)
+ return df
 
+block = gr.Blocks()
 
-demo = gr.Blocks(css=custom_css)
-with demo:
- gr.HTML(TITLE)
- gr.Markdown(INTRODUCTION_TEXT, elem_classes="markdown-text")
 
+with block:
+ gr.Markdown(
+ LEADERBORAD_INTRODUCTION
+ )
  with gr.Tabs(elem_classes="tab-buttons") as tabs:
- with gr.TabItem("🏅 LLM Benchmark", elem_id="llm-benchmark-tab-table", id=0):
+ # table 1
+ with gr.TabItem("🏅 ChronoMagic-Bench", elem_id="ChronoMagic-Bench-tab-table", id=0):
  with gr.Row():
- with gr.Column():
- with gr.Row():
- search_bar = gr.Textbox(
- placeholder=" 🔍 Search for your model (separate multiple queries with `;`) and press ENTER...",
- show_label=False,
- elem_id="search-bar",
- )
- with gr.Row():
- shown_columns = gr.CheckboxGroup(
- choices=[
- c.name
- for c in fields(AutoEvalColumn)
- if not c.hidden and not c.never_hidden
- ],
- value=[
- c.name
- for c in fields(AutoEvalColumn)
- if c.displayed_by_default and not c.hidden and not c.never_hidden
- ],
- label="Select columns to show",
- elem_id="column-select",
- interactive=True,
- )
- with gr.Row():
- deleted_models_visibility = gr.Checkbox(
- value=False, label="Show gated/private/deleted models", interactive=True
- )
- with gr.Column(min_width=320):
- #with gr.Box(elem_id="box-filter"):
- filter_columns_type = gr.CheckboxGroup(
- label="Model types",
- choices=[t.to_str() for t in ModelType],
- value=[t.to_str() for t in ModelType],
- interactive=True,
- elem_id="filter-columns-type",
- )
- filter_columns_precision = gr.CheckboxGroup(
- label="Precision",
- choices=[i.value.name for i in Precision],
- value=[i.value.name for i in Precision],
- interactive=True,
- elem_id="filter-columns-precision",
- )
- filter_columns_size = gr.CheckboxGroup(
- label="Model sizes (in billions of parameters)",
- choices=list(NUMERIC_INTERVALS.keys()),
- value=list(NUMERIC_INTERVALS.keys()),
- interactive=True,
- elem_id="filter-columns-size",
+ with gr.Accordion("Citation", open=False):
+ citation_button = gr.Textbox(
+ value=CITATION_BUTTON_TEXT,
+ label=CITATION_BUTTON_LABEL,
+ elem_id="citation-button",
+ show_copy_button=True
  )
+ 
+ gr.Markdown(
+ TABLE_INTRODUCTION
+ )
+
+ checkbox_group = gr.CheckboxGroup(
+ choices=ALL_RESULTS,
+ value=SELECTED_RESULTS,
+ label="Select options",
+ interactive=True,
+ )
 
- leaderboard_table = gr.components.Dataframe(
- value=leaderboard_df[
- [c.name for c in fields(AutoEvalColumn) if c.never_hidden]
- + shown_columns.value
- ],
- headers=[c.name for c in fields(AutoEvalColumn) if c.never_hidden] + shown_columns.value,
- datatype=TYPES,
- elem_id="leaderboard-table",
+ data_component = gr.components.Dataframe(
+ value=get_baseline_df, 
+ headers=COLUMN_NAMES,
+ type="pandas", 
+ datatype=DATA_TITILE_TYPE,
  interactive=False,
  visible=True,
- )
+ )
+ 
+ def on_checkbox_group_change(selected_columns):
+ selected_columns = [item for item in ALL_RESULTS if item in selected_columns]
+ present_columns = MODEL_INFO + selected_columns
+ updated_data = get_all_df()[present_columns]
+ updated_data = updated_data.sort_values(by=present_columns[1], ascending=False)
+ updated_headers = present_columns
+ update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+ filter_component = gr.components.Dataframe(
+ value=updated_data, 
+ headers=updated_headers,
+ type="pandas", 
+ datatype=update_datatype,
+ interactive=False,
+ visible=True,
+ )
+ 
+ return filter_component
+
+ checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component)
 
- # Dummy leaderboard for handling the case when the user uses backspace key
- hidden_leaderboard_table_for_search = gr.components.Dataframe(
- value=original_df[COLS],
- headers=COLS,
- datatype=TYPES,
- visible=False,
+ # table 2
+ with gr.TabItem("🏅 ChronoMagic-Bench-150", elem_id="ChronoMagic-Bench-150-tab-table", id=1):
+ with gr.Row():
+ with gr.Accordion("Citation", open=False):
+ citation_button = gr.Textbox(
+ value=CITATION_BUTTON_TEXT,
+ label=CITATION_BUTTON_LABEL,
+ elem_id="citation-button",
+ show_copy_button=True
+ )
+ 
+ gr.Markdown(
+ TABLE_INTRODUCTION
  )
- search_bar.submit(
- update_table,
- [
- hidden_leaderboard_table_for_search,
- shown_columns,
- filter_columns_type,
- filter_columns_precision,
- filter_columns_size,
- deleted_models_visibility,
- search_bar,
- ],
- leaderboard_table,
+
+ checkbox_group_150 = gr.CheckboxGroup(
+ choices=ALL_RESULTS,
+ value=SELECTED_RESULTS_150,
+ label="Select options",
+ interactive=True,
  )
- for selector in [shown_columns, filter_columns_type, filter_columns_precision, filter_columns_size, deleted_models_visibility]:
- selector.change(
- update_table,
- [
- hidden_leaderboard_table_for_search,
- shown_columns,
- filter_columns_type,
- filter_columns_precision,
- filter_columns_size,
- deleted_models_visibility,
- search_bar,
- ],
- leaderboard_table,
- queue=True,
+
+ data_component_150 = gr.components.Dataframe(
+ value=get_baseline_df_150, 
+ headers=COLUMN_NAMES,
+ type="pandas", 
+ datatype=DATA_TITILE_TYPE,
+ interactive=False,
+ visible=True,
  )
+ 
+ def on_checkbox_group_150_change(selected_columns):
+ selected_columns = [item for item in ALL_RESULTS if item in selected_columns]
+ present_columns = MODEL_INFO + selected_columns
+ updated_data = get_all_df_150()[present_columns]
+ updated_data = updated_data.sort_values(by=present_columns[1], ascending=False)
+ updated_headers = present_columns
+ update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers]
+
+ filter_component = gr.components.Dataframe(
+ value=updated_data, 
+ headers=updated_headers,
+ type="pandas", 
+ datatype=update_datatype,
+ interactive=False,
+ visible=True,
+ )
+ 
+ return filter_component
 
- with gr.TabItem("📝 About", elem_id="llm-benchmark-tab-table", id=2):
- gr.Markdown(LLM_BENCHMARKS_TEXT, elem_classes="markdown-text")
+ checkbox_group_150.change(fn=on_checkbox_group_150_change, inputs=checkbox_group_150, outputs=data_component_150)
 
- with gr.TabItem("🚀 Submit here! ", elem_id="llm-benchmark-tab-table", id=3):
- with gr.Column():
- with gr.Row():
- gr.Markdown(EVALUATION_QUEUE_TEXT, elem_classes="markdown-text")
+ # table 3 
+ with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=2):
+ with gr.Row():
+ gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text")
 
- with gr.Column():
- with gr.Accordion(
- f"✅ Finished Evaluations ({len(finished_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- finished_eval_table = gr.components.Dataframe(
- value=finished_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
- with gr.Accordion(
- f"🔄 Running Evaluation Queue ({len(running_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- running_eval_table = gr.components.Dataframe(
- value=running_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
-
- with gr.Accordion(
- f"⏳ Pending Evaluation Queue ({len(pending_eval_queue_df)})",
- open=False,
- ):
- with gr.Row():
- pending_eval_table = gr.components.Dataframe(
- value=pending_eval_queue_df,
- headers=EVAL_COLS,
- datatype=EVAL_TYPES,
- row_count=5,
- )
  with gr.Row():
- gr.Markdown("# ✉️✨ Submit your model here!", elem_classes="markdown-text")
+ gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text")
 
  with gr.Row():
  with gr.Column():
- model_name_textbox = gr.Textbox(label="Model name")
- revision_name_textbox = gr.Textbox(label="Revision commit", placeholder="main")
- model_type = gr.Dropdown(
- choices=[t.to_str(" : ") for t in ModelType if t != ModelType.Unknown],
- label="Model type",
- multiselect=False,
- value=None,
- interactive=True,
+ model_name_textbox = gr.Textbox(
+ label="Model name", placeholder="MagicTime"
+ )
+ revision_name_textbox = gr.Textbox(
+ label="Revision Model Name", placeholder="MagicTime"
  )
-
- with gr.Column():
- precision = gr.Dropdown(
- choices=[i.value.name for i in Precision if i != Precision.Unknown],
- label="Precision",
- multiselect=False,
- value="float16",
- interactive=True,
+ backbone_type_dropdown = gr.Dropdown(
+ label="Backbone Type",
+ choices=["DiT", "U-Net"],
+ value="DiT"
  )
- weight_type = gr.Dropdown(
- choices=[i.value.name for i in WeightType],
- label="Weights type",
- multiselect=False,
- value="Original",
- interactive=True,
+ model_link = gr.Textbox(
+ label="Model Link", placeholder="https://github.com/PKU-YuanGroup/MagicTime"
  )
- base_model_name_textbox = gr.Textbox(label="Base model (for delta or adapter weights)")
-
- submit_button = gr.Button("Submit Eval")
- submission_result = gr.Markdown()
- submit_button.click(
- add_new_eval,
- [
- model_name_textbox,
- base_model_name_textbox,
- revision_name_textbox,
- precision,
- weight_type,
- model_type,
- ],
- submission_result,
- )
 
- with gr.Row():
- with gr.Accordion("📙 Citation", open=False):
- citation_button = gr.Textbox(
- value=CITATION_BUTTON_TEXT,
- label=CITATION_BUTTON_LABEL,
- lines=20,
- elem_id="citation-button",
- show_copy_button=True,
- )
+ with gr.Column():
+ input_file = gr.File(label="Click to Upload a json File", type='binary')
+ submit_button = gr.Button("Submit Eval (ChronoMagic-Bench)")
+ submit_button_150 = gr.Button("Submit Eval (ChronoMagic-Bench-150)")
+
+ submission_result = gr.Markdown()
+ submit_button.click(
+ add_new_eval,
+ inputs=[
+ input_file,
+ model_name_textbox,
+ revision_name_textbox,
+ backbone_type_dropdown,
+ model_link,
+ ],
+ outputs=submission_result,
+ )
+ submit_button_150.click(
+ add_new_eval_150,
+ inputs=[
+ input_file,
+ model_name_textbox,
+ revision_name_textbox,
+ backbone_type_dropdown,
+ model_link,
+ ],
+ outputs = submission_result,
+ )
 
-scheduler = BackgroundScheduler()
-scheduler.add_job(restart_space, "interval", seconds=1800)
-scheduler.start()
-demo.queue(default_concurrency_limit=40).launch()
+ with gr.Row():
+ data_run = gr.Button("Refresh")
+ data_run.click(
+ get_baseline_df, outputs=data_component
+ )
+ data_run.click(
+ get_baseline_df_150, outputs=data_component_150
+ )
+ 
+block.launch()

constants.py

@@ -0,0 +1,49 @@
+MODEL_INFO = ["Model", "Backbone"]
+
+ALL_RESULTS = ["UMT-FVD↓", "UMTScore↑", "MTScore↑", "CHScore↑", "GPT4o-MTScore↑"]
+
+SELECTED_RESULTS = ["UMT-FVD↓", "UMTScore↑", "MTScore↑", "CHScore↑", "GPT4o-MTScore↑"]
+SELECTED_RESULTS_150 = ["UMT-FVD↓", "UMTScore↑", "MTScore↑", "GPT4o-MTScore↑"]
+
+DATA_TITILE_TYPE = ["markdown", 'markdown', "number", "number", "number", "number", "number"]
+
+CSV_DIR_CHRONOMAGIC_BENCH = "./file/results_ChronoMagic-Bench.csv"
+CSV_DIR_CHRONOMAGIC_BENCH_150 = "./file/results_ChronoMagic-Bench-150.csv"
+
+COLUMN_NAMES = MODEL_INFO + ALL_RESULTS
+
+LEADERBORAD_INTRODUCTION = """# ChronoMagic-Bench Leaderboard
+ 
+ Welcome to the leaderboard of the ChronoMagic-Bench! 
+ 
+ 🏆ChronoMagic-Bench represents the inaugural benchmark dedicated to assessing T2V models' capabilities in generating time-lapse videos that demonstrate significant metamorphic amplitude and temporal coherence. The benchmark probes T2V models for their physics, biology, and chemistry capabilities, in a free-form text control.
+ 
+ Please refer to [our paper](https://arxiv.org/abs/2311.16103) for more details.
+ """
+
+SUBMIT_INTRODUCTION = """# Submit Introduction
+ Obtain `ChronoMagic-Bench-Input.json` from our [github repository](https://github.com/PKU-YuanGroup/Video-Bench#%EF%B8%8F-evaluate-your-own-model) after evaluation.
+
+
+ ## Submit Example
+ For example, if you want to upload Video-ChatGPT's result in the leaderboard, you need to:
+ 1. Fill in 'MagicTime' in 'Model Name' if it is your first time to submit your result (You can leave 'Revision Model Name' blank).
+ 2. Fill in 'MagicTime' in 'Revision Model Name' if you want to update your result (You can leave 'Model Name' blank).
+ 3. Select ‘Backbone Type’ (DiT or U-Net).
+ 4. Fill in 'https://github.com/x/x' in 'Model Link'.
+ 5. Upload `ChronoMagic-Bench-Input.json`.
+ 6. Click the 'Submit Eval' button.
+ 7. Click 'Refresh' to obtain the uploaded leaderboard.
+"""
+
+TABLE_INTRODUCTION = """In the table below, we summarize each task performance of all the models.
+ We use UMT-FVD, UMTScore, MTScore, CHScore, GPT4o-MTScore as the primary evaluation metric for each tasks.
+ """
+
+CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
+CITATION_BUTTON_TEXT = r"""@article{yuan2024magictime,
+ title={MagicTime: Time-lapse Video Generation Models as Metamorphic Simulators},
+ author={Yuan, Shenghai and Huang, Jinfa and Shi, Yujun and Xu, Yongqi and Zhu, Ruijie and Lin, Bin and Cheng, Xinhua and Yuan, Li and Luo, Jiebo},
+ journal={arXiv preprint arXiv:2404.05014},
+ year={2024}
+}"""

file/ChronoMagic-Bench-Input.json

@@ -0,0 +1,9 @@
+{
+ "test": {
+ "Average_MTScore": 0.2613712220142285,
+ "Average_CHScore": 4.1676077942528345,
+ "Average_GPT4o-MTScore": 2.3333333333333335,
+ "Average_UMT-FVD": -1,
+ "Average_UMTScore": -1
+ }
+}

file/results_ChronoMagic-Bench-150.csv

@@ -0,0 +1,16 @@
+Model,Backbone,UMT-FVD↓,UMTScore↑,MTScore↑,CHScore↑,GPT4o-MTScore↑
+[Gen-2 (20240610)](https://research.runwayml.com/gen2),DiT,218.99,2.4,0.373,5.27,2.62
+[Pika-2.0 (20240610)](https://www.pika.art/),DiT,223.05,2.317,0.347,4.0,2.48
+[Dream Machine (20240610)](https://lumalabs.ai/dream-machine),DiT,214.91,2.387,0.474,2.3,3.11
+[KeLing (20240610)](https://h5.kwaiying.com/officialWebsite),DiT,202.32,2.517,0.369,3.69,2.74
+[ModelScopeT2V](https://huggingface.co/ali-vilab/text-to-video-ms-1.7b),U-Net,230.74,2.783,0.409,10.64,3.01
+[ZeroScope](https://huggingface.co/cerspense/zeroscope_v2_576w),U-Net,260.61,2.232,0.403,24.1,2.29
+[T2V-Zero](https://github.com/Picsart-AI-Research/Text2Video-Zero),U-Net,250.22,2.559,0.399,1.84,2.62
+[LaVie](https://github.com/Vchitect/LaVie),U-Net,210.39,2.714,0.35,9.58,2.5
+[AnimateDiff-V3](https://github.com/guoyww/AnimateDiff),U-Net,239.31,2.837,0.47,11.09,2.62
+[VideoCrafter2](https://github.com/AILab-CVC/VideoCrafter),U-Net,214.06,2.763,0.437,7.78,2.87
+[MagicTime](https://github.com/PKU-YuanGroup/MagicTime),U-Net,294.72,1.763,0.479,11.58,3.05
+[Latte](https://github.com/Vchitect/Latte),DiT,232.29,2.122,0.366,13.79,2.42
+[OpenSora 1.1](https://github.com/hpcaitech/Open-Sora),DiT,241.09,2.676,0.448,10.46,2.57
+[OpenSora 1.2](https://github.com/hpcaitech/Open-Sora),DiT,210.93,2.681,0.383,5.6,2.5
+[OpenSoraPlan v1.1](https://github.com/PKU-YuanGroup/Open-Sora-Plan),DiT,228.7,2.459,0.331,10.32,2.21

file/results_ChronoMagic-Bench.csv

@@ -0,0 +1,12 @@
+Model,Backbone,UMT-FVD↓,UMTScore↑,MTScore↑,CHScore↑,GPT4o-MTScore↑
+[ModelScopeT2V](https://huggingface.co/ali-vilab/text-to-video-ms-1.7b),U-Net,194.77,2.909,0.401,11.03,2.86
+[ZeroScope](https://huggingface.co/cerspense/zeroscope_v2_576w),U-Net,227.02,2.35,0.4,25.13,2.09
+[T2V-Zero](https://github.com/Picsart-AI-Research/Text2Video-Zero),U-Net,209.66,2.661,0.4,1.68,2.55
+[LaVie](https://github.com/Vchitect/LaVie),U-Net,166.97,2.763,0.346,8.6,2.46
+[AnimateDiff-V3](https://github.com/guoyww/AnimateDiff),U-Net,197.89,2.944,0.467,11.36,2.62
+[VideoCrafter2](https://github.com/AILab-CVC/VideoCrafter),U-Net,178.45,2.753,0.433,8.27,2.68
+[MagicTime](https://github.com/PKU-YuanGroup/MagicTime),U-Net,257.56,1.916,0.478,10.66,3.13
+[Latte](https://github.com/Vchitect/Latte),DiT,192.12,2.111,0.363,13.81,2.2
+[OpenSora 1.1](https://github.com/hpcaitech/Open-Sora),DiT,195.43,2.678,0.444,10.03,2.52
+[OpenSora 1.2](https://github.com/hpcaitech/Open-Sora),DiT,166.92,2.781,0.375,4.69,2.56
+[OpenSoraPlan v1.1](https://github.com/PKU-YuanGroup/Open-Sora-Plan),DiT,188.53,2.421,0.327,10.35,2.19

pyproject.toml

@@ -1,13 +0,0 @@
-[tool.ruff]
-# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default.
-select = ["E", "F"]
-ignore = ["E501"] # line too long (black is taking care of this)
-line-length = 119
-fixable = ["A", "B", "C", "D", "E", "F", "G", "I", "N", "Q", "S", "T", "W", "ANN", "ARG", "BLE", "COM", "DJ", "DTZ", "EM", "ERA", "EXE", "FBT", "ICN", "INP", "ISC", "NPY", "PD", "PGH", "PIE", "PL", "PT", "PTH", "PYI", "RET", "RSE", "RUF", "SIM", "SLF", "TCH", "TID", "TRY", "UP", "YTT"]
-
-[tool.isort]
-profile = "black"
-line_length = 119
-
-[tool.black]
-line-length = 119

requirements.txt

@@ -1,18 +1,2 @@
-APScheduler
-black
-click
-datasets
-gradio
-gradio_client
-huggingface-hub>=0.18.0
-matplotlib
-numpy
-pandas
-python-dateutil
-requests
-tqdm
-transformers
-tokenizers>=0.15.0
-git+https://github.com/EleutherAI/lm-evaluation-harness.git@b281b0921b636bc36ad05c0b0b0763bd6dd43463#egg=lm-eval
-accelerate
-sentencepiece
+gradio==4.36.1
+pandas==2.2.2

src/about.py

@@ -1,72 +0,0 @@
-from dataclasses import dataclass
-from enum import Enum
-
-@dataclass
-class Task:
- benchmark: str
- metric: str
- col_name: str
-
-
-# Select your tasks here
-# ---------------------------------------------------
-class Tasks(Enum):
- # task_key in the json file, metric_key in the json file, name to display in the leaderboard 
- task0 = Task("anli_r1", "acc", "ANLI")
- task1 = Task("logiqa", "acc_norm", "LogiQA")
-
-NUM_FEWSHOT = 0 # Change with your few shot
-# ---------------------------------------------------
-
-
-
-# Your leaderboard name
-TITLE = """<h1 align="center" id="space-title">Demo leaderboard</h1>"""
-
-# What does your leaderboard evaluate?
-INTRODUCTION_TEXT = """
-Intro text
-"""
-
-# Which evaluations are you running? how can people reproduce what you have?
-LLM_BENCHMARKS_TEXT = f"""
-## How it works
-
-## Reproducibility
-To reproduce our results, here is the commands you can run:
-
-"""
-
-EVALUATION_QUEUE_TEXT = """
-## Some good practices before submitting a model
-
-### 1) Make sure you can load your model and tokenizer using AutoClasses:
-```python
-from transformers import AutoConfig, AutoModel, AutoTokenizer
-config = AutoConfig.from_pretrained("your model name", revision=revision)
-model = AutoModel.from_pretrained("your model name", revision=revision)
-tokenizer = AutoTokenizer.from_pretrained("your model name", revision=revision)
-```
-If this step fails, follow the error messages to debug your model before submitting it. It's likely your model has been improperly uploaded.
-
-Note: make sure your model is public!
-Note: if your model needs `use_remote_code=True`, we do not support this option yet but we are working on adding it, stay posted!
-
-### 2) Convert your model weights to [safetensors](https://huggingface.co/docs/safetensors/index)
-It's a new format for storing weights which is safer and faster to load and use. It will also allow us to add the number of parameters of your model to the `Extended Viewer`!
-
-### 3) Make sure your model has an open license!
-This is a leaderboard for Open LLMs, and we'd love for as many people as possible to know they can use your model 🤗
-
-### 4) Fill up your model card
-When we add extra information about models to the leaderboard, it will be automatically taken from the model card
-
-## In case of model failure
-If your model is displayed in the `FAILED` category, its execution stopped.
-Make sure you have followed the above steps first.
-If everything is done, check you can launch the EleutherAIHarness on your model locally, using the above command without modifications (you can add `--limit` to limit the number of examples per task).
-"""
-
-CITATION_BUTTON_LABEL = "Copy the following snippet to cite these results"
-CITATION_BUTTON_TEXT = r"""
-"""

src/display/css_html_js.py

@@ -1,105 +0,0 @@
-custom_css = """
-
-.markdown-text {
- font-size: 16px !important;
-}
-
-#models-to-add-text {
- font-size: 18px !important;
-}
-
-#citation-button span {
- font-size: 16px !important;
-}
-
-#citation-button textarea {
- font-size: 16px !important;
-}
-
-#citation-button > label > button {
- margin: 6px;
- transform: scale(1.3);
-}
-
-#leaderboard-table {
- margin-top: 15px
-}
-
-#leaderboard-table-lite {
- margin-top: 15px
-}
-
-#search-bar-table-box > div:first-child {
- background: none;
- border: none;
-}
- 
-#search-bar {
- padding: 0px;
-}
-
-/* Limit the width of the first AutoEvalColumn so that names don't expand too much */
-table td:first-child,
-table th:first-child {
- max-width: 400px;
- overflow: auto;
- white-space: nowrap;
-}
-
-.tab-buttons button {
- font-size: 20px;
-}
-
-#scale-logo {
- border-style: none !important;
- box-shadow: none;
- display: block;
- margin-left: auto;
- margin-right: auto;
- max-width: 600px;
-}
-
-#scale-logo .download {
- display: none;
-}
-#filter_type{
- border: 0;
- padding-left: 0;
- padding-top: 0;
-}
-#filter_type label {
- display: flex;
-}
-#filter_type label > span{
- margin-top: var(--spacing-lg);
- margin-right: 0.5em;
-}
-#filter_type label > .wrap{
- width: 103px;
-}
-#filter_type label > .wrap .wrap-inner{ 
- padding: 2px;
-}
-#filter_type label > .wrap .wrap-inner input{
- width: 1px
-}
-#filter-columns-type{
- border:0;
- padding:0.5;
-}
-#filter-columns-size{
- border:0;
- padding:0.5;
-}
-#box-filter > .form{
- border: 0
-}
-"""
-
-get_window_url_params = """
- function(url_params) {
- const params = new URLSearchParams(window.location.search);
- url_params = Object.fromEntries(params);
- return url_params;
- }
- """

src/display/formatting.py

@@ -1,27 +0,0 @@
-def model_hyperlink(link, model_name):
- return f'<a target="_blank" href="{link}" style="color: var(--link-text-color); text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>'
-
-
-def make_clickable_model(model_name):
- link = f"https://huggingface.co/{model_name}"
- return model_hyperlink(link, model_name)
-
-
-def styled_error(error):
- return f"<p style='color: red; font-size: 20px; text-align: center;'>{error}</p>"
-
-
-def styled_warning(warn):
- return f"<p style='color: orange; font-size: 20px; text-align: center;'>{warn}</p>"
-
-
-def styled_message(message):
- return f"<p style='color: green; font-size: 20px; text-align: center;'>{message}</p>"
-
-
-def has_no_nan_values(df, columns):
- return df[columns].notna().all(axis=1)
-
-
-def has_nan_values(df, columns):
- return df[columns].isna().any(axis=1)

src/display/utils.py

@@ -1,135 +0,0 @@
-from dataclasses import dataclass, make_dataclass
-from enum import Enum
-
-import pandas as pd
-
-from src.about import Tasks
-
-def fields(raw_class):
- return [v for k, v in raw_class.__dict__.items() if k[:2] != "__" and k[-2:] != "__"]
-
-
-# These classes are for user facing column names,
-# to avoid having to change them all around the code
-# when a modif is needed
-@dataclass
-class ColumnContent:
- name: str
- type: str
- displayed_by_default: bool
- hidden: bool = False
- never_hidden: bool = False
-
-## Leaderboard columns
-auto_eval_column_dict = []
-# Init
-auto_eval_column_dict.append(["model_type_symbol", ColumnContent, ColumnContent("T", "str", True, never_hidden=True)])
-auto_eval_column_dict.append(["model", ColumnContent, ColumnContent("Model", "markdown", True, never_hidden=True)])
-#Scores
-auto_eval_column_dict.append(["average", ColumnContent, ColumnContent("Average ⬆️", "number", True)])
-for task in Tasks:
- auto_eval_column_dict.append([task.name, ColumnContent, ColumnContent(task.value.col_name, "number", True)])
-# Model information
-auto_eval_column_dict.append(["model_type", ColumnContent, ColumnContent("Type", "str", False)])
-auto_eval_column_dict.append(["architecture", ColumnContent, ColumnContent("Architecture", "str", False)])
-auto_eval_column_dict.append(["weight_type", ColumnContent, ColumnContent("Weight type", "str", False, True)])
-auto_eval_column_dict.append(["precision", ColumnContent, ColumnContent("Precision", "str", False)])
-auto_eval_column_dict.append(["license", ColumnContent, ColumnContent("Hub License", "str", False)])
-auto_eval_column_dict.append(["params", ColumnContent, ColumnContent("#Params (B)", "number", False)])
-auto_eval_column_dict.append(["likes", ColumnContent, ColumnContent("Hub ❤️", "number", False)])
-auto_eval_column_dict.append(["still_on_hub", ColumnContent, ColumnContent("Available on the hub", "bool", False)])
-auto_eval_column_dict.append(["revision", ColumnContent, ColumnContent("Model sha", "str", False, False)])
-
-# We use make dataclass to dynamically fill the scores from Tasks
-AutoEvalColumn = make_dataclass("AutoEvalColumn", auto_eval_column_dict, frozen=True)
-
-## For the queue columns in the submission tab
-@dataclass(frozen=True)
-class EvalQueueColumn: # Queue column
- model = ColumnContent("model", "markdown", True)
- revision = ColumnContent("revision", "str", True)
- private = ColumnContent("private", "bool", True)
- precision = ColumnContent("precision", "str", True)
- weight_type = ColumnContent("weight_type", "str", "Original")
- status = ColumnContent("status", "str", True)
-
-## All the model information that we might need
-@dataclass
-class ModelDetails:
- name: str
- display_name: str = ""
- symbol: str = "" # emoji
-
-
-class ModelType(Enum):
- PT = ModelDetails(name="pretrained", symbol="🟢")
- FT = ModelDetails(name="fine-tuned", symbol="🔶")
- IFT = ModelDetails(name="instruction-tuned", symbol="⭕")
- RL = ModelDetails(name="RL-tuned", symbol="🟦")
- Unknown = ModelDetails(name="", symbol="?")
-
- def to_str(self, separator=" "):
- return f"{self.value.symbol}{separator}{self.value.name}"
-
- @staticmethod
- def from_str(type):
- if "fine-tuned" in type or "🔶" in type:
- return ModelType.FT
- if "pretrained" in type or "🟢" in type:
- return ModelType.PT
- if "RL-tuned" in type or "🟦" in type:
- return ModelType.RL
- if "instruction-tuned" in type or "⭕" in type:
- return ModelType.IFT
- return ModelType.Unknown
-
-class WeightType(Enum):
- Adapter = ModelDetails("Adapter")
- Original = ModelDetails("Original")
- Delta = ModelDetails("Delta")
-
-class Precision(Enum):
- float16 = ModelDetails("float16")
- bfloat16 = ModelDetails("bfloat16")
- float32 = ModelDetails("float32")
- #qt_8bit = ModelDetails("8bit")
- #qt_4bit = ModelDetails("4bit")
- #qt_GPTQ = ModelDetails("GPTQ")
- Unknown = ModelDetails("?")
-
- def from_str(precision):
- if precision in ["torch.float16", "float16"]:
- return Precision.float16
- if precision in ["torch.bfloat16", "bfloat16"]:
- return Precision.bfloat16
- if precision in ["float32"]:
- return Precision.float32
- #if precision in ["8bit"]:
- # return Precision.qt_8bit
- #if precision in ["4bit"]:
- # return Precision.qt_4bit
- #if precision in ["GPTQ", "None"]:
- # return Precision.qt_GPTQ
- return Precision.Unknown
-
-# Column selection
-COLS = [c.name for c in fields(AutoEvalColumn) if not c.hidden]
-TYPES = [c.type for c in fields(AutoEvalColumn) if not c.hidden]
-COLS_LITE = [c.name for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-TYPES_LITE = [c.type for c in fields(AutoEvalColumn) if c.displayed_by_default and not c.hidden]
-
-EVAL_COLS = [c.name for c in fields(EvalQueueColumn)]
-EVAL_TYPES = [c.type for c in fields(EvalQueueColumn)]
-
-BENCHMARK_COLS = [t.value.col_name for t in Tasks]
-
-NUMERIC_INTERVALS = {
- "?": pd.Interval(-1, 0, closed="right"),
- "~1.5": pd.Interval(0, 2, closed="right"),
- "~3": pd.Interval(2, 4, closed="right"),
- "~7": pd.Interval(4, 9, closed="right"),
- "~13": pd.Interval(9, 20, closed="right"),
- "~35": pd.Interval(20, 45, closed="right"),
- "~60": pd.Interval(45, 70, closed="right"),
- "70+": pd.Interval(70, 10000, closed="right"),
-}

src/envs.py

@@ -1,25 +0,0 @@
-import os
-
-from huggingface_hub import HfApi
-
-# Info to change for your repository
-# ----------------------------------
-TOKEN = os.environ.get("TOKEN") # A read/write token for your org
-
-OWNER = "demo-leaderboard-backend" # Change to your org - don't forget to create a results and request dataset, with the correct format!
-# ----------------------------------
-
-REPO_ID = f"{OWNER}/leaderboard"
-QUEUE_REPO = f"{OWNER}/requests"
-RESULTS_REPO = f"{OWNER}/results"
-
-# If you setup a cache later, just change HF_HOME
-CACHE_PATH=os.getenv("HF_HOME", ".")
-
-# Local caches
-EVAL_REQUESTS_PATH = os.path.join(CACHE_PATH, "eval-queue")
-EVAL_RESULTS_PATH = os.path.join(CACHE_PATH, "eval-results")
-EVAL_REQUESTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-queue-bk")
-EVAL_RESULTS_PATH_BACKEND = os.path.join(CACHE_PATH, "eval-results-bk")
-
-API = HfApi(token=TOKEN)

src/leaderboard/read_evals.py

@@ -1,196 +0,0 @@
-import glob
-import json
-import math
-import os
-from dataclasses import dataclass
-
-import dateutil
-import numpy as np
-
-from src.display.formatting import make_clickable_model
-from src.display.utils import AutoEvalColumn, ModelType, Tasks, Precision, WeightType
-from src.submission.check_validity import is_model_on_hub
-
-
-@dataclass
-class EvalResult:
- """Represents one full evaluation. Built from a combination of the result and request file for a given run.
- """
- eval_name: str # org_model_precision (uid)
- full_model: str # org/model (path on hub)
- org: str 
- model: str
- revision: str # commit hash, "" if main
- results: dict
- precision: Precision = Precision.Unknown
- model_type: ModelType = ModelType.Unknown # Pretrained, fine tuned, ...
- weight_type: WeightType = WeightType.Original # Original or Adapter
- architecture: str = "Unknown" 
- license: str = "?"
- likes: int = 0
- num_params: int = 0
- date: str = "" # submission date of request file
- still_on_hub: bool = False
-
- @classmethod
- def init_from_json_file(self, json_filepath):
- """Inits the result from the specific model result file"""
- with open(json_filepath) as fp:
- data = json.load(fp)
-
- config = data.get("config")
-
- # Precision
- precision = Precision.from_str(config.get("model_dtype"))
-
- # Get model and org
- org_and_model = config.get("model_name", config.get("model_args", None))
- org_and_model = org_and_model.split("/", 1)
-
- if len(org_and_model) == 1:
- org = None
- model = org_and_model[0]
- result_key = f"{model}_{precision.value.name}"
- else:
- org = org_and_model[0]
- model = org_and_model[1]
- result_key = f"{org}_{model}_{precision.value.name}"
- full_model = "/".join(org_and_model)
-
- still_on_hub, _, model_config = is_model_on_hub(
- full_model, config.get("model_sha", "main"), trust_remote_code=True, test_tokenizer=False
- )
- architecture = "?"
- if model_config is not None:
- architectures = getattr(model_config, "architectures", None)
- if architectures:
- architecture = ";".join(architectures)
-
- # Extract results available in this file (some results are split in several files)
- results = {}
- for task in Tasks:
- task = task.value
-
- # We average all scores of a given metric (not all metrics are present in all files)
- accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
- if accs.size == 0 or any([acc is None for acc in accs]):
- continue
-
- mean_acc = np.mean(accs) * 100.0
- results[task.benchmark] = mean_acc
-
- return self(
- eval_name=result_key,
- full_model=full_model,
- org=org,
- model=model,
- results=results,
- precision=precision, 
- revision= config.get("model_sha", ""),
- still_on_hub=still_on_hub,
- architecture=architecture
- )
-
- def update_with_request_file(self, requests_path):
- """Finds the relevant request file for the current model and updates info with it"""
- request_file = get_request_file_for_model(requests_path, self.full_model, self.precision.value.name)
-
- try:
- with open(request_file, "r") as f:
- request = json.load(f)
- self.model_type = ModelType.from_str(request.get("model_type", ""))
- self.weight_type = WeightType[request.get("weight_type", "Original")]
- self.license = request.get("license", "?")
- self.likes = request.get("likes", 0)
- self.num_params = request.get("params", 0)
- self.date = request.get("submitted_time", "")
- except Exception:
- print(f"Could not find request file for {self.org}/{self.model} with precision {self.precision.value.name}")
-
- def to_dict(self):
- """Converts the Eval Result to a dict compatible with our dataframe display"""
- average = sum([v for v in self.results.values() if v is not None]) / len(Tasks)
- data_dict = {
- "eval_name": self.eval_name, # not a column, just a save name,
- AutoEvalColumn.precision.name: self.precision.value.name,
- AutoEvalColumn.model_type.name: self.model_type.value.name,
- AutoEvalColumn.model_type_symbol.name: self.model_type.value.symbol,
- AutoEvalColumn.weight_type.name: self.weight_type.value.name,
- AutoEvalColumn.architecture.name: self.architecture,
- AutoEvalColumn.model.name: make_clickable_model(self.full_model),
- AutoEvalColumn.revision.name: self.revision,
- AutoEvalColumn.average.name: average,
- AutoEvalColumn.license.name: self.license,
- AutoEvalColumn.likes.name: self.likes,
- AutoEvalColumn.params.name: self.num_params,
- AutoEvalColumn.still_on_hub.name: self.still_on_hub,
- }
-
- for task in Tasks:
- data_dict[task.value.col_name] = self.results[task.value.benchmark]
-
- return data_dict
-
-
-def get_request_file_for_model(requests_path, model_name, precision):
- """Selects the correct request file for a given model. Only keeps runs tagged as FINISHED"""
- request_files = os.path.join(
- requests_path,
- f"{model_name}_eval_request_*.json",
- )
- request_files = glob.glob(request_files)
-
- # Select correct request file (precision)
- request_file = ""
- request_files = sorted(request_files, reverse=True)
- for tmp_request_file in request_files:
- with open(tmp_request_file, "r") as f:
- req_content = json.load(f)
- if (
- req_content["status"] in ["FINISHED"]
- and req_content["precision"] == precision.split(".")[-1]
- ):
- request_file = tmp_request_file
- return request_file
-
-
-def get_raw_eval_results(results_path: str, requests_path: str) -> list[EvalResult]:
- """From the path of the results folder root, extract all needed info for results"""
- model_result_filepaths = []
-
- for root, _, files in os.walk(results_path):
- # We should only have json files in model results
- if len(files) == 0 or any([not f.endswith(".json") for f in files]):
- continue
-
- # Sort the files by date
- try:
- files.sort(key=lambda x: x.removesuffix(".json").removeprefix("results_")[:-7])
- except dateutil.parser._parser.ParserError:
- files = [files[-1]]
-
- for file in files:
- model_result_filepaths.append(os.path.join(root, file))
-
- eval_results = {}
- for model_result_filepath in model_result_filepaths:
- # Creation of result
- eval_result = EvalResult.init_from_json_file(model_result_filepath)
- eval_result.update_with_request_file(requests_path)
-
- # Store results of same eval together
- eval_name = eval_result.eval_name
- if eval_name in eval_results.keys():
- eval_results[eval_name].results.update({k: v for k, v in eval_result.results.items() if v is not None})
- else:
- eval_results[eval_name] = eval_result
-
- results = []
- for v in eval_results.values():
- try:
- v.to_dict() # we test if the dict version is complete
- results.append(v)
- except KeyError: # not all eval values present
- continue
-
- return results

src/populate.py

@@ -1,58 +0,0 @@
-import json
-import os
-
-import pandas as pd
-
-from src.display.formatting import has_no_nan_values, make_clickable_model
-from src.display.utils import AutoEvalColumn, EvalQueueColumn
-from src.leaderboard.read_evals import get_raw_eval_results
-
-
-def get_leaderboard_df(results_path: str, requests_path: str, cols: list, benchmark_cols: list) -> pd.DataFrame:
- """Creates a dataframe from all the individual experiment results"""
- raw_data = get_raw_eval_results(results_path, requests_path)
- all_data_json = [v.to_dict() for v in raw_data]
-
- df = pd.DataFrame.from_records(all_data_json)
- df = df.sort_values(by=[AutoEvalColumn.average.name], ascending=False)
- df = df[cols].round(decimals=2)
-
- # filter out if any of the benchmarks have not been produced
- df = df[has_no_nan_values(df, benchmark_cols)]
- return raw_data, df
-
-
-def get_evaluation_queue_df(save_path: str, cols: list) -> list[pd.DataFrame]:
- """Creates the different dataframes for the evaluation queues requestes"""
- entries = [entry for entry in os.listdir(save_path) if not entry.startswith(".")]
- all_evals = []
-
- for entry in entries:
- if ".json" in entry:
- file_path = os.path.join(save_path, entry)
- with open(file_path) as fp:
- data = json.load(fp)
-
- data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
- data[EvalQueueColumn.revision.name] = data.get("revision", "main")
-
- all_evals.append(data)
- elif ".md" not in entry:
- # this is a folder
- sub_entries = [e for e in os.listdir(f"{save_path}/{entry}") if not e.startswith(".")]
- for sub_entry in sub_entries:
- file_path = os.path.join(save_path, entry, sub_entry)
- with open(file_path) as fp:
- data = json.load(fp)
-
- data[EvalQueueColumn.model.name] = make_clickable_model(data["model"])
- data[EvalQueueColumn.revision.name] = data.get("revision", "main")
- all_evals.append(data)
-
- pending_list = [e for e in all_evals if e["status"] in ["PENDING", "RERUN"]]
- running_list = [e for e in all_evals if e["status"] == "RUNNING"]
- finished_list = [e for e in all_evals if e["status"].startswith("FINISHED") or e["status"] == "PENDING_NEW_EVAL"]
- df_pending = pd.DataFrame.from_records(pending_list, columns=cols)
- df_running = pd.DataFrame.from_records(running_list, columns=cols)
- df_finished = pd.DataFrame.from_records(finished_list, columns=cols)
- return df_finished[cols], df_running[cols], df_pending[cols]

src/submission/check_validity.py

@@ -1,99 +0,0 @@
-import json
-import os
-import re
-from collections import defaultdict
-from datetime import datetime, timedelta, timezone
-
-import huggingface_hub
-from huggingface_hub import ModelCard
-from huggingface_hub.hf_api import ModelInfo
-from transformers import AutoConfig
-from transformers.models.auto.tokenization_auto import AutoTokenizer
-
-def check_model_card(repo_id: str) -> tuple[bool, str]:
- """Checks if the model card and license exist and have been filled"""
- try:
- card = ModelCard.load(repo_id)
- except huggingface_hub.utils.EntryNotFoundError:
- return False, "Please add a model card to your model to explain how you trained/fine-tuned it."
-
- # Enforce license metadata
- if card.data.license is None:
- if not ("license_name" in card.data and "license_link" in card.data):
- return False, (
- "License not found. Please add a license to your model card using the `license` metadata or a"
- " `license_name`/`license_link` pair."
- )
-
- # Enforce card content
- if len(card.text) < 200:
- return False, "Please add a description to your model card, it is too short."
-
- return True, ""
-
-def is_model_on_hub(model_name: str, revision: str, token: str = None, trust_remote_code=False, test_tokenizer=False) -> tuple[bool, str]:
- """Checks if the model model_name is on the hub, and whether it (and its tokenizer) can be loaded with AutoClasses."""
- try:
- config = AutoConfig.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
- if test_tokenizer:
- try:
- tk = AutoTokenizer.from_pretrained(model_name, revision=revision, trust_remote_code=trust_remote_code, token=token)
- except ValueError as e:
- return (
- False,
- f"uses a tokenizer which is not in a transformers release: {e}",
- None
- )
- except Exception as e:
- return (False, "'s tokenizer cannot be loaded. Is your tokenizer class in a stable transformers release, and correctly configured?", None)
- return True, None, config
-
- except ValueError:
- return (
- False,
- "needs to be launched with `trust_remote_code=True`. For safety reason, we do not allow these models to be automatically submitted to the leaderboard.",
- None
- )
-
- except Exception as e:
- return False, "was not found on hub!", None
-
-
-def get_model_size(model_info: ModelInfo, precision: str):
- """Gets the model size from the configuration, or the model name if the configuration does not contain the information."""
- try:
- model_size = round(model_info.safetensors["total"] / 1e9, 3)
- except (AttributeError, TypeError):
- return 0 # Unknown model sizes are indicated as 0, see NUMERIC_INTERVALS in app.py
-
- size_factor = 8 if (precision == "GPTQ" or "gptq" in model_info.modelId.lower()) else 1
- model_size = size_factor * model_size
- return model_size
-
-def get_model_arch(model_info: ModelInfo):
- """Gets the model architecture from the configuration"""
- return model_info.config.get("architectures", "Unknown")
-
-def already_submitted_models(requested_models_dir: str) -> set[str]:
- """Gather a list of already submitted models to avoid duplicates"""
- depth = 1
- file_names = []
- users_to_submission_dates = defaultdict(list)
-
- for root, _, files in os.walk(requested_models_dir):
- current_depth = root.count(os.sep) - requested_models_dir.count(os.sep)
- if current_depth == depth:
- for file in files:
- if not file.endswith(".json"):
- continue
- with open(os.path.join(root, file), "r") as f:
- info = json.load(f)
- file_names.append(f"{info['model']}_{info['revision']}_{info['precision']}")
-
- # Select organisation
- if info["model"].count("/") == 0 or "submitted_time" not in info:
- continue
- organisation, _ = info["model"].split("/")
- users_to_submission_dates[organisation].append(info["submitted_time"])
-
- return set(file_names), users_to_submission_dates

src/submission/submit.py

@@ -1,119 +0,0 @@
-import json
-import os
-from datetime import datetime, timezone
-
-from src.display.formatting import styled_error, styled_message, styled_warning
-from src.envs import API, EVAL_REQUESTS_PATH, TOKEN, QUEUE_REPO
-from src.submission.check_validity import (
- already_submitted_models,
- check_model_card,
- get_model_size,
- is_model_on_hub,
-)
-
-REQUESTED_MODELS = None
-USERS_TO_SUBMISSION_DATES = None
-
-def add_new_eval(
- model: str,
- base_model: str,
- revision: str,
- precision: str,
- weight_type: str,
- model_type: str,
-):
- global REQUESTED_MODELS
- global USERS_TO_SUBMISSION_DATES
- if not REQUESTED_MODELS:
- REQUESTED_MODELS, USERS_TO_SUBMISSION_DATES = already_submitted_models(EVAL_REQUESTS_PATH)
-
- user_name = ""
- model_path = model
- if "/" in model:
- user_name = model.split("/")[0]
- model_path = model.split("/")[1]
-
- precision = precision.split(" ")[0]
- current_time = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
-
- if model_type is None or model_type == "":
- return styled_error("Please select a model type.")
-
- # Does the model actually exist?
- if revision == "":
- revision = "main"
-
- # Is the model on the hub?
- if weight_type in ["Delta", "Adapter"]:
- base_model_on_hub, error, _ = is_model_on_hub(model_name=base_model, revision=revision, token=TOKEN, test_tokenizer=True)
- if not base_model_on_hub:
- return styled_error(f'Base model "{base_model}" {error}')
-
- if not weight_type == "Adapter":
- model_on_hub, error, _ = is_model_on_hub(model_name=model, revision=revision, token=TOKEN, test_tokenizer=True)
- if not model_on_hub:
- return styled_error(f'Model "{model}" {error}')
-
- # Is the model info correctly filled?
- try:
- model_info = API.model_info(repo_id=model, revision=revision)
- except Exception:
- return styled_error("Could not get your model information. Please fill it up properly.")
-
- model_size = get_model_size(model_info=model_info, precision=precision)
-
- # Were the model card and license filled?
- try:
- license = model_info.cardData["license"]
- except Exception:
- return styled_error("Please select a license for your model")
-
- modelcard_OK, error_msg = check_model_card(model)
- if not modelcard_OK:
- return styled_error(error_msg)
-
- # Seems good, creating the eval
- print("Adding new eval")
-
- eval_entry = {
- "model": model,
- "base_model": base_model,
- "revision": revision,
- "precision": precision,
- "weight_type": weight_type,
- "status": "PENDING",
- "submitted_time": current_time,
- "model_type": model_type,
- "likes": model_info.likes,
- "params": model_size,
- "license": license,
- "private": False,
- }
-
- # Check for duplicate submission
- if f"{model}_{revision}_{precision}" in REQUESTED_MODELS:
- return styled_warning("This model has been already submitted.")
-
- print("Creating eval file")
- OUT_DIR = f"{EVAL_REQUESTS_PATH}/{user_name}"
- os.makedirs(OUT_DIR, exist_ok=True)
- out_path = f"{OUT_DIR}/{model_path}_eval_request_False_{precision}_{weight_type}.json"
-
- with open(out_path, "w") as f:
- f.write(json.dumps(eval_entry))
-
- print("Uploading eval file")
- API.upload_file(
- path_or_fileobj=out_path,
- path_in_repo=out_path.split("eval-queue/")[1],
- repo_id=QUEUE_REPO,
- repo_type="dataset",
- commit_message=f"Add {model} to eval queue",
- )
-
- # Remove the local file
- os.remove(out_path)
-
- return styled_message(
- "Your request has been submitted to the evaluation queue!\nPlease wait for up to an hour for the model to show in the PENDING list."
- )