|
__all__ = ['block', 'make_clickable_model', 'make_clickable_user', 'get_submissions'] |
|
|
|
import gradio as gr |
|
import pandas as pd |
|
import json |
|
import io |
|
|
|
from constants import * |
|
|
|
global data_component, data_component_150, filter_component |
|
|
|
def upload_file(files): |
|
file_paths = [file.name for file in files] |
|
return file_paths |
|
|
|
def compute_scores(input_data): |
|
return [None, [ |
|
input_data["Average_MTScore"], |
|
input_data["Average_CHScore"], |
|
input_data["Average_GPT4o-MTScore"], |
|
input_data["Average_UMT-FVD"], |
|
input_data["Average_UMTScore"] |
|
]] |
|
|
|
def add_new_eval( |
|
input_file, |
|
model_name_textbox: str, |
|
revision_name_textbox: str, |
|
backbone_type_dropdown: str, |
|
model_link: str, |
|
): |
|
if input_file is None: |
|
return "Error! Empty file!" |
|
else: |
|
input_json = json.load(io.BytesIO(input_file)) |
|
|
|
if model_name_textbox not in input_json: |
|
return f"Error! Model '{model_name_textbox}' not found in input file!" |
|
|
|
selected_model_data = input_json[model_name_textbox] |
|
|
|
scores = compute_scores(selected_model_data) |
|
input_data = scores[1] |
|
input_data = [float(i) for i in input_data] |
|
|
|
csv_data = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH) |
|
|
|
if revision_name_textbox == '': |
|
col = csv_data.shape[0] |
|
model_name = model_name_textbox |
|
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in csv_data['Model']] |
|
assert model_name not in name_list |
|
else: |
|
model_name = revision_name_textbox |
|
model_name_list = csv_data['Model'] |
|
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in model_name_list] |
|
if revision_name_textbox not in name_list: |
|
col = csv_data.shape[0] |
|
else: |
|
col = name_list.index(revision_name_textbox) |
|
|
|
if model_link == '': |
|
model_name = model_name |
|
else: |
|
model_name = '[' + model_name + '](' + model_link + ')' |
|
|
|
backbone = backbone_type_dropdown |
|
|
|
new_data = [ |
|
model_name, |
|
backbone, |
|
input_data[3], |
|
input_data[4], |
|
input_data[0], |
|
input_data[1], |
|
input_data[2], |
|
] |
|
csv_data.loc[col] = new_data |
|
csv_data.to_csv(CSV_DIR_CHRONOMAGIC_BENCH, index=False) |
|
return "Evaluation successfully submitted!" |
|
|
|
def add_new_eval_150( |
|
input_file, |
|
model_name_textbox: str, |
|
revision_name_textbox: str, |
|
backbone_type_dropdown: str, |
|
model_link: str, |
|
): |
|
if input_file is None: |
|
return "Error! Empty file!" |
|
else: |
|
input_json = json.load(io.BytesIO(input_file)) |
|
|
|
if model_name_textbox not in input_json: |
|
return f"Error! Model '{model_name_textbox}' not found in input file!" |
|
|
|
selected_model_data = input_json[model_name_textbox] |
|
|
|
scores = compute_scores(selected_model_data) |
|
input_data = scores[1] |
|
input_data = [float(i) for i in input_data] |
|
|
|
csv_data = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150) |
|
|
|
if revision_name_textbox == '': |
|
col = csv_data.shape[0] |
|
model_name = model_name_textbox |
|
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in csv_data['Model']] |
|
assert model_name not in name_list |
|
else: |
|
model_name = revision_name_textbox |
|
model_name_list = csv_data['Model'] |
|
name_list = [name.split(']')[0][1:] if name.endswith(')') else name for name in model_name_list] |
|
if revision_name_textbox not in name_list: |
|
col = csv_data.shape[0] |
|
else: |
|
col = name_list.index(revision_name_textbox) |
|
|
|
if model_link == '': |
|
model_name = model_name |
|
else: |
|
model_name = '[' + model_name + '](' + model_link + ')' |
|
|
|
backbone = backbone_type_dropdown |
|
|
|
new_data = [ |
|
model_name, |
|
backbone, |
|
input_data[3], |
|
input_data[4], |
|
input_data[0], |
|
input_data[1], |
|
input_data[2], |
|
] |
|
csv_data.loc[col] = new_data |
|
csv_data.to_csv(CSV_DIR_CHRONOMAGIC_BENCH_150, index=False) |
|
return "Evaluation (150) successfully submitted!" |
|
|
|
def get_baseline_df(): |
|
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH) |
|
df = df.sort_values(by="MTScore↑", ascending=False) |
|
present_columns = MODEL_INFO + checkbox_group.value |
|
df = df[present_columns] |
|
return df |
|
|
|
def get_baseline_df_150(): |
|
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150) |
|
df = df.sort_values(by="MTScore↑", ascending=False) |
|
present_columns = MODEL_INFO + checkbox_group_150.value |
|
df = df[present_columns] |
|
return df |
|
|
|
def get_all_df(): |
|
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH) |
|
df = df.sort_values(by="MTScore↑", ascending=False) |
|
return df |
|
|
|
def get_all_df_150(): |
|
df = pd.read_csv(CSV_DIR_CHRONOMAGIC_BENCH_150) |
|
df = df.sort_values(by="MTScore↑", ascending=False) |
|
return df |
|
|
|
block = gr.Blocks() |
|
|
|
|
|
with block: |
|
gr.Markdown( |
|
LEADERBORAD_INTRODUCTION |
|
) |
|
with gr.Tabs(elem_classes="tab-buttons") as tabs: |
|
|
|
with gr.TabItem("🏅 ChronoMagic-Bench", elem_id="ChronoMagic-Bench-tab-table", id=0): |
|
with gr.Row(): |
|
with gr.Accordion("Citation", open=False): |
|
citation_button = gr.Textbox( |
|
value=CITATION_BUTTON_TEXT, |
|
label=CITATION_BUTTON_LABEL, |
|
elem_id="citation-button", |
|
show_copy_button=True |
|
) |
|
|
|
gr.Markdown( |
|
TABLE_INTRODUCTION |
|
) |
|
|
|
checkbox_group = gr.CheckboxGroup( |
|
choices=ALL_RESULTS, |
|
value=SELECTED_RESULTS, |
|
label="Select options", |
|
interactive=True, |
|
) |
|
|
|
data_component = gr.components.Dataframe( |
|
value=get_baseline_df, |
|
headers=COLUMN_NAMES, |
|
type="pandas", |
|
datatype=DATA_TITILE_TYPE, |
|
interactive=False, |
|
visible=True, |
|
) |
|
|
|
def on_checkbox_group_change(selected_columns): |
|
selected_columns = [item for item in ALL_RESULTS if item in selected_columns] |
|
present_columns = MODEL_INFO + selected_columns |
|
updated_data = get_all_df()[present_columns] |
|
updated_data = updated_data.sort_values(by=present_columns[1], ascending=False) |
|
updated_headers = present_columns |
|
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] |
|
|
|
filter_component = gr.components.Dataframe( |
|
value=updated_data, |
|
headers=updated_headers, |
|
type="pandas", |
|
datatype=update_datatype, |
|
interactive=False, |
|
visible=True, |
|
) |
|
|
|
return filter_component |
|
|
|
checkbox_group.change(fn=on_checkbox_group_change, inputs=checkbox_group, outputs=data_component) |
|
|
|
|
|
with gr.TabItem("🏅 ChronoMagic-Bench-150", elem_id="ChronoMagic-Bench-150-tab-table", id=1): |
|
with gr.Row(): |
|
with gr.Accordion("Citation", open=False): |
|
citation_button = gr.Textbox( |
|
value=CITATION_BUTTON_TEXT, |
|
label=CITATION_BUTTON_LABEL, |
|
elem_id="citation-button", |
|
show_copy_button=True |
|
) |
|
|
|
gr.Markdown( |
|
TABLE_INTRODUCTION |
|
) |
|
|
|
checkbox_group_150 = gr.CheckboxGroup( |
|
choices=ALL_RESULTS, |
|
value=SELECTED_RESULTS_150, |
|
label="Select options", |
|
interactive=True, |
|
) |
|
|
|
data_component_150 = gr.components.Dataframe( |
|
value=get_baseline_df_150, |
|
headers=COLUMN_NAMES, |
|
type="pandas", |
|
datatype=DATA_TITILE_TYPE, |
|
interactive=False, |
|
visible=True, |
|
) |
|
|
|
def on_checkbox_group_150_change(selected_columns): |
|
selected_columns = [item for item in ALL_RESULTS if item in selected_columns] |
|
present_columns = MODEL_INFO + selected_columns |
|
updated_data = get_all_df_150()[present_columns] |
|
updated_data = updated_data.sort_values(by=present_columns[1], ascending=False) |
|
updated_headers = present_columns |
|
update_datatype = [DATA_TITILE_TYPE[COLUMN_NAMES.index(x)] for x in updated_headers] |
|
|
|
filter_component = gr.components.Dataframe( |
|
value=updated_data, |
|
headers=updated_headers, |
|
type="pandas", |
|
datatype=update_datatype, |
|
interactive=False, |
|
visible=True, |
|
) |
|
|
|
return filter_component |
|
|
|
checkbox_group_150.change(fn=on_checkbox_group_150_change, inputs=checkbox_group_150, outputs=data_component_150) |
|
|
|
|
|
with gr.TabItem("🚀 Submit here! ", elem_id="seed-benchmark-tab-table", id=2): |
|
with gr.Row(): |
|
gr.Markdown(SUBMIT_INTRODUCTION, elem_classes="markdown-text") |
|
|
|
with gr.Row(): |
|
gr.Markdown("# ✉️✨ Submit your model evaluation json file here!", elem_classes="markdown-text") |
|
|
|
with gr.Row(): |
|
with gr.Column(): |
|
model_name_textbox = gr.Textbox( |
|
label="Model name", placeholder="MagicTime" |
|
) |
|
revision_name_textbox = gr.Textbox( |
|
label="Revision Model Name", placeholder="MagicTime" |
|
) |
|
backbone_type_dropdown = gr.Dropdown( |
|
label="Backbone Type", |
|
choices=["DiT", "U-Net"], |
|
value="DiT" |
|
) |
|
model_link = gr.Textbox( |
|
label="Model Link", placeholder="https://github.com/PKU-YuanGroup/MagicTime" |
|
) |
|
|
|
with gr.Column(): |
|
input_file = gr.File(label="Click to Upload a json File", type='binary') |
|
submit_button = gr.Button("Submit Eval (ChronoMagic-Bench)") |
|
submit_button_150 = gr.Button("Submit Eval (ChronoMagic-Bench-150)") |
|
|
|
submission_result = gr.Markdown() |
|
submit_button.click( |
|
add_new_eval, |
|
inputs=[ |
|
input_file, |
|
model_name_textbox, |
|
revision_name_textbox, |
|
backbone_type_dropdown, |
|
model_link, |
|
], |
|
outputs=submission_result, |
|
) |
|
submit_button_150.click( |
|
add_new_eval_150, |
|
inputs=[ |
|
input_file, |
|
model_name_textbox, |
|
revision_name_textbox, |
|
backbone_type_dropdown, |
|
model_link, |
|
], |
|
outputs = submission_result, |
|
) |
|
|
|
with gr.Row(): |
|
data_run = gr.Button("Refresh") |
|
data_run.click( |
|
get_baseline_df, outputs=data_component |
|
) |
|
data_run.click( |
|
get_baseline_df_150, outputs=data_component_150 |
|
) |
|
|
|
block.launch() |