Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
import os | |
import shutil | |
import numpy as np | |
import gradio as gr | |
from huggingface_hub import Repository | |
import json | |
from apscheduler.schedulers.background import BackgroundScheduler | |
import pandas as pd | |
# clone / pull the lmeh eval data | |
H4_TOKEN = os.environ.get("H4_TOKEN", None) | |
repo=None | |
if H4_TOKEN: | |
# try: | |
# shutil.rmtree("./evals/") | |
# except: | |
# pass | |
repo = Repository( | |
local_dir="./evals/", clone_from="HuggingFaceH4/lmeh_evaluations", use_auth_token=H4_TOKEN, repo_type="dataset" | |
) | |
repo.git_pull() | |
# parse the results | |
BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"] | |
BENCH_TO_NAME = { | |
"arc_challenge":"ARC", | |
"hellaswag":"HellaSwag", | |
"hendrycks":"MMLU", | |
"truthfulqa_mc":"TruthQA", | |
} | |
METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"] | |
entries = [entry for entry in os.listdir("evals") if not entry.startswith('.')] | |
model_directories = [entry for entry in entries if os.path.isdir(os.path.join("evals", entry))] | |
def make_clickable_model(model_name): | |
# remove user from model name | |
#model_name_show = ' '.join(model_name.split('/')[1:]) | |
link = "https://huggingface.co/" + model_name | |
return f'<a target="_blank" href="{link}" style="color: blue; text-decoration: underline;text-decoration-style: dotted;">{model_name}</a>' | |
def load_results(model, benchmark, metric): | |
file_path = os.path.join("evals", model, f"{model}-eval_{benchmark}.json") | |
if not os.path.exists(file_path): | |
return 0.0, None | |
with open(file_path) as fp: | |
data = json.load(fp) | |
accs = np.array([v[metric] for k, v in data["results"].items()]) | |
mean_acc = np.mean(accs) | |
return mean_acc, data["config"]["model_args"] | |
COLS = ["eval_name", "total", "ARC", "HellaSwag", "MMLU", "TruthQA", "base_model"] | |
TYPES = ["str", "number", "number", "number", "number", "number","markdown", ] | |
def get_leaderboard(): | |
if repo: | |
repo.git_pull() | |
all_data = [] | |
for model in model_directories: | |
model_data = {"base_model": None} | |
model_data = {"eval_name": model} | |
for benchmark, metric in zip(BENCHMARKS, METRICS): | |
value, base_model = load_results(model, benchmark, metric) | |
model_data[BENCH_TO_NAME[benchmark]] = value | |
if base_model is not None: # in case the last benchmark failed | |
model_data["base_model"] = base_model | |
model_data["total"] = sum(model_data[benchmark] for benchmark in BENCH_TO_NAME.values()) | |
if model_data["base_model"] is not None: | |
model_data["base_model"] = make_clickable_model(model_data["base_model"]) | |
all_data.append(model_data) | |
dataframe = pd.DataFrame.from_records(all_data) | |
dataframe = dataframe.sort_values(by=['total'], ascending=False) | |
dataframe = dataframe[COLS] | |
return dataframe | |
leaderboard = get_leaderboard() | |
block = gr.Blocks() | |
with block: | |
gr.Markdown(f""" | |
# H4 Model Evaluation leaderboard using the <a href="https://github.com/EleutherAI/lm-evaluation-harness" target="_blank"> LMEH benchmark suite </a>. | |
Evaluation is performed against 4 popular benchmarks AI2 Reasoning Challenge, HellaSwag, MMLU, and TruthFul QC MC. To run your own benchmarks, refer to the README in the H4 repo. | |
""") | |
with gr.Row(): | |
leaderboard_table = gr.components.Dataframe(value=leaderboard, headers=COLS, | |
datatype=TYPES, max_rows=5) | |
with gr.Row(): | |
refresh_button = gr.Button("Refresh") | |
refresh_button.click(get_leaderboard, inputs=[], outputs=leaderboard_table) | |
block.launch() | |
def refresh_leaderboard(): | |
leaderboard_table = get_leaderboard() | |
print("leaderboard updated") | |
scheduler = BackgroundScheduler() | |
scheduler.add_job(func=refresh_leaderboard, trigger="interval", seconds=300) # refresh every 5 mins | |
scheduler.start() |