import os
import shutil
import numpy as np
import gradio as gr
from huggingface_hub import Repository, HfApi
from transformers import AutoConfig
import json
from apscheduler.schedulers.background import BackgroundScheduler
import pandas as pd
import datetime
# clone / pull the lmeh eval data
H4_TOKEN = os.environ.get("H4_TOKEN", None)
LMEH_REPO = "HuggingFaceH4/lmeh_evaluations"
repo=None
if H4_TOKEN:
print("pulling repo")
# try:
# shutil.rmtree("./evals/")
# except:
# pass
repo = Repository(
local_dir="./evals/", clone_from=LMEH_REPO, use_auth_token=H4_TOKEN, repo_type="dataset"
)
repo.git_pull()
# parse the results
BENCHMARKS = ["arc_challenge", "hellaswag", "hendrycks", "truthfulqa_mc"]
BENCH_TO_NAME = {
"arc_challenge":"ARC (25-shot) ⬆️",
"hellaswag":"HellaSwag (10-shot) ⬆️",
"hendrycks":"MMLU (5-shot) ⬆️",
"truthfulqa_mc":"TruthQA (0-shot) ⬆️",
}
METRICS = ["acc_norm", "acc_norm", "acc_norm", "mc2"]
def make_clickable_model(model_name):
# remove user from model name
#model_name_show = ' '.join(model_name.split('/')[1:])
link = "https://huggingface.co/" + model_name
return f'{model_name}'
def load_results(model, benchmark, metric):
file_path = os.path.join("evals", model, f"{model}-eval_{benchmark}.json")
if not os.path.exists(file_path):
return 0.0, None
with open(file_path) as fp:
data = json.load(fp)
accs = np.array([v[metric] for k, v in data["results"].items()])
mean_acc = np.mean(accs)
return mean_acc, data["config"]["model_args"]
def get_n_params(base_model):
# config = AutoConfig.from_pretrained(model_name)
# # Retrieve the number of parameters from the configuration
# try:
# num_params = config.n_parameters
# except AttributeError:
# print(f"Error: The number of parameters is not available in the config for the model '{model_name}'.")
# return None
# return num_params
now = datetime.datetime.now()
time_string = now.strftime("%Y-%m-%d %H:%M:%S")
return time_string
COLS = ["eval_name", "# params", "total ⬆️", "ARC (25-shot) ⬆️", "HellaSwag (10-shot) ⬆️", "MMLU (5-shot) ⬆️", "TruthQA (0-shot) ⬆️", "base_model"]
TYPES = ["str","str", "number", "number", "number", "number", "number","markdown", ]
EVAL_COLS = ["model","# params", "private", "8bit_eval", "is_delta_weight", "status"]
EVAL_TYPES = ["markdown","str", "bool", "bool", "bool", "str"]
def get_leaderboard():
if repo:
print("pulling changes")
repo.git_pull()
entries = [entry for entry in os.listdir("evals") if not (entry.startswith('.') or entry=="eval_requests" or entry=="evals")]
model_directories = [entry for entry in entries if os.path.isdir(os.path.join("evals", entry))]
all_data = []
for model in model_directories:
model_data = {"base_model": None, "eval_name": model}
for benchmark, metric in zip(BENCHMARKS, METRICS):
value, base_model = load_results(model, benchmark, metric)
model_data[BENCH_TO_NAME[benchmark]] = round(value,3)
if base_model is not None: # in case the last benchmark failed
model_data["base_model"] = base_model
model_data["total ⬆️"] = round(sum(model_data[benchmark] for benchmark in BENCH_TO_NAME.values()),3)
if model_data["base_model"] is not None:
model_data["base_model"] = make_clickable_model(model_data["base_model"])
model_data["# params"] = get_n_params(model_data["base_model"])
if model_data["base_model"] is not None:
all_data.append(model_data)
dataframe = pd.DataFrame.from_records(all_data)
dataframe = dataframe.sort_values(by=['total ⬆️'], ascending=False)
dataframe = dataframe[COLS]
return dataframe
def get_eval_table():
if repo:
print("pulling changes for eval")
repo.git_pull()
entries = [entry for entry in os.listdir("evals/eval_requests") if not entry.startswith('.')]
all_evals = []
for entry in entries:
print(entry)
if ".json"in entry:
file_path = os.path.join("evals/eval_requests", entry)
with open(file_path) as fp:
data = json.load(fp)
data["# params"] = get_n_params(data["model"])
data["model"] = make_clickable_model(data["model"])
all_evals.append(data)
else:
# this is a folder
sub_entries = [e for e in os.listdir(f"evals/eval_requests/{entry}") if not e.startswith('.')]
for sub_entry in sub_entries:
file_path = os.path.join("evals/eval_requests", entry, sub_entry)
with open(file_path) as fp:
data = json.load(fp)
data["# params"] = get_n_params(data["model"])
data["model"] = make_clickable_model(data["model"])
all_evals.append(data)
dataframe = pd.DataFrame.from_records(all_evals)
return dataframe[EVAL_COLS]
leaderboard = get_leaderboard()
eval_queue = get_eval_table()
def is_model_on_hub(model_name) -> bool:
try:
config = AutoConfig.from_pretrained(model_name)
return True
except Exception as e:
print("Could not get the model config from the hub")
print(e)
return False
def add_new_eval(model:str, private:bool, is_8_bit_eval: bool, is_delta_weight:bool):
# check the model actually exists before adding the eval
if not is_model_on_hub(model):
print(model, "not found on hub")
return
print("adding new eval")
eval_entry = {
"model" : model,
"private" : private,
"8bit_eval" : is_8_bit_eval,
"is_delta_weight" : is_delta_weight,
"status" : "PENDING"
}
user_name = ""
model_path = model
if "/" in model:
user_name = model.split("/")[0]
model_path = model.split("/")[1]
OUT_DIR=f"eval_requests/{user_name}"
os.makedirs(OUT_DIR, exist_ok=True)
out_path = f"{OUT_DIR}/{model_path}_eval_request_{private}_{is_8_bit_eval}_{is_delta_weight}.json"
with open(out_path, "w") as f:
f.write(json.dumps(eval_entry))
LMEH_REPO = "HuggingFaceH4/lmeh_evaluations"
api = HfApi()
api.upload_file(
path_or_fileobj=out_path,
path_in_repo=out_path,
repo_id=LMEH_REPO,
token=H4_TOKEN,
repo_type="dataset",
)
def refresh():
return get_leaderboard(), get_eval_table()
block = gr.Blocks()
with block:
with gr.Row():
gr.Markdown(f"""
# 🤗 H4 Model Evaluation leaderboard using the LMEH benchmark suite .
Evaluation is performed against 4 popular benchmarks AI2 Reasoning Challenge, HellaSwag, MMLU, and TruthFul QC MC. To run your own benchmarks, refer to the README in the H4 repo.
""")
with gr.Row():
leaderboard_table = gr.components.Dataframe(value=leaderboard, headers=COLS,
datatype=TYPES, max_rows=5)
with gr.Row():
gr.Markdown(f"""
# Evaluation Queue for the LMEH benchmarks, these models will be automatically evaluated on the 🤗 cluster
""")
with gr.Row():
eval_table = gr.components.Dataframe(value=eval_queue, headers=EVAL_COLS,
datatype=EVAL_TYPES, max_rows=5)
with gr.Row():
refresh_button = gr.Button("Refresh")
refresh_button.click(refresh, inputs=[], outputs=[leaderboard_table, eval_table])
with gr.Accordion("Submit a new model for evaluation"):
# with gr.Row():
# gr.Markdown(f"""# Submit a new model for evaluation""")
with gr.Row():
model_name_textbox = gr.Textbox(label="model_name")
is_8bit_toggle = gr.Checkbox(False, label="8 bit Eval?")
private = gr.Checkbox(False, label="Private?")
is_delta_weight = gr.Checkbox(False, label="Delta Weights?")
with gr.Row():
submit_button = gr.Button("Submit Eval")
submit_button.click(add_new_eval, [model_name_textbox, is_8bit_toggle, private, is_delta_weight])
print("adding refresh leaderboard")
def refresh_leaderboard():
leaderboard_table = get_leaderboard()
print("leaderboard updated")
scheduler = BackgroundScheduler()
scheduler.add_job(func=refresh_leaderboard, trigger="interval", seconds=300) # refresh every 5 mins
scheduler.start()
block.launch()