better hanfdle refresh

app.py

@@ -1,6 +1,5 @@
 import json
 import os
-import re
 
 import gradio as gr
 import numpy as np
@@ -9,7 +8,7 @@ from apscheduler.schedulers.background import BackgroundScheduler
 from huggingface_hub import HfApi
 
 from src.backend import backend_routine
-
+import glob
 from src.logging import configure_root_logger, setup_logger
 
 
@@ -17,71 +16,72 @@ configure_root_logger()
 logger = setup_logger(__name__)
 
 API = HfApi(token=os.environ.get("TOKEN"))
-RESULTS_REPO = f"open-rl-leaderboard/results"
+RESULTS_REPO = "open-rl-leaderboard/results"
+REFRESH_RATE = 5 * 60 # 5 minutes
 ALL_ENV_IDS = {
  "Atari": [
- "Adventure",
- "AirRaid",
- "Alien",
- "Amidar",
- "Assault",
- "Asterix",
- "Asteroids",
- "Atlantis",
- "BankHeist",
- "BattleZone",
- "BeamRider",
- "Berzerk",
- "Bowling",
- "Boxing",
- "Breakout",
- "Carnival",
- "Centipede",
- "ChopperCommand",
- "CrazyClimber",
- "Defender",
- "DemonAttack",
- "DoubleDunk",
- "ElevatorAction",
- "Enduro",
- "FishingDerby",
- "Freeway",
- "Frostbite",
- "Gopher",
- "Gravitar",
- "Hero",
- "IceHockey",
- "Jamesbond",
- "JourneyEscape",
- "Kangaroo",
- "Krull",
- "KungFuMaster",
- "MontezumaRevenge",
- "MsPacman",
- "NameThisGame",
- "Phoenix",
- "Pitfall",
- "Pong",
- "Pooyan",
- "PrivateEye",
- "Qbert",
- "Riverraid",
- "RoadRunner",
- "Robotank",
- "Seaquest",
- "Skiing",
- "Solaris",
- "SpaceInvaders",
- "StarGunner",
- "Tennis",
- "TimePilot",
- "Tutankham",
- "UpNDown",
- "Venture",
- "VideoPinball",
- "WizardOfWor",
- "YarsRevenge",
- "Zaxxon",
+ "AdventureNoFrameskip-v4",
+ "AirRaidNoFrameskip-v4",
+ "AlienNoFrameskip-v4",
+ "AmidarNoFrameskip-v4",
+ "AssaultNoFrameskip-v4",
+ "AsterixNoFrameskip-v4",
+ "AsteroidsNoFrameskip-v4",
+ "AtlantisNoFrameskip-v4",
+ "BankHeistNoFrameskip-v4",
+ "BattleZoneNoFrameskip-v4",
+ "BeamRiderNoFrameskip-v4",
+ "BerzerkNoFrameskip-v4",
+ "BowlingNoFrameskip-v4",
+ "BoxingNoFrameskip-v4",
+ "BreakoutNoFrameskip-v4",
+ "CarnivalNoFrameskip-v4",
+ "CentipedeNoFrameskip-v4",
+ "ChopperCommandNoFrameskip-v4",
+ "CrazyClimberNoFrameskip-v4",
+ "DefenderNoFrameskip-v4",
+ "DemonAttackNoFrameskip-v4",
+ "DoubleDunkNoFrameskip-v4",
+ "ElevatorActionNoFrameskip-v4",
+ "EnduroNoFrameskip-v4",
+ "FishingDerbyNoFrameskip-v4",
+ "FreewayNoFrameskip-v4",
+ "FrostbiteNoFrameskip-v4",
+ "GopherNoFrameskip-v4",
+ "GravitarNoFrameskip-v4",
+ "HeroNoFrameskip-v4",
+ "IceHockeyNoFrameskip-v4",
+ "JamesbondNoFrameskip-v4",
+ "JourneyEscapeNoFrameskip-v4",
+ "KangarooNoFrameskip-v4",
+ "KrullNoFrameskip-v4",
+ "KungFuMasterNoFrameskip-v4",
+ "MontezumaRevengeNoFrameskip-v4",
+ "MsPacmanNoFrameskip-v4",
+ "NameThisGameNoFrameskip-v4",
+ "PhoenixNoFrameskip-v4",
+ "PitfallNoFrameskip-v4",
+ "PongNoFrameskip-v4",
+ "PooyanNoFrameskip-v4",
+ "PrivateEyeNoFrameskip-v4",
+ "QbertNoFrameskip-v4",
+ "RiverraidNoFrameskip-v4",
+ "RoadRunnerNoFrameskip-v4",
+ "RobotankNoFrameskip-v4",
+ "SeaquestNoFrameskip-v4",
+ "SkiingNoFrameskip-v4",
+ "SolarisNoFrameskip-v4",
+ "SpaceInvadersNoFrameskip-v4",
+ "StarGunnerNoFrameskip-v4",
+ "TennisNoFrameskip-v4",
+ "TimePilotNoFrameskip-v4",
+ "TutankhamNoFrameskip-v4",
+ "UpNDownNoFrameskip-v4",
+ "VentureNoFrameskip-v4",
+ "VideoPinballNoFrameskip-v4",
+ "WizardOfWorNoFrameskip-v4",
+ "YarsRevengeNoFrameskip-v4",
+ "ZaxxonNoFrameskip-v4",
  ],
  "Box2D": [
  "BipedalWalker-v3",
@@ -120,18 +120,16 @@ ALL_ENV_IDS = {
 
 
 def get_leaderboard_df():
- # List all results files in results repo
- pattern = re.compile(r"^[^/]*/[^/]*/[^/]*results_[a-f0-9]+\.json$")
- filenames = API.list_repo_files(RESULTS_REPO, repo_type="dataset")
- filenames = [filename for filename in filenames if pattern.match(filename)]
+ dir_path = API.snapshot_download(repo_id=RESULTS_REPO, repo_type="dataset")
+ pattern = os.path.join(dir_path, "**", "results_*.json")
+ filenames = glob.glob(pattern, recursive=True)
 
  data = []
  for filename in filenames:
- path = API.hf_hub_download(repo_id=RESULTS_REPO, filename=filename, repo_type="dataset")
- with open(path) as fp:
+ with open(filename) as fp:
  report = json.load(fp)
  user_id, model_id = report["config"]["model_id"].split("/")
- row = {"user_id": user_id, "model_id": model_id}
+ row = {"user_id": user_id, "model_id": model_id, "model_sha": report["config"]["model_sha"]}
  if report["status"] == "DONE" and len(report["results"]) > 0:
  env_ids = list(report["results"].keys())
  assert len(env_ids) == 1, "Only one environment supported for the moment"
@@ -165,6 +163,29 @@ def format_df(df: pd.DataFrame):
  return df.values.tolist()
 
 
+def refresh_dataframes():
+ df = get_leaderboard_df()
+ all_dfs = [format_df(select_env(df, env_id)) for env_id in all_env_ids]
+ return all_dfs
+
+
+def refresh_videos():
+ df = get_leaderboard_df()
+ outputs = []
+ for env_id in all_env_ids:
+ env_df = select_env(df, env_id)
+ if not env_df.empty:
+ user_id = env_df.iloc[0]["user_id"]
+ model_id = env_df.iloc[0]["model_id"]
+ model_sha = env_df.iloc[0]["model_sha"]
+ repo_id = f"{user_id}/{model_id}"
+ video_path = API.hf_hub_download(repo_id=repo_id, filename="replay.mp4", revision=model_sha, repo_type="model")
+ outputs.append(video_path)
+ else:
+ outputs.append(None)
+ return outputs
+
+
 HEADING = """
 # 🥇 Open RL Leaderboard 🥇
 
@@ -243,54 +264,53 @@ If you encounter any issue, please [open an issue](https://huggingface.co/spaces
 ```
 """
 
-
 with gr.Blocks() as demo:
  gr.Markdown(HEADING)
  with gr.Tabs(elem_classes="tab-buttons") as tabs:
  with gr.TabItem("🏅 Leaderboard"):
  df = get_leaderboard_df()
+ all_env_ids = []
+ all_gr_dfs = []
+ all_gr_videos = []
  for env_domain, env_ids in ALL_ENV_IDS.items():
  with gr.TabItem(env_domain):
  for env_id in env_ids:
- with gr.TabItem(env_id):
+ # If the env_id envs with "NoFrameskip-v4", we remove it
+ tab_env_id = env_id[: -len("NoFrameskip-v4")] if env_id.endswith("NoFrameskip-v4") else env_id
+ with gr.TabItem(tab_env_id):
+ logger.info(f"Creating tab for {env_id}")
  with gr.Row(equal_height=False):
- if env_domain == "Atari":
- env_id = f"{env_id}NoFrameskip-v4"
- env_df = select_env(df, env_id)
- gr.components.Dataframe(
- value=format_df(env_df),
+ # Display the leaderboard
+ gr_df = gr.components.Dataframe(
  headers=["🏆", "🧑 User", "🤖 Model id", "📊 Mean episodic return"],
  datatype=["number", "markdown", "markdown", "number"],
  row_count=(10, "fixed"),
  scale=3,
  )
- # Get the best model and
- if not env_df.empty:
- user_id = env_df.iloc[0]["user_id"]
- model_id = env_df.iloc[0]["model_id"]
- video_path = API.hf_hub_download(
- repo_id=f"{user_id}/{model_id}",
- filename="replay.mp4",
- revision="main",
- repo_type="model",
- )
- video = gr.PlayableVideo(
- video_path,
- label=model_id,
- scale=1,
- min_width=50,
- autoplay=True,
- show_download_button=False,
- show_share_button=False,
- )
- # Doesn't loop for the moment, see https://github.com/gradio-app/gradio/issues/7689
+
+ # Play the video of the best model
+ gr_video = gr.PlayableVideo( # Doesn't loop for the moment, see https://github.com/gradio-app/gradio/issues/7689
+ scale=1,
+ min_width=50,
+ autoplay=True,
+ show_download_button=False,
+ show_share_button=False,
+ show_label=False,
+ )
+
+ all_env_ids.append(env_id)
+ all_gr_dfs.append(gr_df)
+ all_gr_videos.append(gr_video)
 
  with gr.TabItem("📝 About"):
  gr.Markdown(ABOUT_TEXT)
 
+ demo.load(refresh_dataframes, outputs=all_gr_dfs, every=REFRESH_RATE)
+ demo.load(refresh_videos, outputs=all_gr_videos, every=REFRESH_RATE)
+
 
 scheduler = BackgroundScheduler()
-scheduler.add_job(func=backend_routine, trigger="interval", seconds=10 * 60, max_instances=1)
+scheduler.add_job(func=backend_routine, trigger="interval", seconds=REFRESH_RATE, max_instances=1)
 scheduler.start()