File size: 4,477 Bytes
c02fbf1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 |
from __future__ import annotations
import numpy as np
import pandas as pd
import requests
from huggingface_hub.hf_api import SpaceInfo
url = 'https://docs.google.com/spreadsheets/d/1RoM2DgzaYJg6Ias1YNC2kQN01xSWJb1KEER9efb0X7A/edit#gid=0'
csv_url = url.replace('/edit#gid=', '/export?format=csv&gid=')
class DatasetList:
def __init__(self):
self.table = pd.read_csv(csv_url)
self._preprocess_table()
self.table_header = '''
<tr>
<td width="15%">Dataset Name</td>
<td width="10%">Question Type</td>
<td width="10%">Applied In Paper</td>
<td width="10%">Reference Paper</td>
<td width="20%">Brief Description</td>
<td width="5%">Count</td>
<td width="10%">Original Access Link</td>
<td width="10%">Publicly Available?</td>
<td width="10%">Access link on 🤗</td>
</tr>'''
def _preprocess_table(self) -> None:
self.table['dataset_name_lowercase'] = self.table.dataset_name.str.lower()
self.table['count'] = self.table['count'].apply(str)
rows = []
for row in self.table.itertuples():
dataset_name = f'{row.dataset_name}' if isinstance(row.dataset_name, str) else ''
question_type = f'{row.question_type}' if isinstance(row.question_type, str) else ''
used_in_paper = f'{row.used_in_paper}' if isinstance(row.used_in_paper, str) else ''
reference_paper = f'<a href="{row.reference_paper}" target="_blank">Paper</a>' if isinstance(row.reference_paper, str) else ''
brief_description = f'{row.brief_description}' if isinstance(row.brief_description, str) else ''
count = f'{row.count}' if isinstance(row.count, str) else ''
original_link = f'<a href="{row.original_link}" target="_blank">Access Link</a>' if isinstance(row.original_link, str) else ''
publicly_available = f'<a href="{row.publicly_available}" target="_blank">License</a>' if isinstance(row.publicly_available, str) else ''
huggingface_link = f'<a href="{row.huggingface_link}" target="_blank">HF Link</a>' if isinstance(row.huggingface_link, str) else ''
row = f'''
<tr>
<td>{dataset_name}</td>
<td>{question_type}</td>
<td>{used_in_paper}</td>
<td>{reference_paper}</td>
<td>{brief_description}</td>
<td>{count}</td>
<td>{original_link}</td>
<td>{publicly_available}</td>
<td>{huggingface_link}</td>
</tr>'''
rows.append(row)
self.table['html_table_content'] = rows
def render(self, search_query: str,
case_sensitive: bool,
filter_names: list[str]
) -> tuple[int, str]:
df = self.table
if search_query:
if case_sensitive:
df = df[df.dataset_name.str.contains(search_query)]
else:
df = df[df.dataset_name_lowercase.str.contains(search_query.lower())]
has_dataset = 'Dataset' in filter_names
has_datalink = 'Data Link' in filter_names
has_paper = 'Paper' in filter_names
df = self.filter_table(df, has_dataset, has_datalink, has_paper)
#df = self.filter_table(df, has_paper, has_github, has_model, data_types, model_types)
return len(df), self.to_html(df, self.table_header)
@staticmethod
def filter_table(df: pd.DataFrame,
has_dataset: bool,
has_datalink: bool,
has_paper: bool
) -> pd.DataFrame:
if has_dataset:
df = df[~df.dataset_name.isna()]
if has_datalink:
df = df[~df.huggingface_link.isna() | ~df.original_link.isna()]
if has_paper:
df = df[~df.reference_paper.isna()]
# df = df[df.data_type.isin(set(data_types))]
#df = df[df.base_model.isin(set(model_types))]
# df = df[df.year.isin(set(years))]
return df
@staticmethod
def to_html(df: pd.DataFrame, table_header: str) -> str:
table_data = ''.join(df.html_table_content)
html = f'''
<table>
{table_header}
{table_data}
</table>'''
return html
|