metric / hf_utils.py
Elron's picture
Upload hf_utils.py with huggingface_hub
de6938d
raw
history blame
1.41 kB
from pathlib import Path
from datasets import disable_caching, enable_caching, is_caching_enabled
from datasets.utils.py_utils import get_imports
from .file_utils import get_all_files_in_dir
HF_CACHING_ENABLED = False
class HFCachingContextManager:
def __init__(self, should_cache):
self.should_cache = should_cache
self.was_caching_enabled = None
def __enter__(self):
self.was_caching_enabled = is_caching_enabled()
if self.should_cache:
if not self.was_caching_enabled:
enable_caching()
else:
if self.was_caching_enabled:
disable_caching()
def __exit__(self, exc_type, exc_val, exc_tb):
if self.was_caching_enabled != is_caching_enabled():
if self.was_caching_enabled:
enable_caching()
else:
disable_caching()
def get_missing_imports(file, exclude=None):
if exclude is None:
exclude = []
src_dir = Path(__file__).parent
python_files = get_all_files_in_dir(src_dir, file_extension=".py")
# get only the file without the path and extension
required_modules = [Path(p).stem for p in python_files]
imports = get_imports(file)
imported_modules = [i[1] for i in imports if i[0] == "internal"]
return [
i for i in required_modules if i not in imported_modules and i not in exclude
]