Elron commited on
Commit
334775e
1 Parent(s): a2d02fa

Upload catalog.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. catalog.py +31 -10
catalog.py CHANGED
@@ -1,6 +1,8 @@
 
1
  import os
2
  import re
3
  from pathlib import Path
 
4
 
5
  import requests
6
 
@@ -34,16 +36,19 @@ class LocalCatalog(Catalog):
34
  location: str = default_catalog_path
35
 
36
  def path(self, artifact_identifier: str):
37
- assert artifact_identifier.strip(), "artifact_identifier should not be an empty string."
 
 
38
  parts = artifact_identifier.split(COLLECTION_SEPARATOR)
39
  parts[-1] = parts[-1] + ".json"
40
  return os.path.join(self.location, *parts)
41
 
42
  def load(self, artifact_identifier: str):
43
- assert artifact_identifier in self, "Artifact with name {} does not exist".format(artifact_identifier)
 
 
44
  path = self.path(artifact_identifier)
45
- artifact_instance = Artifact.load(path)
46
- return artifact_instance
47
 
48
  def __getitem__(self, name) -> Artifact:
49
  return self.load(name)
@@ -56,8 +61,16 @@ class LocalCatalog(Catalog):
56
  return False
57
  return os.path.exists(path) and os.path.isfile(path)
58
 
59
- def save_artifact(self, artifact: Artifact, artifact_identifier: str, overwrite: bool = False):
60
- assert isinstance(artifact, Artifact), f"Input artifact must be an instance of Artifact, got {type(artifact)}"
 
 
 
 
 
 
 
 
61
  if not overwrite:
62
  assert (
63
  artifact_identifier not in self
@@ -65,7 +78,8 @@ class LocalCatalog(Catalog):
65
  path = self.path(artifact_identifier)
66
  os.makedirs(Path(path).parent.absolute(), exist_ok=True)
67
  artifact.save(path)
68
- print(f"Artifact {artifact_identifier} saved to {path}")
 
69
 
70
 
71
  class EnvironmentLocalCatalog(LocalCatalog):
@@ -97,16 +111,23 @@ class GithubCatalog(LocalCatalog):
97
  def verify_legal_catalog_name(name):
98
  assert re.match(
99
  r"^[\w" + COLLECTION_SEPARATOR + "]+$", name
100
- ), 'Catalog name should be alphanumeric, ":" should specify dirs (instead of "/").'
101
 
102
 
103
  def add_to_catalog(
104
- artifact: Artifact, name: str, catalog: Catalog = None, overwrite: bool = False, catalog_path: str = None
 
 
 
 
 
105
  ):
106
  if catalog is None:
107
  if catalog_path is None:
108
  catalog_path = default_catalog_path
109
  catalog = LocalCatalog(location=catalog_path)
110
  verify_legal_catalog_name(name)
111
- catalog.save_artifact(artifact, name, overwrite=overwrite) # remove collection (its actually the dir).
 
 
112
  # verify name
 
1
+ import logging
2
  import os
3
  import re
4
  from pathlib import Path
5
+ from typing import Optional
6
 
7
  import requests
8
 
 
36
  location: str = default_catalog_path
37
 
38
  def path(self, artifact_identifier: str):
39
+ assert (
40
+ artifact_identifier.strip()
41
+ ), "artifact_identifier should not be an empty string."
42
  parts = artifact_identifier.split(COLLECTION_SEPARATOR)
43
  parts[-1] = parts[-1] + ".json"
44
  return os.path.join(self.location, *parts)
45
 
46
  def load(self, artifact_identifier: str):
47
+ assert (
48
+ artifact_identifier in self
49
+ ), f"Artifact with name {artifact_identifier} does not exist"
50
  path = self.path(artifact_identifier)
51
+ return Artifact.load(path)
 
52
 
53
  def __getitem__(self, name) -> Artifact:
54
  return self.load(name)
 
61
  return False
62
  return os.path.exists(path) and os.path.isfile(path)
63
 
64
+ def save_artifact(
65
+ self,
66
+ artifact: Artifact,
67
+ artifact_identifier: str,
68
+ overwrite: bool = False,
69
+ verbose: bool = True,
70
+ ):
71
+ assert isinstance(
72
+ artifact, Artifact
73
+ ), f"Input artifact must be an instance of Artifact, got {type(artifact)}"
74
  if not overwrite:
75
  assert (
76
  artifact_identifier not in self
 
78
  path = self.path(artifact_identifier)
79
  os.makedirs(Path(path).parent.absolute(), exist_ok=True)
80
  artifact.save(path)
81
+ if verbose:
82
+ logging.info(f"Artifact {artifact_identifier} saved to {path}")
83
 
84
 
85
  class EnvironmentLocalCatalog(LocalCatalog):
 
111
  def verify_legal_catalog_name(name):
112
  assert re.match(
113
  r"^[\w" + COLLECTION_SEPARATOR + "]+$", name
114
+ ), f'Artifict name ("{name}") should be alphanumeric. Use "." for nesting (e.g. myfolder.my_artifact)'
115
 
116
 
117
  def add_to_catalog(
118
+ artifact: Artifact,
119
+ name: str,
120
+ catalog: Catalog = None,
121
+ overwrite: bool = False,
122
+ catalog_path: Optional[str] = None,
123
+ verbose=True,
124
  ):
125
  if catalog is None:
126
  if catalog_path is None:
127
  catalog_path = default_catalog_path
128
  catalog = LocalCatalog(location=catalog_path)
129
  verify_legal_catalog_name(name)
130
+ catalog.save_artifact(
131
+ artifact, name, overwrite=overwrite, verbose=verbose
132
+ ) # remove collection (its actually the dir).
133
  # verify name