Skip to content

Commit 6917c5d

Browse files
committed
refactor linkUtil with h5json
1 parent c66d632 commit 6917c5d

5 files changed

Lines changed: 92 additions & 189 deletions

File tree

hsds/link_dn.py

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,9 @@
2222

2323
from h5json.objid import isValidUuid
2424
from h5json.time_util import getNow
25+
from h5json.link_util import validateLinkName, getLinkClass, isEqualLink
2526

2627
from .util.globparser import globmatch
27-
from .util.linkUtil import validateLinkName, getLinkClass, isEqualLink
2828
from .util.domainUtil import isValidBucketName
2929
from .datanode_lib import get_obj_id, get_metadata_obj, save_metadata_obj
3030
from . import config
@@ -156,6 +156,10 @@ async def GET_Links(request):
156156
link = copy(link_dict[title])
157157
log.debug(f"link list[{i}: {link}")
158158
link["title"] = title
159+
if link.get("h5domain"):
160+
# deprecated key, replace with file
161+
link["file"] = link["h5domain"]
162+
del link["h5domain"]
159163
link_list.append(link)
160164

161165
resp_json = {"links": link_list}
@@ -218,6 +222,7 @@ async def POST_Links(request):
218222
log.info(f"Link name {title} not found in group: {group_id}")
219223
continue
220224
link_json = links[title]
225+
log.debug(f"POST Links got link_json: {link_json}")
221226
item = {}
222227
if "class" not in link_json:
223228
log.warn(f"expected to find class key for link: {title}")
@@ -245,15 +250,19 @@ async def POST_Links(request):
245250
log.warn(f"expected to find h5path for external link: {title}")
246251
continue
247252
item["h5path"] = link_json["h5path"]
248-
if "h5domain" not in link_json:
249-
log.warn(f"expted to find h5domain for external link: {title}")
253+
if "h5domain" in link_json:
254+
item["file"] = link_json["h5domain"]
255+
elif "file" in link_json:
256+
item["file"] = link_json["file"]
257+
else:
258+
log.warn(f"expected to find h5domain or file for external link: {title}")
250259
continue
251-
item["h5domain"] = link_json["h5domain"]
252260
else:
253261
log.warn(f"unexpected to link class {link_class} for link: {title}")
254262
continue
255263

256264
item["title"] = title
265+
log.debug(f"adding link item: {item}")
257266

258267
link_list.append(item)
259268

hsds/link_sn.py

Lines changed: 35 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,23 @@
1717
from json import JSONDecodeError
1818

1919
from h5json.objid import isValidUuid, getCollectionForId
20+
from h5json.link_util import validateLinkName, getLinkClass, getLinkId
21+
from h5json.link_util import getLinkPath, getLinkFilePath
2022

2123
from .util.nodeUtil import getDataNodeUrl
2224
from .util.httpUtil import getHref, getBooleanParam
2325
from .util.httpUtil import jsonResponse
2426
from .util.globparser import globmatch
2527
from .util.authUtil import getUserPasswordFromRequest, validateUserPassword
26-
from .util.domainUtil import getDomainFromRequest, isValidDomain, verifyRoot
27-
from .util.domainUtil import getBucketForDomain
28-
from .util.linkUtil import validateLinkName, getLinkClass
28+
from .util.domainUtil import getDomainFromRequest, isValidDomain, verifyRoot, getBucketForDomain
29+
from .util.linkUtil import getRequestLink
30+
31+
2932
from .servicenode_lib import getDomainJson, validateAction
3033
from .servicenode_lib import getLink, putLink, putLinks, getLinks, deleteLinks
3134
from .domain_crawl import DomainCrawler
3235
from . import hsds_logger as log
36+
from . import config
3337

3438

3539
async def GET_Links(request):
@@ -221,13 +225,13 @@ async def GET_Link(request):
221225
link_class = link_json["class"]
222226
resp_link["class"] = link_class
223227
if link_class == "H5L_TYPE_HARD":
224-
resp_link["id"] = link_json["id"]
228+
resp_link["id"] = getLinkId(link_json)
225229
resp_link["collection"] = getCollectionForId(link_json["id"])
226230
elif link_class == "H5L_TYPE_SOFT":
227-
resp_link["h5path"] = link_json["h5path"]
231+
resp_link["h5path"] = getLinkPath(link_json)
228232
elif link_class == "H5L_TYPE_EXTERNAL":
229-
resp_link["h5path"] = link_json["h5path"]
230-
resp_link["h5domain"] = link_json["h5domain"]
233+
resp_link["h5path"] = getLinkPath(link_json)
234+
resp_link["file"] = getLinkFilePath(link_json)
231235
else:
232236
log.warn(f"Unexpected link class: {link_class}")
233237
resp_json = {}
@@ -291,17 +295,32 @@ async def PUT_Link(request):
291295
msg = f"Invalid domain: {domain}"
292296
log.warn(msg)
293297
raise HTTPBadRequest(reason=msg)
294-
bucket = getBucketForDomain(domain)
295298

296299
await validateAction(app, domain, group_id, username, "create")
297-
# putLink will validate these arguments
298-
kwargs = {"bucket": bucket}
299-
kwargs["tgt_id"] = body.get("id")
300-
kwargs["h5path"] = body.get("h5path")
301-
kwargs["h5domain"] = body.get("h5domain")
302-
created = body.get("created")
303-
if created:
304-
kwargs["created"] = created
300+
301+
predate_max_time = config.get("predate_max_time", default=10.0)
302+
303+
try:
304+
link_json = getRequestLink(link_title, body, predate_max_time=predate_max_time)
305+
except (KeyError, TypeError, ValueError) as e:
306+
raise HTTPBadRequest(reason=str(e))
307+
308+
link_class = getLinkClass(link_json)
309+
310+
kwargs = {}
311+
kwargs["bucket"] = getBucketForDomain(domain)
312+
if link_class == "H5L_TYPE_HARD":
313+
kwargs["tgt_id"] = getLinkId(link_json)
314+
elif link_class == "H5L_TYPE_SOFT":
315+
kwargs["h5path"] = getLinkPath(link_json)
316+
elif link_class == "H5L_TYPE_EXTERNAL":
317+
kwargs["h5path"] = getLinkPath(link_json)
318+
kwargs["h5domain"] = getLinkFilePath(link_json)
319+
else:
320+
raise HTTPBadRequest(reason=f"unexpected link class: {link_class}")
321+
322+
if "created" in link_json:
323+
kwargs["created"] = link_json["created"]
305324

306325
status = await putLink(app, group_id, link_title, **kwargs)
307326

hsds/servicenode_lib.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,12 @@
3131
from h5json.shape_util import getShapeDims, getShapeClass
3232
from h5json.dset_util import getChunkSize, generateLayout
3333
from h5json.dset_util import getDataSize, validateDatasetCreationProps
34+
from h5json.link_util import h5Join, validateLinkName, getLinkClass, getLinkFilePath
3435
from h5json.time_util import getNow
3536

3637
from .util.nodeUtil import getDataNodeUrl
3738
from .util.authUtil import getAclKeys
38-
from .util.linkUtil import h5Join, validateLinkName, getLinkClass, getRequestLinks
39+
from .util.linkUtil import getRequestLinks
3940
from .util.storUtil import getStorJSONObj, isStorObj, getSupportedFilters
4041
from .util.authUtil import aclCheck
4142
from .util.httpUtil import http_get, http_put, http_post, http_delete
@@ -479,7 +480,7 @@ async def putLink(app, group_id, title,
479480
if h5path:
480481
link_json["h5path"] = h5path
481482
if h5domain:
482-
link_json["h5domain"] = h5domain
483+
link_json["file"] = h5domain
483484
if created:
484485
link_json["created"] = created
485486

@@ -659,7 +660,7 @@ async def getObjectIdByPath(app, obj_id, h5path, bucket=None, refresh=False, dom
659660
raise HTTPBadRequest(reason=msg)
660661

661662
# find domain object is stored under
662-
domain = link_json["h5domain"]
663+
domain = getLinkFilePath(link_json)
663664

664665
if domain.startswith("hdf5:/"):
665666
# strip off prefix

hsds/util/linkUtil.py

Lines changed: 5 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -13,129 +13,12 @@
1313
# linkdUtil:
1414
# link related functions
1515
#
16-
import time
16+
from h5json.time_util import getNow
17+
from h5json.link_util import validateLinkName, getLinkClass, getLinkPath, getLinkFilePath
1718

1819
from .. import hsds_logger as log
1920

2021

21-
def validateLinkName(name):
22-
""" verify the link name is valid """
23-
if not isinstance(name, str):
24-
msg = "Unexpected type for link name"
25-
log.warn(msg)
26-
raise ValueError(msg)
27-
if name.find("/") >= 0:
28-
msg = "link name contains slash"
29-
log.warn(msg)
30-
raise ValueError(msg)
31-
32-
33-
def getLinkClass(link_json):
34-
""" verify this is a valid link
35-
returns the link class """
36-
log.debug(f"getLinkClass({link_json})")
37-
if "class" in link_json:
38-
link_class = link_json["class"]
39-
else:
40-
link_class = None
41-
if "h5path" in link_json and "id" in link_json:
42-
msg = "link tgt_id and h5path both set"
43-
log.warn(msg)
44-
raise ValueError(msg)
45-
if "id" in link_json:
46-
tgt_id = link_json["id"]
47-
if not isinstance(tgt_id, str) or len(tgt_id) < 38:
48-
msg = f"link with invalid id: {tgt_id}"
49-
log.warn(msg)
50-
raise ValueError(msg)
51-
if tgt_id[:2] not in ("g-", "t-", "d-"):
52-
msg = "link tgt must be group, datatype or dataset uuid"
53-
log.warn(msg)
54-
raise ValueError(msg)
55-
if link_class:
56-
if link_class != "H5L_TYPE_HARD":
57-
msg = f"expected link class to be H5L_TYPE_HARD but got: {link_class}"
58-
log.warn(msg)
59-
raise ValueError(msg)
60-
else:
61-
link_class = "H5L_TYPE_HARD"
62-
elif "h5path" in link_json:
63-
h5path = link_json["h5path"]
64-
log.debug(f"link path: {h5path}")
65-
if "h5domain" in link_json:
66-
if link_class:
67-
if link_class != "H5L_TYPE_EXTERNAL":
68-
msg = f"expected link class to be H5L_TYPE_EXTERNAL but got: {link_class}"
69-
log.warn(msg)
70-
raise ValueError(msg)
71-
else:
72-
link_class = "H5L_TYPE_EXTERNAL"
73-
else:
74-
if link_class:
75-
if link_class != "H5L_TYPE_SOFT":
76-
msg = f"expected link class to be H5L_TYPE_SOFT but got: {link_class}"
77-
log.warn(msg)
78-
raise ValueError(msg)
79-
else:
80-
link_class = "H5L_TYPE_SOFT"
81-
else:
82-
msg = "link with no id or h5path"
83-
log.warn(msg)
84-
raise ValueError(msg)
85-
86-
return link_class
87-
88-
89-
def isEqualLink(link1, link2):
90-
""" Return True if the two links are the same """
91-
92-
for obj in (link1, link2):
93-
if not isinstance(obj, dict):
94-
raise TypeError(f"unexpected type: {type(obj)}")
95-
if "class" not in obj:
96-
raise TypeError("expected class key for link")
97-
if link1["class"] != link2["class"]:
98-
return False # different link types
99-
link_class = link1["class"]
100-
if link_class == "H5L_TYPE_HARD":
101-
for obj in (link1, link2):
102-
if "id" not in obj:
103-
raise TypeError(f"expected id key for link: {obj}")
104-
if link1["id"] != link2["id"]:
105-
return False
106-
elif link_class == "H5L_TYPE_SOFT":
107-
for obj in (link1, link2):
108-
if "h5path" not in obj:
109-
raise TypeError(f"expected h5path key for link: {obj}")
110-
if link1["h5path"] != link2["h5path"]:
111-
return False
112-
elif link_class == "H5L_TYPE_EXTERNAL":
113-
for obj in (link1, link2):
114-
for k in ("h5path", "h5domain"):
115-
if k not in obj:
116-
raise TypeError(f"expected {k} key for link: {obj}")
117-
if link1["h5path"] != link2["h5path"]:
118-
return False
119-
if link1["h5domain"] != link2["h5domain"]:
120-
return False
121-
else:
122-
raise TypeError(f"unexpected link class: {link_class}")
123-
return True
124-
125-
126-
def h5Join(path, paths):
127-
h5path = path
128-
if not paths:
129-
return h5path
130-
if isinstance(paths, str):
131-
paths = (paths,)
132-
for s in paths:
133-
if h5path[-1] != "/":
134-
h5path += "/"
135-
h5path += s
136-
return h5path
137-
138-
13922
def getRequestLink(title, link_json, predate_max_time=0.0):
14023
""" return normalized link from request json
14124
Throw value error if badly formatted """
@@ -148,16 +31,11 @@ def getRequestLink(title, link_json, predate_max_time=0.0):
14831
log.debug(f"getRequestLink title: {title} link_json: {link_json}")
14932
link_item = {} # normalized link item to return
15033

151-
now = time.time()
34+
now = getNow()
15235

15336
validateLinkName(title) # will raise ValueError is invalid
15437

15538
link_class = getLinkClass(link_json)
156-
if "class" in link_item:
157-
if link_class != link_json["class"]:
158-
msg = f"expected link class of: {link_class} but got {link_json}"
159-
log.warn(msg)
160-
raise ValueError(msg)
16139

16240
link_item = {"class": link_class}
16341

@@ -169,17 +47,10 @@ def getRequestLink(title, link_json, predate_max_time=0.0):
16947
link_item["id"] = link_json["id"]
17048
else:
17149
if link_class in ("H5L_TYPE_SOFT", "H5L_TYPE_EXTERNAL"):
172-
if "h5path" not in link_json:
173-
msg = "expected h5path key for soft link"
174-
log.warn(msg)
175-
raise ValueError(msg)
176-
link_item["h5path"] = link_json["h5path"]
50+
link_item["h5path"] = getLinkPath(link_json)
17751

17852
if link_class == "H5L_TYPE_EXTERNAL":
179-
if "h5domain" not in link_json:
180-
msg = "expected h5domain key for external link"
181-
log.warn(msg)
182-
raise ValueError(msg)
53+
link_item["file"] = getLinkFilePath(link_json)
18354

18455
if "created" in link_json:
18556
created = link_json["created"]

0 commit comments

Comments
 (0)