multiScanByPrefBack cleanup

Author: Cisphyx

synapse/datamodel.py

@@ -669,10 +669,6 @@ def __init__(self, core=None):
                     'computed': True,
                     'doc': 'The form of node which is referenced.'}),
 
-                ('ndef', ('ndef', {}), {
-                    'computed': True,
-                    'doc': 'The (form, valu) of the node which is referenced.'}),
-
                 ('value', ('data', {}), {
                     'computed': True,
                     'doc': 'The primary property value of the node which is referenced.'}),

synapse/lib/lmdbslab.py

@@ -1673,36 +1673,39 @@ async def multiScanByPrefBack(self, pref, multilen, byts, db=None):
         size = preflen + len(byts)
         maxv = int.from_bytes(b'\xff' * multilen, 'big')
 
+        intpref = int.from_bytes(pref, 'big')
+        maxpref = int.from_bytes(b'\xff' * len(pref), 'big')
+
+        intbyts = int.from_bytes(byts, 'big')
+        maxbyts = int.from_bytes(b'\xff' * len(byts), 'big')
+
         async def scangenr(pval):
             with ScanBack(self, db) as scan:
                 skey = pval.to_bytes(multilen, 'big')
 
                 while True:
-                    intoff = int.from_bytes(byts, "big")
-                    intoff += 1
-                    try:
-                        nextbyts = intoff.to_bytes(len(byts), "big")
+                    if (intbyts + 1) <= maxbyts:
+                        nextbyts = (intbyts + 1).to_bytes(len(byts), "big")
                         nextpref = pref + skey + nextbyts
-                        if not scan.set_range(nextpref):
-                            return
-
-                        if scan.atitem[0] == nextpref:
-                            if not scan.next_key():
-                                return
-
-                    except OverflowError:
-                        if (pval := int.from_bytes(skey, 'big') + 1) > maxv:
-                            if not scan.first():
-                                return
 
+                    elif (pval := int.from_bytes(skey, 'big') + 1) <= maxv:
+                        # Prefix can't be incremented, try to increment multi key instead
                         skey = pval.to_bytes(multilen, 'big')
                         nextpref = pref + skey
-                        if not scan.set_range(nextpref) and not scan.first():
-                            return
 
-                        if scan.atitem[0] == nextpref:
-                            if not scan.next_key():
-                                return
+                    elif (intpref + 1) <= maxpref:
+                        # Multi key can't be incremented, try to increment prefix
+                        nextpref = (intpref + 1).to_bytes(len(pref), "big")
+
+                    else:
+                        # No room to increment any keys, just go to the last item in the db
+                        nextpref = None
+
+                    if nextpref is not None and scan.set_range(nextpref):
+                        if scan.atitem[0] == nextpref and not scan.next_key():
+                            return
+                    elif not scan.first():
+                        return
 
                     fkey = scan.atitem[0]
                     if not fkey.startswith(pref):

synapse/lib/types.py

@@ -580,7 +580,7 @@ def postTypeInit(self):
         if (typeopts := self.opts.get('typeopts')) is None:
             typeopts = {}
 
-        # TODO can we polyprop with multiple type+typeopts defs????
+        # TODO allow polyprop with multiple type+typeopts defs?
         if not typeopts:
             if typename in self.modl.ifaces or ((forminfo := self.modl.forminfos.get(typename)) is not None and not forminfo.get('runt')):
                 typename = (typename,)
@@ -707,17 +707,6 @@ async def _normPyTuple(self, valu, view=None, newinfos=None):
             'virts': {vkey: dict(vval) for vkey, vval in virts.items()}
         }
 
-#        if virts:
-#            realvirts = collections.defaultdict(list)
-#            for norm in norms:
-#                if (virt := virts.get(norm)) is not None:
-#                    for vkey, vval in virt.items():
-#                        realvirts[vkey].append(vval)
-#
-#            norminfo['virts'] = dict(realvirts)
-#        else:
-#            norminfo['virts'] = {}
-
         return tuple(norms), norminfo
 
     def repr(self, valu):
@@ -2357,7 +2346,6 @@ async def getStorCmprs(self, cmpr, valu, virts=None):
                     return (('ndef=', valu.valu, s_layer.STOR_TYPE_POLY),)
                 valu = valu.valu[1]
 
-            # TODO better runtnode handling?
             elif isinstance(valu, s_node.RuntNode):
                 if self.formfilter(valu.form):
                     return (('=', valu.ndef[1], valu.form.type.stortype | s_layer.STOR_FLAG_POLY),)
@@ -2370,7 +2358,6 @@ async def getStorCmprs(self, cmpr, valu, virts=None):
 
         for ntyp in self.modl.getTypeSet(forms=self.forms, interfaces=self.ifaces):
             try:
-                # TODO: better way to keep these ordered?
                 for ncmpr in await ntyp.getStorCmprs(cmpr, valu, virts=virts):
                     cmprs[ncmpr] = True
                 isvalid = True

synapse/tests/test_lib_lmdbslab.py

@@ -122,6 +122,8 @@ async def test_lmdbslab_multiscan(self):
                 multilen = 8
 
                 self.eq((), await s_t_utils.alist(slab.multiScanByPref(pref, multilen, b'newp', db=testdb)))
+                self.eq((), await s_t_utils.alist(slab.multiScanByPrefBack(pref, multilen, b'\xff', db=testdb)))
+                self.eq((), await s_t_utils.alist(slab.multiScanByPrefBack(b'\xff', multilen, b'\xff', db=testdb)))
 
                 await slab.put(pref + s_common.int64en(0) + b'abar', b'haha', db=testdb)
                 await slab.put(pref + s_common.int64en(0) + b'afoo', b'haha', db=testdb)
@@ -147,6 +149,12 @@ async def test_lmdbslab_multiscan(self):
 
                 self.eq(exp[::-1], await s_t_utils.alist(slab.multiScanByPrefBack(pref, multilen, b'a', db=testdb)))
 
+                self.eq((), await s_t_utils.alist(slab.multiScanByPrefBack(pref, multilen, b'0', db=testdb)))
+                self.eq((), await s_t_utils.alist(slab.multiScanByPrefBack(pref, multilen, b'abaq', db=testdb)))
+                self.eq((), await s_t_utils.alist(slab.multiScanByPrefBack(pref, multilen, b'\xff', db=testdb)))
+                self.eq((), await s_t_utils.alist(slab.multiScanByPrefBack(pref, multilen, b'\xfe', db=testdb)))
+                self.eq((), await s_t_utils.alist(slab.multiScanByPrefBack(pref, multilen, b'\xfe', db=testdb)))
+
                 exp = (
                     (pref + s_common.int64en(0) + b'abar', b'haha'),
                     (pref + s_common.int64en(2) + b'afaz', b'haha'),