From 17a45f1e73169a3a30560e00c1552d48b6109362 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Thu, 19 Feb 2026 13:45:24 -0500 Subject: [PATCH 01/15] cleanup --- synapse/lib/base.py | 30 +- synapse/lib/cell.py | 598 ++++++++------------------------- synapse/lib/const.py | 7 +- synapse/lib/drive.py | 93 +++-- synapse/lib/spawner.py | 80 +++++ synapse/tests/test_lib_base.py | 54 ++- synapse/tests/test_lib_cell.py | 518 +++++----------------------- 7 files changed, 437 insertions(+), 943 deletions(-) create mode 100644 synapse/lib/spawner.py diff --git a/synapse/lib/base.py b/synapse/lib/base.py index 0dd73850ab9..2ca48abd6b0 100644 --- a/synapse/lib/base.py +++ b/synapse/lib/base.py @@ -41,7 +41,7 @@ def _fini_atexit(): # pragma: no cover if __debug__: logger.debug(f'At exit: Missing fini for {item}') for depth, call in enumerate(item.call_stack[:-2]): - logger.debug(f'{depth+1:3}: {call.strip()}') + logger.debug(f'{depth + 1:3}: {call.strip()}') continue try: @@ -347,16 +347,12 @@ async def dist(self, mesg): try: ret.append(await s_coro.ornot(func, mesg)) - except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only - raise except Exception: logger.exception('base %s error with mesg %s', self, mesg) for func in self._syn_links: try: ret.append(await s_coro.ornot(func, mesg)) - except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only - raise except Exception: logger.exception('base %s error with mesg %s', self, mesg) @@ -400,8 +396,6 @@ async def fini(self): for base in list(self.tofini): await base.fini() - # Do not continue to hold a reference to the last item we iterated on. - base = None # NOQA await self._kill_active_tasks() @@ -455,9 +449,25 @@ def onWith(self, evnt, func): finally: self.off(evnt, func) - def _wouldfini(self): - '''Check if a Base would be fini() if fini() was called on it.''' - return self._syn_refs == 1 + @contextlib.contextmanager + def onWithMulti(self, evnts, func): + ''' + A context manager which can be used to add a callbacks and remove them when + using a ``with`` statement. + + Args: + evnts (list): A list of event names + func (function): A callback function to receive event tufo + ''' + for evnt in evnts: + self.on(evnt, func) + # Allow exceptions to propagate during the context manager + # but ensure we cleanup our temporary callback + try: + yield self + finally: + for evnt in evnts: + self.off(evnt, func) async def waitfini(self, timeout=None): ''' diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index d7cda728924..22511d75bad 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -2,6 +2,7 @@ import os import ssl import copy +import stat import time import fcntl import shutil @@ -33,7 +34,6 @@ import synapse.lib.base as s_base import synapse.lib.boss as s_boss import synapse.lib.coro as s_coro -import synapse.lib.hive as s_hive import synapse.lib.link as s_link import synapse.lib.task as s_task import synapse.lib.cache as s_cache @@ -55,7 +55,6 @@ import synapse.lib.version as s_version import synapse.lib.lmdbslab as s_lmdbslab import synapse.lib.thisplat as s_thisplat -import synapse.lib.processpool as s_processpool import synapse.lib.crypto.passwd as s_passwd @@ -149,8 +148,7 @@ async def _doIterBackup(path, chunksize=1024): link0, file1 = await s_link.linkfile() def dowrite(fd): - # TODO: When we are 3.12+ convert this back to w|gz - see https://github.com/python/cpython/pull/2962 - with tarfile.open(output_filename, 'w:gz', fileobj=fd, compresslevel=1) as tar: + with tarfile.open(output_filename, 'w|gz', fileobj=fd, compresslevel=1) as tar: tar.add(path, arcname=os.path.basename(path)) fd.close() @@ -211,18 +209,18 @@ async def initCellApi(self): pass @adminapi(log=True) - async def shutdown(self, timeout=None): + async def shutdown(self, *, timeout=None): return await self.cell.shutdown(timeout=timeout) @adminapi(log=True) - async def freeze(self, timeout=30): + async def freeze(self, *, timeout=30): return await self.cell.freeze(timeout=timeout) @adminapi(log=True) async def resume(self): return await self.cell.resume() - async def allowed(self, perm, default=None): + async def allowed(self, perm, *, default=None): ''' Check if the user has the requested permission. @@ -346,7 +344,7 @@ async def rotateNexsLog(self): return await self.cell.rotateNexsLog() @adminapi(log=True) - async def trimNexsLog(self, consumers=None, timeout=60): + async def trimNexsLog(self, *, consumers=None, timeout=60): ''' Rotate and cull the Nexus log (and those of any consumers) at the current offset. @@ -366,7 +364,7 @@ async def trimNexsLog(self, consumers=None, timeout=60): return await self.cell.trimNexsLog(consumers=consumers, timeout=timeout) @adminapi() - async def waitNexsOffs(self, offs, timeout=None): + async def waitNexsOffs(self, offs, *, timeout=None): ''' Wait for the Nexus log to write an offset. @@ -380,11 +378,11 @@ async def waitNexsOffs(self, offs, timeout=None): return await self.cell.waitNexsOffs(offs, timeout=timeout) @adminapi(log=True) - async def promote(self, graceful=False): + async def promote(self, *, graceful=False): return await self.cell.promote(graceful=graceful) @adminapi(log=True) - async def handoff(self, turl, timeout=30): + async def handoff(self, turl, *, timeout=30): return await self.cell.handoff(turl, timeout=timeout) @adminapi(log=True) @@ -408,7 +406,7 @@ async def getSystemInfo(self): - volfree - Volume where cell is running free space - backupvolsize - Backup directory volume total space - backupvolfree - Backup directory volume free space - - celluptime - Cell uptime in milliseconds + - celluptime - Cell uptime in microseconds - cellrealdisk - Cell's use of disk, equivalent to du - cellapprdisk - Cell's apparent use of disk, equivalent to ls -l - osversion - OS version/architecture @@ -448,16 +446,16 @@ async def kill(self, iden): return await self.cell.kill(self.user, iden) @adminapi() - async def getTasks(self, peers=True, timeout=None): + async def getTasks(self, *, peers=True, timeout=None): async for task in self.cell.getTasks(peers=peers, timeout=timeout): yield task @adminapi() - async def getTask(self, iden, peers=True, timeout=None): + async def getTask(self, iden, *, peers=True, timeout=None): return await self.cell.getTask(iden, peers=peers, timeout=timeout) @adminapi() - async def killTask(self, iden, peers=True, timeout=None): + async def killTask(self, iden, *, peers=True, timeout=None): return await self.cell.killTask(iden, peers=peers, timeout=timeout) @adminapi(log=True) @@ -469,7 +467,7 @@ async def behold(self): yield mesg @adminapi(log=True) - async def addUser(self, name, passwd=None, email=None, iden=None): + async def addUser(self, name, *, passwd=None, email=None, iden=None): return await self.cell.addUser(name, passwd=passwd, email=email, iden=iden) @adminapi(log=True) @@ -477,14 +475,14 @@ async def delUser(self, iden): return await self.cell.delUser(iden) @adminapi(log=True) - async def addRole(self, name, iden=None): + async def addRole(self, name, *, iden=None): return await self.cell.addRole(name, iden=iden) @adminapi(log=True) async def delRole(self, iden): return await self.cell.delRole(iden) - async def addUserApiKey(self, name, duration=None, useriden=None): + async def addUserApiKey(self, name, *, duration=None, useriden=None): if useriden is None: useriden = self.user.iden @@ -495,7 +493,7 @@ async def addUserApiKey(self, name, duration=None, useriden=None): return await self.cell.addUserApiKey(useriden, name, duration=duration) - async def listUserApiKeys(self, useriden=None): + async def listUserApiKeys(self, *, useriden=None): if useriden is None: useriden = self.user.iden @@ -522,16 +520,16 @@ async def delUserApiKey(self, iden): return await self.cell.delUserApiKey(iden) @adminapi() - async def dyncall(self, iden, todo, gatekeys=()): + async def dyncall(self, iden, todo, *, gatekeys=()): return await self.cell.dyncall(iden, todo, gatekeys=gatekeys) @adminapi() - async def dyniter(self, iden, todo, gatekeys=()): + async def dyniter(self, iden, todo, *, gatekeys=()): async for item in self.cell.dyniter(iden, todo, gatekeys=gatekeys): yield item @adminapi() - async def issue(self, nexsiden: str, event: str, args, kwargs, meta=None, wait=True): + async def issue(self, nexsiden: str, event: str, args, kwargs, *, meta=None, wait=True): return await self.cell.nexsroot.issue(nexsiden, event, args, kwargs, meta, wait=wait) @adminapi(log=True) @@ -548,7 +546,7 @@ async def delAuthRole(self, name): await self.cell.auth.delRole(name) @adminapi() - async def getAuthUsers(self, archived=False): + async def getAuthUsers(self, *, archived=False): ''' Args: archived (bool): If true, list all users, else list non-archived users @@ -560,76 +558,33 @@ async def getAuthRoles(self): return await self.cell.getAuthRoles() @adminapi(log=True) - async def addUserRule(self, iden, rule, indx=None, gateiden=None): + async def addUserRule(self, iden, rule, *, indx=None, gateiden=None): return await self.cell.addUserRule(iden, rule, indx=indx, gateiden=gateiden) @adminapi(log=True) - async def setUserRules(self, iden, rules, gateiden=None): + async def setUserRules(self, iden, rules, *, gateiden=None): return await self.cell.setUserRules(iden, rules, gateiden=gateiden) @adminapi(log=True) - async def setRoleRules(self, iden, rules, gateiden=None): + async def setRoleRules(self, iden, rules, *, gateiden=None): return await self.cell.setRoleRules(iden, rules, gateiden=gateiden) @adminapi(log=True) - async def addRoleRule(self, iden, rule, indx=None, gateiden=None): + async def addRoleRule(self, iden, rule, *, indx=None, gateiden=None): return await self.cell.addRoleRule(iden, rule, indx=indx, gateiden=gateiden) @adminapi(log=True) - async def delUserRule(self, iden, rule, gateiden=None): + async def delUserRule(self, iden, rule, *, gateiden=None): return await self.cell.delUserRule(iden, rule, gateiden=gateiden) @adminapi(log=True) - async def delRoleRule(self, iden, rule, gateiden=None): + async def delRoleRule(self, iden, rule, *, gateiden=None): return await self.cell.delRoleRule(iden, rule, gateiden=gateiden) @adminapi(log=True) - async def setUserAdmin(self, iden, admin, gateiden=None): + async def setUserAdmin(self, iden, admin, *, gateiden=None): return await self.cell.setUserAdmin(iden, admin, gateiden=gateiden) - @adminapi() - async def getAuthInfo(self, name): - '''This API is deprecated.''' - s_common.deprecated('CellApi.getAuthInfo') - user = await self.cell.auth.getUserByName(name) - if user is not None: - info = user.pack() - info['roles'] = [self.cell.auth.role(r).name for r in info['roles']] - return info - - role = await self.cell.auth.getRoleByName(name) - if role is not None: - return role.pack() - - raise s_exc.NoSuchName(name=name) - - @adminapi(log=True) - async def addAuthRule(self, name, rule, indx=None, gateiden=None): - '''This API is deprecated.''' - s_common.deprecated('CellApi.addAuthRule') - item = await self.cell.auth.getUserByName(name) - if item is None: - item = await self.cell.auth.getRoleByName(name) - await item.addRule(rule, indx=indx, gateiden=gateiden) - - @adminapi(log=True) - async def delAuthRule(self, name, rule, gateiden=None): - '''This API is deprecated.''' - s_common.deprecated('CellApi.delAuthRule') - item = await self.cell.auth.getUserByName(name) - if item is None: - item = await self.cell.auth.getRoleByName(name) - await item.delRule(rule, gateiden=gateiden) - - @adminapi(log=True) - async def setAuthAdmin(self, name, isadmin): - '''This API is deprecated.''' - s_common.deprecated('CellApi.setAuthAdmin') - item = await self.cell.auth.getUserByName(name) - if item is None: - item = await self.cell.auth.getRoleByName(name) - await item.setAdmin(isadmin) - async def setUserPasswd(self, iden, passwd): await self.cell.auth.reqUser(iden) @@ -642,7 +597,7 @@ async def setUserPasswd(self, iden, passwd): return await self.cell.setUserPasswd(iden, passwd) @adminapi() - async def genUserOnepass(self, iden, duration=60000): + async def genUserOnepass(self, iden, *, duration=60000): return await self.cell.genUserOnepass(iden, duration) @adminapi(log=True) @@ -658,7 +613,7 @@ async def setUserEmail(self, useriden, email): return await self.cell.setUserEmail(useriden, email) @adminapi(log=True) - async def addUserRole(self, useriden, roleiden, indx=None): + async def addUserRole(self, useriden, roleiden, *, indx=None): return await self.cell.addUserRole(useriden, roleiden, indx=indx) @adminapi(log=True) @@ -688,7 +643,7 @@ async def getRoleInfo(self, name): raise s_exc.AuthDeny(mesg=mesg, user=self.user.iden, username=self.user.name) @adminapi() - async def getUserDef(self, iden, packroles=True): + async def getUserDef(self, iden, *, packroles=True): return await self.cell.getUserDef(iden, packroles=packroles) @adminapi() @@ -720,11 +675,11 @@ async def getRoleDefs(self): return await self.cell.getRoleDefs() @adminapi() - async def isUserAllowed(self, iden, perm, gateiden=None, default=False): + async def isUserAllowed(self, iden, perm, *, gateiden=None, default=False): return await self.cell.isUserAllowed(iden, perm, gateiden=gateiden, default=default) @adminapi() - async def isRoleAllowed(self, iden, perm, gateiden=None): + async def isRoleAllowed(self, iden, perm, *, gateiden=None): return await self.cell.isRoleAllowed(iden, perm, gateiden=gateiden) @adminapi() @@ -744,7 +699,7 @@ async def setUserProfInfo(self, iden, name, valu): return await self.cell.setUserProfInfo(iden, name, valu) @adminapi() - async def popUserProfInfo(self, iden, name, default=None): + async def popUserProfInfo(self, iden, name, *, default=None): return await self.cell.popUserProfInfo(iden, name, default=default) @adminapi() @@ -760,42 +715,12 @@ async def getDmonSessions(self): return await self.cell.getDmonSessions() @adminapi() - async def listHiveKey(self, path=None): - s_common.deprecated('CellApi.listHiveKey', curv='2.167.0') - return await self.cell.listHiveKey(path=path) - - @adminapi(log=True) - async def getHiveKeys(self, path): - s_common.deprecated('CellApi.getHiveKeys', curv='2.167.0') - return await self.cell.getHiveKeys(path) - - @adminapi(log=True) - async def getHiveKey(self, path): - s_common.deprecated('CellApi.getHiveKey', curv='2.167.0') - return await self.cell.getHiveKey(path) - - @adminapi(log=True) - async def setHiveKey(self, path, valu): - s_common.deprecated('CellApi.setHiveKey', curv='2.167.0') - return await self.cell.setHiveKey(path, valu) - - @adminapi(log=True) - async def popHiveKey(self, path): - s_common.deprecated('CellApi.popHiveKey', curv='2.167.0') - return await self.cell.popHiveKey(path) - - @adminapi(log=True) - async def saveHiveTree(self, path=()): - s_common.deprecated('CellApi.saveHiveTree', curv='2.167.0') - return await self.cell.saveHiveTree(path=path) - - @adminapi() - async def getNexusChanges(self, offs, tellready=False, wait=True): + async def getNexusChanges(self, offs, *, tellready=False, wait=True): async for item in self.cell.getNexusChanges(offs, tellready=tellready, wait=wait): yield item @adminapi() - async def runBackup(self, name=None, wait=True): + async def runBackup(self, *, name=None, wait=True): ''' Run a new backup. @@ -816,8 +741,8 @@ async def getBackupInfo(self): Returns: (dict) It has the following keys: - currduration - If backup currently running, time in ms since backup started, otherwise None - - laststart - Last time (in epoch milliseconds) a backup started - - lastend - Last time (in epoch milliseconds) a backup ended + - laststart - Last time (in epoch microseconds) a backup started + - lastend - Last time (in epoch microseconds) a backup ended - lastduration - How long last backup took in ms - lastsize - Disk usage of last backup completed - lastupload - Time a backup was last completed being uploaded via iter(New)BackupArchive @@ -864,7 +789,7 @@ async def iterBackupArchive(self, name): yield @adminapi() - async def iterNewBackupArchive(self, name=None, remove=False): + async def iterNewBackupArchive(self, *, name=None, remove=False): ''' Run a new backup and return it as a compressed stream of bytes. @@ -887,7 +812,7 @@ async def getDiagInfo(self): } @adminapi() - async def runGcCollect(self, generation=2): + async def runGcCollect(self, *, generation=2): ''' For diagnostic purposes only! @@ -912,7 +837,7 @@ async def getReloadableSystems(self): return self.cell.getReloadableSystems() @adminapi(log=True) - async def reload(self, subsystem=None): + async def reload(self, *, subsystem=None): return await self.cell.reload(subsystem=subsystem) class Cell(s_nexus.Pusher, s_telepath.Aware): @@ -986,13 +911,6 @@ class Cell(s_nexus.Pusher, s_telepath.Aware): 'description': 'Record all changes to a stream file on disk. Required for mirroring (on both sides).', 'type': 'boolean', }, - 'nexslog:async': { - 'default': True, - 'description': 'Deprecated. This option ignored.', - 'type': 'boolean', - 'hidedocs': True, - 'hidecmdl': True, - }, 'dmon:listen': { 'description': 'A config-driven way to specify the telepath bind URL.', 'type': ['string', 'null'], @@ -1056,7 +974,11 @@ class Cell(s_nexus.Pusher, s_telepath.Aware): 'aha:registry': { 'description': 'The telepath URL of the aha service registry.', 'type': ['string', 'array'], - 'items': {'type': 'string'}, + 'items': { + 'type': 'string', + 'pattern': '^ssl://.+$' + }, + 'pattern': '^ssl://.+$' }, 'aha:provision': { 'description': 'The telepath URL of the aha provisioning service.', @@ -1176,11 +1098,13 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): if conf is None: conf = {} - self.starttime = time.monotonic() # Used for uptime calc - self.startms = s_common.now() # Used to report start time + self.starttime = time.monotonic_ns() // 1000 # Used for uptime calc + self.startmicros = s_common.now() # Used to report start time s_telepath.Aware.__init__(self) self.dirn = s_common.gendir(dirn) + self.sockdirn = s_common.gendir(dirn, 'sockets') + self.runid = s_common.guid() self.auth = None @@ -1205,7 +1129,6 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): 'tellready': 1, 'dynmirror': 1, 'tasks': 1, - 'issuewait': 1 } self.safemode = self.conf.req('safemode') @@ -1316,17 +1239,12 @@ async def fini(): self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE) - self.hive = await self._initCellHive() - self.cellinfo = self.slab.getSafeKeyVal('cell:info') self.cellvers = self.slab.getSafeKeyVal('cell:vers') - await self._bumpCellVers('cell:storage', ( - (1, self._storCellHiveMigration), - ), nexs=False) - if self.inaugural: self.cellinfo.set('nexus:version', NEXUS_VERSION) + self.cellvers.set('cell:storage', 1) # Check the cell version didn't regress if (lastver := self.cellinfo.get('cell:version')) is not None and self.VERSION < lastver: @@ -1347,10 +1265,6 @@ async def fini(): self.nexsvers = self.cellinfo.get('nexus:version', (0, 0)) self.nexspatches = () - await self._bumpCellVers('cell:storage', ( - (2, self._storCellAuthMigration), - ), nexs=False) - self.auth = await self._initCellAuth() auth_passwd = self.conf.get('auth:passwd') @@ -1398,138 +1312,6 @@ async def fini(): # phase 5 - service networking await self.initServiceNetwork() - async def _storCellHiveMigration(self): - logger.warning(f'migrating Cell ({self.getCellType()}) info out of hive') - - async with await self.hive.open(('cellvers',)) as versnode: - versdict = await versnode.dict() - for key, valu in versdict.items(): - self.cellvers.set(key, valu) - - async with await self.hive.open(('cellinfo',)) as infonode: - infodict = await infonode.dict() - for key, valu in infodict.items(): - self.cellinfo.set(key, valu) - - logger.warning(f'...Cell ({self.getCellType()}) info migration complete!') - - async def _storCellAuthMigration(self): - if self.conf.get('auth:ctor') is not None: - return - - logger.warning(f'migrating Cell ({self.getCellType()}) auth out of hive') - - authkv = self.slab.getSafeKeyVal('auth') - - async with await self.hive.open(('auth',)) as rootnode: - - rolekv = authkv.getSubKeyVal('role:info:') - rolenamekv = authkv.getSubKeyVal('role:name:') - - async with await rootnode.open(('roles',)) as roles: - for iden, node in roles: - roledict = await node.dict() - roleinfo = roledict.pack() - - roleinfo['iden'] = iden - roleinfo['name'] = node.valu - roleinfo['authgates'] = {} - roleinfo.setdefault('admin', False) - roleinfo.setdefault('rules', ()) - - rolekv.set(iden, roleinfo) - rolenamekv.set(node.valu, iden) - - userkv = authkv.getSubKeyVal('user:info:') - usernamekv = authkv.getSubKeyVal('user:name:') - - async with await rootnode.open(('users',)) as users: - for iden, node in users: - userdict = await node.dict() - userinfo = userdict.pack() - - userinfo['iden'] = iden - userinfo['name'] = node.valu - userinfo['authgates'] = {} - userinfo.setdefault('admin', False) - userinfo.setdefault('rules', ()) - userinfo.setdefault('locked', False) - userinfo.setdefault('passwd', None) - userinfo.setdefault('archived', False) - - realroles = [] - for userrole in userinfo.get('roles', ()): - if rolekv.get(userrole) is None: - mesg = f'Unknown role {userrole} on user {iden} during migration, ignoring.' - logger.warning(mesg) - continue - - realroles.append(userrole) - - userinfo['roles'] = tuple(realroles) - - userkv.set(iden, userinfo) - usernamekv.set(node.valu, iden) - - varskv = authkv.getSubKeyVal(f'user:{iden}:vars:') - async with await node.open(('vars',)) as varnodes: - for name, varnode in varnodes: - varskv.set(name, varnode.valu) - - profkv = authkv.getSubKeyVal(f'user:{iden}:profile:') - async with await node.open(('profile',)) as profnodes: - for name, profnode in profnodes: - profkv.set(name, profnode.valu) - - gatekv = authkv.getSubKeyVal('gate:info:') - async with await rootnode.open(('authgates',)) as authgates: - for gateiden, node in authgates: - gateinfo = { - 'iden': gateiden, - 'type': node.valu - } - gatekv.set(gateiden, gateinfo) - - async with await node.open(('users',)) as usernodes: - for useriden, usernode in usernodes: - if (user := userkv.get(useriden)) is None: - mesg = f'Unknown user {useriden} on gate {gateiden} during migration, ignoring.' - logger.warning(mesg) - continue - - userinfo = await usernode.dict() - userdict = userinfo.pack() - authkv.set(f'gate:{gateiden}:user:{useriden}', userdict) - - user['authgates'][gateiden] = userdict - userkv.set(useriden, user) - - async with await node.open(('roles',)) as rolenodes: - for roleiden, rolenode in rolenodes: - if (role := rolekv.get(roleiden)) is None: - mesg = f'Unknown role {roleiden} on gate {gateiden} during migration, ignoring.' - logger.warning(mesg) - continue - - roleinfo = await rolenode.dict() - roledict = roleinfo.pack() - authkv.set(f'gate:{gateiden}:role:{roleiden}', roledict) - - role['authgates'][gateiden] = roledict - rolekv.set(roleiden, role) - - logger.warning(f'...Cell ({self.getCellType()}) auth migration complete!') - - async def _drivePermMigration(self): - for lkey, lval in self.slab.scanByPref(s_drive.LKEY_INFO, db=self.drive.dbname): - info = s_msgpack.un(lval) - perm = info.pop('perm', None) - if perm is not None: - perm.setdefault('users', {}) - perm.setdefault('roles', {}) - info['permissions'] = perm - self.slab.put(lkey, s_msgpack.en(info), db=self.drive.dbname) - def getPermDef(self, perm): perm = tuple(perm) if self.permlook is None: @@ -1564,13 +1346,7 @@ def getDmonUser(self): async def fini(self): '''Fini override that ensures locking teardown order.''' - - # First we teardown our activebase if it is set. This allows those tasks to be - # cancelled and do any cleanup that they may need to perform. - if self._wouldfini() and self.activebase: - await self.activebase.fini() - - # we inherit from Pusher to make the Cell a Base subclass, so we tear it down through that. + # we inherit from Pusher to make the Cell a Base subclass retn = await s_nexus.Pusher.fini(self) if retn == 0: self._onFiniCellGuid() @@ -1650,7 +1426,7 @@ async def _onBootOptimize(self): for i, lmdbpath in enumerate(lmdbs): - logger.warning(f'... {i+1}/{size} {lmdbpath}') + logger.warning(f'... {i + 1}/{size} {lmdbpath}') with self.getTempDir() as backpath: @@ -1672,22 +1448,33 @@ def _delTmpFiles(self): tdir = s_common.gendir(self.dirn, 'tmp') names = os.listdir(tdir) - if not names: - return + if names: + logger.info(f'Removing {len(names)} temporary files/folders in: {tdir}') - logger.warning(f'Removing {len(names)} temporary files/folders in: {tdir}') + for name in names: - for name in names: + path = os.path.join(tdir, name) - path = os.path.join(tdir, name) + if os.path.isfile(path): + os.unlink(path) + continue - if os.path.isfile(path): - os.unlink(path) - continue + if os.path.isdir(path): + shutil.rmtree(path, ignore_errors=True) + continue - if os.path.isdir(path): - shutil.rmtree(path, ignore_errors=True) - continue + names = os.listdir(self.sockdirn) + if names: + logger.info(f'Removing {len(names)} old sockets in: {self.sockdirn}') + for name in names: + path = os.path.join(self.sockdirn, name) + try: + if stat.S_ISSOCK(os.stat(path).st_mode): + os.unlink(path) + except OSError: # pragma: no cover + pass + + # FIXME - recursively remove sockets dir here? async def _execCellUpdates(self): # implement to apply updates to a fully initialized active cell @@ -1844,10 +1631,6 @@ async def onlink(proxy): elif isinstance(oldurls, list): oldurls = tuple(oldurls) if newurls and newurls != oldurls: - if oldurls[0].startswith('tcp://'): - s_common.deprecated('aha:registry: tcp:// client values.') - return - self.modCellConf({'aha:registry': newurls}) self.ahaclient.setBootUrls(newurls) @@ -1899,10 +1682,16 @@ async def initServiceEarly(self): pass async def initCellStorage(self): - self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') - await self._bumpCellVers('drive:storage', ( - (1, self._drivePermMigration), - ), nexs=False) + + path = s_common.gendir(self.dirn, 'slabs', 'drive.lmdb') + sockpath = s_common.genpath(self.sockdirn, 'drive') + + if len(sockpath) > s_const.UNIX_PATH_MAX: + sockpath = None + + spawner = s_drive.FileDrive.spawner(base=self, sockpath=sockpath) + + self.drive = await spawner(path) self.onfini(self.drive.fini) @@ -1922,7 +1711,7 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): # replay safety... iden = info.get('iden') - if self.drive.hasItemInfo(iden): # pragma: no cover + if await self.drive.hasItemInfo(iden): # pragma: no cover return await self.drive.getItemPath(iden) # TODO: Remove this in synapse-3xx @@ -1935,10 +1724,10 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): return await self.drive.addItemInfo(info, path=path, reldir=reldir) async def getDriveInfo(self, iden, typename=None): - return self.drive.getItemInfo(iden, typename=typename) + return await self.drive.getItemInfo(iden, typename=typename) - def reqDriveInfo(self, iden, typename=None): - return self.drive.reqItemInfo(iden, typename=typename) + async def reqDriveInfo(self, iden, typename=None): + return await self.drive.reqItemInfo(iden, typename=typename) async def getDrivePath(self, path, reldir=s_drive.rootdir): ''' @@ -1961,14 +1750,16 @@ async def addDrivePath(self, path, perm=None, reldir=s_drive.rootdir): ''' tick = s_common.now() user = self.auth.rootuser.iden - path = self.drive.getPathNorm(path) + path = await self.drive.getPathNorm(path) if perm is None: perm = {'users': {}, 'roles': {}} for name in path: - info = self.drive.getStepInfo(reldir, name) + info = await self.drive.getStepInfo(reldir, name) + + # we could skip this now ;) await asyncio.sleep(0) if info is not None: @@ -1992,7 +1783,7 @@ async def getDriveData(self, iden, vers=None): Return the data associated with the drive item by iden. If vers is specified, return that specific version. ''' - return self.drive.getItemData(iden, vers=vers) + return await self.drive.getItemData(iden, vers=vers) async def getDriveDataVersions(self, iden): async for item in self.drive.getItemDataVersions(iden): @@ -2000,12 +1791,12 @@ async def getDriveDataVersions(self, iden): @s_nexus.Pusher.onPushAuto('drive:del') async def delDriveInfo(self, iden): - if self.drive.getItemInfo(iden) is not None: + if await self.drive.getItemInfo(iden) is not None: await self.drive.delItemInfo(iden) @s_nexus.Pusher.onPushAuto('drive:set:perm') async def setDriveInfoPerm(self, iden, perm): - return self.drive.setItemPerm(iden, perm) + return await self.drive.setItemPerm(iden, perm) @s_nexus.Pusher.onPushAuto('drive:data:path:set') async def setDriveItemProp(self, iden, vers, path, valu): @@ -2054,7 +1845,7 @@ async def delDriveItemProp(self, iden, vers, path): @s_nexus.Pusher.onPushAuto('drive:set:path') async def setDriveInfoPath(self, iden, path): - path = self.drive.getPathNorm(path) + path = await self.drive.getPathNorm(path) pathinfo = await self.drive.getItemPath(iden) if path == [p.get('name') for p in pathinfo]: return pathinfo @@ -2067,13 +1858,13 @@ async def setDriveData(self, iden, versinfo, data): async def delDriveData(self, iden, vers=None): if vers is None: - info = self.drive.reqItemInfo(iden) + info = await self.drive.reqItemInfo(iden) vers = info.get('version') return await self._push('drive:data:del', iden, vers) @s_nexus.Pusher.onPush('drive:data:del') async def _delDriveData(self, iden, vers): - return self.drive.delItemData(iden, vers) + return await self.drive.delItemData(iden, vers) async def getDriveKids(self, iden): async for info in self.drive.getItemKids(iden): @@ -2108,11 +1899,11 @@ async def _bindDmonListen(self): logger.error('LOCAL UNIX SOCKET WILL BE UNAVAILABLE') except Exception: # pragma: no cover logging.exception('Unknown dmon listen error.') - - turl = self._getDmonListen() - if turl is not None: - logger.info(f'dmon listening: {turl}') - self.sockaddr = await self.dmon.listen(turl) + else: + turl = self._getDmonListen() + if turl is not None: + logger.info(f'dmon listening: {turl}') + self.sockaddr = await self.dmon.listen(turl) async def initServiceNetwork(self): @@ -2167,6 +1958,7 @@ async def getAhaInfo(self): 'leader': ahalead, 'urlinfo': urlinfo, 'ready': ready, + 'isleader': self.isactive, 'promotable': self.conf.get('aha:promotable'), } @@ -2201,9 +1993,9 @@ async def _runAhaRegLoop(): proxy = await self.ahaclient.proxy() info = await self.getAhaInfo() - await proxy.addAhaSvc(ahaname, info, network=ahanetw) + await proxy.addAhaSvc(f'{ahaname}...', info) if self.isactive and ahalead is not None: - await proxy.addAhaSvc(ahalead, info, network=ahanetw) + await proxy.addAhaSvc(f'{ahalead}...', info) except Exception as e: logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}') @@ -2285,10 +2077,6 @@ async def waitFor(turl_sani, prox_): return cullat - @s_nexus.Pusher.onPushAuto('nexslog:setindex') - async def setNexsIndx(self, indx): - return await self.nexsroot.setindex(indx) - def getMyUrl(self, user='root'): host = self.conf.req('aha:name') network = self.conf.req('aha:network') @@ -2479,38 +2267,21 @@ async def reqAhaProxy(self, timeout=None, feats=None): return proxy - async def _setAhaActive(self): + async def _bumpAhaProxy(self): if self.ahaclient is None: return - ahainfo = await self.getAhaInfo() - if ahainfo is None: - return - - ahalead = self.conf.get('aha:leader') - if ahalead is None: - return - + # force a reconnect to AHA to update service info try: - proxy = await self.ahaclient.proxy(timeout=2) + proxy = await self.ahaclient.proxy(timeout=5) + if proxy is not None: + await proxy.fini() - except TimeoutError: # pragma: no cover - return None - - # if we went inactive, bump the aha proxy - if not self.isactive: - await proxy.fini() - return - - ahanetw = self.conf.req('aha:network') - try: - await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw) - except asyncio.CancelledError: # pragma: no cover - raise - except Exception as e: # pragma: no cover - logger.warning(f'_setAhaActive failed: {e}') + except Exception as e: + extra = await self.getLogExtra(name=self.conf.get('aha:name')) + logger.exception('Error forcing AHA reconnect.', extra=extra) def isActiveCoro(self, iden): return self.activecoros.get(iden) is not None @@ -2622,7 +2393,7 @@ async def setCellActive(self, active): self.activebase = None await self.initServicePassive() - await self._setAhaActive() + await self._bumpAhaProxy() def runActiveTask(self, coro): # an API for active coroutines to use when running an @@ -2990,18 +2761,10 @@ async def getUserProfInfo(self, iden, name, default=None): user = await self.auth.reqUser(iden) return user.profile.get(name, defv=default) - async def _setUserProfInfoV0(self, iden, name, valu): - path = ('auth', 'users', iden, 'profile', name) - return await self.hive._push('hive:set', path, valu) - async def setUserProfInfo(self, iden, name, valu): user = await self.auth.reqUser(iden) return await user.setProfileValu(name, valu) - async def _popUserProfInfoV0(self, iden, name, default=None): - path = ('auth', 'users', iden, 'profile', name) - return await self.hive._push('hive:pop', path) - async def popUserProfInfo(self, iden, name, default=None): user = await self.auth.reqUser(iden) return await user.popProfileValu(name, default=default) @@ -3022,18 +2785,10 @@ async def getUserVarValu(self, iden, name, default=None): user = await self.auth.reqUser(iden) return user.vars.get(name, defv=default) - async def _setUserVarValuV0(self, iden, name, valu): - path = ('auth', 'users', iden, 'vars', name) - return await self.hive._push('hive:set', path, valu) - async def setUserVarValu(self, iden, name, valu): user = await self.auth.reqUser(iden) return await user.setVarValu(name, valu) - async def _popUserVarValuV0(self, iden, name, default=None): - path = ('auth', 'users', iden, 'vars', name) - return await self.hive._push('hive:pop', path) - async def popUserVarValu(self, iden, name, default=None): user = await self.auth.reqUser(iden) return await user.popVarValu(name, default=default) @@ -3598,13 +3353,6 @@ async def _initCellDmon(self): self.onfini(self.dmon.fini) - async def _initCellHive(self): - db = self.slab.initdb('hive') - hive = await s_hive.SlabHive.anit(self.slab, db=db, nexsroot=self.getCellNexsRoot(), cell=self) - self.onfini(hive) - - return hive - async def _initSlabFile(self, path, readonly=False, ephemeral=False): slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly) slab.addResizeCallback(self.checkFreeSpace) @@ -3622,7 +3370,6 @@ async def _initCellSlab(self, readonly=False): if not os.path.exists(path) and readonly: logger.warning('Creating a slab for a readonly cell.') _slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE) - _slab.initdb('hive') await _slab.fini() self.slab = await self._initSlabFile(path) @@ -3633,16 +3380,10 @@ async def _initCellAuth(self): self.on('user:del', self._onUserDelEvnt) self.on('user:lock', self._onUserLockEvnt) - authctor = self.conf.get('auth:ctor') - if authctor is not None: - s_common.deprecated('auth:ctor cell config option', curv='2.157.0') - ctor = s_dyndeps.getDynLocal(authctor) - return await ctor(self) - maxusers = self.conf.get('max:users') policy = self.conf.get('auth:passwd:policy') - seed = s_common.guid((self.iden, 'hive', 'auth')) + seed = s_common.guid((self.iden, 'auth')) auth = await s_auth.Auth.anit( self.slab, @@ -4135,7 +3876,7 @@ async def _initBootRestore(cls, dirn): content_length = int(resp.headers.get('content-length', 0)) if content_length > 100: - logger.warning(f'Downloading {content_length/s_const.megabyte:.3f} MB of data.') + logger.warning(f'Downloading {content_length / s_const.megabyte:.3f} MB of data.') pvals = [int((content_length * 0.01) * i) for i in range(1, 100)] else: # pragma: no cover logger.warning(f'Odd content-length encountered: {content_length}') @@ -4151,7 +3892,7 @@ async def _initBootRestore(cls, dirn): if pvals and tsize > pvals[0]: pvals.pop(0) percentage = (tsize / content_length) * 100 - logger.warning(f'Downloaded {tsize/s_const.megabyte:.3f} MB, {percentage:.3f}%') + logger.warning(f'Downloaded {tsize / s_const.megabyte:.3f} MB, {percentage:.3f}%') logger.warning(f'Extracting {tarpath} to {dirn}') @@ -4161,7 +3902,7 @@ async def _initBootRestore(cls, dirn): continue memb.name = memb.name.split('/', 1)[1] logger.warning(f'Extracting {memb.name}') - tgz.extract(memb, dirn) + tgz.extract(memb, dirn, filter='data') # and record the rurliden with s_common.genfile(donepath) as fd: @@ -4362,7 +4103,7 @@ async def _initCloneCell(self, proxy): if memb.name.find('/') == -1: continue memb.name = memb.name.split('/', 1)[1] - tgz.extract(memb, self.dirn) + tgz.extract(memb, self.dirn, filter='data') finally: @@ -4403,6 +4144,7 @@ async def _bootCellMirror(self, pnfo): logger.warning(f'Bootstrap mirror from: {murl} DONE!') async def getMirrorUrls(self): + if self.ahaclient is None: raise s_exc.BadConfValu(mesg='Enumerating mirror URLs is only supported when AHA is configured') @@ -4414,7 +4156,7 @@ async def getMirrorUrls(self): mesg = 'Service must be configured with AHA to enumerate mirror URLs' raise s_exc.NoSuchName(mesg=mesg, name=self.ahasvcname) - return [f'aha://{svc["svcname"]}.{svc["svcnetw"]}' for svc in mirrors] + return [f'aha://{svc["name"]}' for svc in mirrors] @classmethod async def initFromArgv(cls, argv, outp=None): @@ -4466,7 +4208,7 @@ async def initFromArgv(cls, argv, outp=None): logger.exception(f'Error while bootstrapping cell config.') raise - s_processpool.set_pool_logging(logger, logconf=conf['_log_conf']) + s_coro.set_pool_logging(logger, logconf=conf['_log_conf']) try: cell = await cls.anit(opts.dirn, conf=conf) @@ -4596,59 +4338,6 @@ async def _cellHealth(self, health): async def getDmonSessions(self): return await self.dmon.getSessInfo() - # ----- Change distributed Auth methods ---- - - async def listHiveKey(self, path=None): - if path is None: - path = () - items = self.hive.dir(path) - if items is None: - return None - return [item[0] for item in items] - - async def getHiveKeys(self, path): - ''' - Return a list of (name, value) tuples for nodes under the path. - ''' - items = self.hive.dir(path) - if items is None: - return () - - return [(i[0], i[1]) for i in items] - - async def getHiveKey(self, path): - ''' - Get the value of a key in the cell default hive - ''' - return await self.hive.get(path) - - async def setHiveKey(self, path, valu): - ''' - Set or change the value of a key in the cell default hive - ''' - return await self.hive.set(path, valu, nexs=True) - - async def popHiveKey(self, path): - ''' - Remove and return the value of a key in the cell default hive. - - Note: this is for expert emergency use only. - ''' - return await self.hive.pop(path, nexs=True) - - async def saveHiveTree(self, path=()): - return await self.hive.saveHiveTree(path=path) - - async def loadHiveTree(self, tree, path=(), trim=False): - ''' - Note: this is for expert emergency use only. - ''' - return await self._push('hive:loadtree', tree, path, trim) - - @s_nexus.Pusher.onPush('hive:loadtree') - async def _onLoadHiveTree(self, tree, path, trim): - return await self.hive.loadHiveTree(tree, path=path, trim=trim) - async def iterSlabData(self, name, prefix=''): slabkv = self.slab.getSafeKeyVal(name, prefix=prefix, create=False) for key, valu in slabkv.items(): @@ -4822,8 +4511,6 @@ async def getCellInfo(self): if mirror is not None: mirror = s_urlhelp.sanitizeUrl(mirror) - nxfo = await self.nexsroot.getNexsInfo() - ret = { 'synapse': { 'commit': s_version.commit, @@ -4836,15 +4523,15 @@ async def getCellInfo(self): 'iden': self.getCellIden(), 'paused': self.paused, 'active': self.isactive, + 'started': self.startmicros, 'safemode': self.safemode, - 'started': self.startms, - 'ready': nxfo['ready'], # TODO: Remove in 3.x.x + 'ready': self.nexsroot.ready.is_set(), 'commit': self.COMMIT, 'version': self.VERSION, 'verstring': self.VERSTRING, 'cellvers': dict(self.cellvers.items()), - 'nexsindx': nxfo['indx'], # TODO: Remove in 3.x.x - 'uplink': nxfo['uplink:ready'], # TODO: Remove in 3.x.x + 'nexsindx': await self.getNexsIndx(), + 'uplink': self.nexsroot.miruplink.is_set(), 'mirror': mirror, 'aha': { 'name': self.conf.get('aha:name'), @@ -4853,8 +4540,7 @@ async def getCellInfo(self): }, 'network': { 'https': self.https_listeners, - }, - 'nexus': nxfo, + } }, 'features': self.features, } @@ -4873,8 +4559,8 @@ async def getSystemInfo(self): - volfree - Volume where cell is running free space - backupvolsize - Backup directory volume total space - backupvolfree - Backup directory volume free space - - cellstarttime - Cell start time in epoch milliseconds - - celluptime - Cell uptime in milliseconds + - cellstarttime - Cell start time in epoch microseconds + - celluptime - Cell uptime in microseconds - cellrealdisk - Cell's use of disk, equivalent to du - cellapprdisk - Cell's apparent use of disk, equivalent to ls -l - osversion - OS version/architecture @@ -4884,7 +4570,7 @@ async def getSystemInfo(self): - cpucount - Number of CPUs on system - tmpdir - The temporary directory interpreted by the Python runtime. ''' - uptime = int((time.monotonic() - self.starttime) * 1000) + uptime = time.monotonic_ns() // 1000 - self.starttime disk = shutil.disk_usage(self.dirn) if self.backdirn: @@ -4906,8 +4592,8 @@ async def getSystemInfo(self): 'volfree': disk.free, # Volume where cell is running free bytes 'backupvolsize': backupvolsize, # Cell's backup directory volume total bytes 'backupvolfree': backupvolfree, # Cell's backup directory volume free bytes - 'cellstarttime': self.startms, # cell start time in epoch millis - 'celluptime': uptime, # cell uptime in ms + 'cellstarttime': self.startmicros, # Cell's start time in epoch micros + 'celluptime': uptime, # Cell's uptime in micros 'cellrealdisk': myusage, # Cell's use of disk, equivalent to du 'cellapprdisk': myappusage, # Cell's apparent use of disk, equivalent to ls -l 'osversion': platform.platform(), # OS version/architecture @@ -5000,7 +4686,7 @@ async def addUserApiKey(self, useriden, name, duration=None): Args: useriden (str): User iden value. name (str): Name of the API key. - duration (int or None): Duration of time for the API key to be valid ( in milliseconds ). + duration (int or None): Duration of time for the API key to be valid ( in microseconds ). Returns: tuple: A tuple of the secret API key value and the API key metadata information. @@ -5037,9 +4723,9 @@ async def addUserApiKey(self, useriden, name, duration=None): async def _genUserApiKey(self, kdef): iden = s_common.uhex(kdef.get('iden')) user = s_common.uhex(kdef.get('user')) - self.slab.put(iden, user, db=self.apikeydb) + await self.slab.put(iden, user, db=self.apikeydb) lkey = user + b'apikey' + iden - self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) + await self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) async def getUserApiKey(self, iden): ''' @@ -5195,7 +4881,7 @@ async def _setUserApiKey(self, user, iden, vals): raise s_exc.NoSuchIden(mesg=f'User API key does not exist: {iden}') kdef = s_msgpack.un(buf) kdef.update(vals) - self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) + await self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) return kdef async def delUserApiKey(self, iden): @@ -5269,6 +4955,8 @@ def _makeCachedSslCtx(self, opts): else: sslctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) + sslctx.verify_flags &= ~ssl.VERIFY_X509_STRICT + if not opts['verify']: sslctx.check_hostname = False sslctx.verify_mode = ssl.CERT_NONE @@ -5312,14 +5000,12 @@ def _makeCachedSslCtx(self, opts): return sslctx - def getCachedSslCtx(self, opts=None, verify=None): + def getCachedSslCtx(self, opts=None): + # Default to verifying SSL/TLS certificates if opts is None: opts = {} - if verify is not None: - opts['verify'] = verify - opts = s_schemas.reqValidSslCtxOpts(opts) key = tuple(sorted(opts.items())) diff --git a/synapse/lib/const.py b/synapse/lib/const.py index 2ae64e83764..655af704aaa 100644 --- a/synapse/lib/const.py +++ b/synapse/lib/const.py @@ -31,8 +31,8 @@ zebibyte = 1024 * exbibyte yobibyte = 1024 * zebibyte -# time (in millis) constants -second = 1000 +# time (in micros) constants +second = 1000000 minute = second * 60 hour = minute * 60 day = hour * 24 @@ -51,3 +51,6 @@ # HTTP header constants MAX_LINE_SIZE = kibibyte * 64 MAX_FIELD_SIZE = kibibyte * 64 + +# Socket constants +UNIX_PATH_MAX = 107 diff --git a/synapse/lib/drive.py b/synapse/lib/drive.py index 375d6f05dee..e051fcaabfd 100644 --- a/synapse/lib/drive.py +++ b/synapse/lib/drive.py @@ -6,8 +6,10 @@ import synapse.lib.base as s_base import synapse.lib.config as s_config +import synapse.lib.dyndeps as s_dyndeps import synapse.lib.msgpack as s_msgpack import synapse.lib.schemas as s_schemas +import synapse.lib.spawner as s_spawner nameregex = regex.compile(s_schemas.re_drivename) def reqValidName(name): @@ -54,7 +56,7 @@ async def __anit__(self, slab, name): self.dbname = slab.initdb(f'drive:{name}') self.validators = {} - def getPathNorm(self, path): + async def getPathNorm(self, path): if isinstance(path, str): path = path.strip().strip('/').split('/') @@ -67,7 +69,7 @@ def _reqInfoType(self, info, typename): mesg = f'Drive item has the wrong type. Expected: {typename} got {infotype}.' raise s_exc.TypeMismatch(mesg=mesg, expected=typename, got=infotype) - def getItemInfo(self, iden, typename=None): + async def getItemInfo(self, iden, typename=None): info = self._getItemInfo(s_common.uhex(iden)) if not info: return @@ -81,7 +83,7 @@ def _getItemInfo(self, bidn): if byts is not None: return s_msgpack.un(byts) - def reqItemInfo(self, iden, typename=None): + async def reqItemInfo(self, iden, typename=None): return self._reqItemInfo(s_common.uhex(iden), typename=typename) def _reqItemInfo(self, bidn, typename=None): @@ -105,7 +107,7 @@ async def getItemPath(self, iden): pathinfo = [] while iden is not None: - info = self.reqItemInfo(iden) + info = await self.reqItemInfo(iden) pathinfo.append(info) iden = info.get('parent') @@ -117,7 +119,7 @@ async def getItemPath(self, iden): async def _setItemPath(self, bidn, path, reldir=rootdir): - path = self.getPathNorm(path) + path = await self.getPathNorm(path) # new parent iden / bidn parinfo = None @@ -171,7 +173,7 @@ async def _setItemPath(self, bidn, path, reldir=rootdir): def _hasStepItem(self, bidn, name): return self.slab.has(LKEY_DIRN + bidn + name.encode(), db=self.dbname) - def getStepInfo(self, iden, name): + async def getStepInfo(self, iden, name): return self._getStepInfo(s_common.uhex(iden), name) def _getStepInfo(self, bidn, name): @@ -208,14 +210,14 @@ async def _addStepInfo(self, parbidn, parinfo, info): await self.slab.putmulti(rows, db=self.dbname) - def setItemPerm(self, iden, perm): + async def setItemPerm(self, iden, perm): return self._setItemPerm(s_common.uhex(iden), perm) def _setItemPerm(self, bidn, perm): info = self._reqItemInfo(bidn) info['permissions'] = perm s_schemas.reqValidDriveInfo(info) - self.slab.put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) + self.slab._put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) return info async def getPathInfo(self, path, reldir=rootdir): @@ -227,7 +229,7 @@ async def getPathInfo(self, path, reldir=rootdir): and potentially check permissions on each level to control access. ''' - path = self.getPathNorm(path) + path = await self.getPathNorm(path) parbidn = s_common.uhex(reldir) pathinfo = [] @@ -244,7 +246,7 @@ async def getPathInfo(self, path, reldir=rootdir): return pathinfo - def hasItemInfo(self, iden): + async def hasItemInfo(self, iden): return self._hasItemInfo(s_common.uhex(iden)) def _hasItemInfo(self, bidn): @@ -254,7 +256,7 @@ async def hasPathInfo(self, path, reldir=rootdir): ''' Check for a path existing relative to reldir. ''' - path = self.getPathNorm(path) + path = await self.getPathNorm(path) parbidn = s_common.uhex(reldir) for part in path: @@ -277,7 +279,7 @@ async def addItemInfo(self, info, path=None, reldir=rootdir): pathinfo = [] if path is not None: - path = self.getPathNorm(path) + path = await self.getPathNorm(path) pathinfo = await self.getPathInfo(path, reldir=reldir) if pathinfo: pariden = pathinfo[-1].get('iden') @@ -300,7 +302,7 @@ async def addItemInfo(self, info, path=None, reldir=rootdir): bidn = s_common.uhex(iden) if typename is not None: - self.reqTypeValidator(typename) + await self.reqTypeValidator(typename) if self._getItemInfo(bidn) is not None: mesg = f'A drive entry with ID {iden} already exists.' @@ -311,7 +313,7 @@ async def addItemInfo(self, info, path=None, reldir=rootdir): pathinfo.append(info) return pathinfo - def reqFreeStep(self, iden, name): + async def reqFreeStep(self, iden, name): return self._reqFreeStep(s_common.uhex(iden), name) def _reqFreeStep(self, bidn, name): @@ -362,7 +364,7 @@ async def _walkItemInfo(self, bidn): async def walkPathInfo(self, path, reldir=rootdir): - path = self.getPathNorm(path) + path = await self.getPathNorm(path) pathinfo = await self.getPathInfo(path, reldir=reldir) bidn = s_common.uhex(pathinfo[-1].get('iden')) @@ -409,7 +411,7 @@ async def _setItemData(self, bidn, versinfo, data): typename = info.get('type') - self.reqValidData(typename, data) + await self.reqValidData(typename, data) byts = s_msgpack.en(data) @@ -439,7 +441,7 @@ async def _setItemData(self, bidn, versinfo, data): return info, versinfo - def getItemData(self, iden, vers=None): + async def getItemData(self, iden, vers=None): ''' Return a (versinfo, data) tuple for the given iden. If version is not specified, the current version is returned. @@ -465,7 +467,7 @@ def _getItemData(self, bidn, vers=None): return s_msgpack.un(versbyts), s_msgpack.un(databyts) - def delItemData(self, iden, vers=None): + async def delItemData(self, iden, vers=None): return self._delItemData(s_common.uhex(iden), vers=vers) def _delItemData(self, bidn, vers=None): @@ -490,7 +492,7 @@ def _delItemData(self, bidn, vers=None): else: info.update(versinfo) - self.slab.put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) + self.slab._put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) return info def _getLastDataVers(self, bidn): @@ -507,12 +509,12 @@ async def getItemDataVersions(self, iden): yield s_msgpack.un(byts) await asyncio.sleep(0) - def getTypeSchema(self, typename): + async def getTypeSchema(self, typename): byts = self.slab.get(LKEY_TYPE + typename.encode(), db=self.dbname) if byts is not None: return s_msgpack.un(byts, use_list=True) - def getTypeSchemaVersion(self, typename): + async def getTypeSchemaVersion(self, typename): verskey = LKEY_TYPE_VERS + typename.encode() byts = self.slab.get(verskey, db=self.dbname) if byts is not None: @@ -522,9 +524,16 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): reqValidName(typename) + if isinstance(callback, str): + callback = s_dyndeps.getDynLocal(callback) + + # if we were invoked via telepath, the schmea needs to be mutable... + schema = s_msgpack.deepcopy(schema, use_list=True) + + curv = await self.getTypeSchemaVersion(typename) + if vers is not None: vers = int(vers) - curv = self.getTypeSchemaVersion(typename) if curv is not None: if vers == curv: return False @@ -539,11 +548,11 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): lkey = LKEY_TYPE + typename.encode() - self.slab.put(lkey, s_msgpack.en(schema), db=self.dbname) + await self.slab.put(lkey, s_msgpack.en(schema), db=self.dbname) if vers is not None: verskey = LKEY_TYPE_VERS + typename.encode() - self.slab.put(verskey, s_msgpack.en(vers), db=self.dbname) + await self.slab.put(verskey, s_msgpack.en(vers), db=self.dbname) if callback is not None: async for info in self.getItemsByType(typename): @@ -551,12 +560,25 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): for lkey, byts in self.slab.scanByPref(LKEY_VERS + bidn, db=self.dbname): versindx = lkey[-9:] databyts = self.slab.get(LKEY_DATA + bidn + versindx, db=self.dbname) - data = await callback(info, s_msgpack.un(byts), s_msgpack.un(databyts)) + data = await callback(info, s_msgpack.un(byts), s_msgpack.un(databyts), curv) vtor(data) - self.slab.put(LKEY_DATA + bidn + versindx, s_msgpack.en(data), db=self.dbname) + await self.slab.put(LKEY_DATA + bidn + versindx, s_msgpack.en(data), db=self.dbname) await asyncio.sleep(0) return True + async def getMigrRows(self, typename): + + async for info in self.getItemsByType(typename): + + bidn = s_common.uhex(info.get('iden')) + for lkey, byts in self.slab.scanByPref(LKEY_VERS + bidn, db=self.dbname): + datakey = LKEY_DATA + bidn + lkey[-9:] + databyts = self.slab.get(datakey, db=self.dbname) + yield datakey, s_msgpack.un(databyts) + + async def setMigrRow(self, datakey, data): + self.slab.put(datakey, s_msgpack.en(data), db=self.dbname) + async def getItemsByType(self, typename): tkey = typename.encode() + b'\x00' for lkey in self.slab.scanKeysByPref(LKEY_INFO_BYTYPE + tkey, db=self.dbname): @@ -565,12 +587,12 @@ async def getItemsByType(self, typename): if info is not None: yield info - def getTypeValidator(self, typename): + async def getTypeValidator(self, typename): vtor = self.validators.get(typename) if vtor is not None: return vtor - schema = self.getTypeSchema(typename) + schema = await self.getTypeSchema(typename) if schema is None: return None @@ -579,13 +601,20 @@ def getTypeValidator(self, typename): return vtor - def reqTypeValidator(self, typename): - vtor = self.getTypeValidator(typename) + async def reqTypeValidator(self, typename): + vtor = await self.getTypeValidator(typename) if vtor is not None: return vtor mesg = f'No schema registered with name: {typename}' raise s_exc.NoSuchType(mesg=mesg) - def reqValidData(self, typename, item): - self.reqTypeValidator(typename)(item) + async def reqValidData(self, typename, item): + return (await self.reqTypeValidator(typename))(item) + +class FileDrive(Drive, s_spawner.SpawnerMixin): + + async def __anit__(self, path): + import synapse.lib.lmdbslab as s_lmdbslab + slab = await s_lmdbslab.Slab.anit(path) + return await Drive.__anit__(self, slab, 'drive') diff --git a/synapse/lib/spawner.py b/synapse/lib/spawner.py new file mode 100644 index 00000000000..bd386f12a7c --- /dev/null +++ b/synapse/lib/spawner.py @@ -0,0 +1,80 @@ +import asyncio +import logging +import tempfile + +import synapse.exc as s_exc +import synapse.common as s_common +import synapse.daemon as s_daemon +import synapse.telepath as s_telepath + +import synapse.lib.base as s_base +import synapse.lib.coro as s_coro +import synapse.lib.link as s_link + +logger = logging.getLogger(__name__) + +def _ioWorkProc(todo, sockpath): + + async def workloop(): + + async with await s_daemon.Daemon.anit() as dmon: + + func, args, kwargs = todo + + item = await func(*args, **kwargs) + + dmon.share('dmon', dmon) + dmon.share('item', item) + + # bind last so we're ready to go... + await dmon.listen(f'unix://{sockpath}') + await item.waitfini() + + asyncio.run(workloop()) + +class SpawnerMixin: + + @classmethod + def spawner(cls, base=None, sockpath=None): + async def _spawn(*args, **kwargs): + return await cls._spawn(args, kwargs, base=base, sockpath=sockpath) + return _spawn + + @classmethod + async def _spawn(cls, args, kwargs, base=None, sockpath=None): + + todo = (cls.anit, args, kwargs) + + iden = s_common.guid() + + if sockpath is None: + tmpdir = tempfile.gettempdir() + sockpath = s_common.genpath(tmpdir, iden) + + if base is None: + base = await s_base.Base.anit() + + base.schedCoro(s_coro.spawn((_ioWorkProc, (todo, sockpath), {}))) + + await s_link.unixwait(sockpath) + + proxy = await s_telepath.openurl(f'unix://{sockpath}:item') + + async def fini(): + + try: + async with await s_telepath.openurl(f'unix://{sockpath}:item') as finiproxy: + await finiproxy.task(('fini', (), {})) + except (s_exc.LinkErr, s_exc.NoSuchPath, asyncio.CancelledError): + # This can fail if the subprocess was terminated from outside... + pass + + if not base.isfini: + logger.error(f'IO Worker Socket Closed: {sockpath}') + + await base.fini() + + proxy.onfini(fini) + base.onfini(proxy) + + return proxy diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index faa8a4df7f1..88090ada089 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -10,6 +10,7 @@ import synapse.lib.base as s_base import synapse.lib.coro as s_coro import synapse.lib.scope as s_scope +import synapse.lib.spawner as s_spawner import synapse.tests.utils as s_t_utils @@ -45,6 +46,14 @@ async def postAnit(self): if self.foo == -1: raise s_exc.BadArg(mesg='boom') +class Haha(s_base.Base, s_spawner.SpawnerMixin): + + async def __anit__(self): + await s_base.Base.__anit__(self) + + async def fini(self): + await s_base.Base.fini(self) + class BaseTest(s_t_utils.SynTest): async def test_base_basics(self): @@ -296,18 +305,14 @@ async def callfini(): async def test_base_refcount(self): base = await s_base.Base.anit() - self.true(base._wouldfini()) self.eq(base.incref(), 2) - self.false(base._wouldfini()) self.eq(await base.fini(), 1) self.false(base.isfini) - self.true(base._wouldfini()) self.eq(await base.fini(), 0) self.true(base.isfini) - self.false(base._wouldfini()) async def test_baseref_gen(self): @@ -436,6 +441,21 @@ def onHehe1(mesg): self.len(2, l0) self.len(1, l1) + # set the 'hehe' and haha callback with onWithMulti + with base.onWithMulti(('hehe', 'haha'), onHehe1) as e: + self.true(e is base) + await base.fire('hehe') + self.len(3, l0) + self.len(2, l1) + + await base.fire('haha') + self.len(3, l0) + self.len(3, l1) + + await base.fire('hehe') + self.len(4, l0) + self.len(3, l1) + async def test_base_mixin(self): data = [] @@ -570,3 +590,29 @@ async def func3(bobj, key, valu): # The scope data set in the task is not present outside of it. self.none(s_scope.get('hehe')) + + async def test_base_spawner_fini(self): + + # Test proxy fini, base should fini + base = await s_base.Base.anit() + spawner = Haha.spawner(base=base) + proxy = await spawner() + + self.false(proxy.isfini) + self.false(base.isfini) + + await proxy.fini() + self.true(proxy.isfini) + self.true(base.isfini) + + # Test base fini, proxy should fini + base = await s_base.Base.anit() + spawner = Haha.spawner(base=base) + proxy = await spawner() + + self.false(proxy.isfini) + self.false(base.isfini) + + await base.fini() + self.true(base.isfini) + self.true(proxy.isfini) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 353d6793695..80c165c0680 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -28,6 +28,7 @@ import synapse.lib.coro as s_coro import synapse.lib.json as s_json import synapse.lib.link as s_link +import synapse.lib.const as s_const import synapse.lib.drive as s_drive import synapse.lib.nexus as s_nexus import synapse.lib.config as s_config @@ -56,6 +57,11 @@ def _exiterProc(pipe, srcdir, dstdir, lmdbpaths, logconf): def _backupSleep(path, linkinfo): time.sleep(3.0) +async def migrate_v1(info, versinfo, data, curv): + assert curv == 1 + data['woot'] = 'woot' + return data + async def _doEOFBackup(path): return @@ -143,32 +149,6 @@ async def stream(self, doraise=False): if doraise: raise s_exc.BadTime(mesg='call again later') -async def altAuthCtor(cell): - authconf = cell.conf.get('auth:conf') - assert authconf['foo'] == 'bar' - authconf['baz'] = 'faz' - - maxusers = cell.conf.get('max:users') - - seed = s_common.guid((cell.iden, 'hive', 'auth')) - - auth = await s_auth.Auth.anit( - cell.slab, - 'auth', - seed=seed, - nexsroot=cell.getCellNexsRoot(), - maxusers=maxusers - ) - - auth.link(cell.dist) - - def finilink(): - auth.unlink(cell.dist) - - cell.onfini(finilink) - cell.onfini(auth.fini) - return auth - testDataSchema_v0 = { 'type': 'object', 'properties': { @@ -274,9 +254,9 @@ async def test_cell_drive(self): neatrole = await cell.auth.addRole('neatrole') await fooser.grant(neatrole.iden) - with self.raises(s_exc.SchemaViolation): - versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) + # with self.raises(s_exc.SchemaViolation): + # versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + # await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20, 'stuff': 12}) @@ -332,11 +312,10 @@ async def test_cell_drive(self): self.eq(data[1]['stuff'], 1234) # This will be done by the cell in a cell storage version migration... - async def migrate_v1(info, versinfo, data): - data['woot'] = 'woot' - return data + callback = 'synapse.tests.test_lib_cell.migrate_v1' + await cell.drive.setTypeSchema('woot', testDataSchema_v1, callback=callback) - await cell.drive.setTypeSchema('woot', testDataSchema_v1, migrate_v1) + await cell.setDriveItemProp(iden, versinfo, 'woot', 'woot') versinfo['version'] = (1, 1, 1) await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829) @@ -438,7 +417,7 @@ async def migrate_v1(info, versinfo, data): with self.raises(s_exc.DupName): iden = pathinfo[-2].get('iden') name = pathinfo[-1].get('name') - cell.drive.reqFreeStep(iden, name) + await cell.drive.reqFreeStep(iden, name) walks = [item async for item in cell.drive.walkPathInfo('hehe')] self.len(3, walks) @@ -454,10 +433,10 @@ async def migrate_v1(info, versinfo, data): self.eq('haha', walks[1].get('name')) self.eq('hehe', walks[2].get('name')) - self.none(cell.drive.getTypeSchema('newp')) + self.none(await cell.drive.getTypeSchema('newp')) - cell.drive.validators.pop('woot') - self.nn(cell.drive.getTypeValidator('woot')) + # cell.drive.validators.pop('woot') + # self.nn(cell.drive.getTypeValidator('woot')) # move to root dir pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop') @@ -472,7 +451,8 @@ async def migrate_v1(info, versinfo, data): # explicitly clear out the cache JsValidators, otherwise we get the cached, pre-msgpack # version of the validator, which will be correct and skip the point of this test. s_config._JsValidators.clear() - cell.drive.reqValidData('woot', data) + # FIXME + # await cell.drive.reqValidData('woot', data) async def test_cell_auth(self): @@ -568,30 +548,6 @@ async def test_cell_auth(self): self.true(await proxy.icando('foo', 'bar')) await self.asyncraises(s_exc.AuthDeny, proxy.icando('foo', 'newp')) - # happy path perms - await visi.addRule((True, ('hive:set', 'foo', 'bar'))) - await visi.addRule((True, ('hive:get', 'foo', 'bar'))) - await visi.addRule((True, ('hive:pop', 'foo', 'bar'))) - - val = await echo.setHiveKey(('foo', 'bar'), 'thefirstval') - self.eq(None, val) - - # check that we get the old val back - val = await echo.setHiveKey(('foo', 'bar'), 'wootisetit') - self.eq('thefirstval', val) - - val = await echo.getHiveKey(('foo', 'bar')) - self.eq('wootisetit', val) - - val = await echo.popHiveKey(('foo', 'bar')) - self.eq('wootisetit', val) - - val = await echo.setHiveKey(('foo', 'bar', 'baz'), 'a') - val = await echo.setHiveKey(('foo', 'bar', 'faz'), 'b') - val = await echo.setHiveKey(('foo', 'bar', 'haz'), 'c') - val = await echo.listHiveKey(('foo', 'bar')) - self.eq(('baz', 'faz', 'haz'), val) - # visi user can change visi user pass await proxy.setUserPasswd(visi.iden, 'foobar') # non admin visi user cannot change root user pass @@ -683,70 +639,20 @@ async def test_cell_auth(self): await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(visi_url)) - await echo.setHiveKey(('foo', 'bar'), [1, 2, 3, 4]) - self.eq([1, 2, 3, 4], await echo.getHiveKey(('foo', 'bar'))) - self.isin('foo', await echo.listHiveKey()) - self.eq(['bar'], await echo.listHiveKey(('foo',))) - await echo.popHiveKey(('foo', 'bar')) - self.eq([], await echo.listHiveKey(('foo',))) - # Ensure we can delete a rule by its item and index position async with echo.getLocalProxy() as proxy: # type: EchoAuthApi - rule = (True, ('hive:set', 'foo', 'bar')) + rule = (True, ('foo', 'bar')) self.isin(rule, visi.info.get('rules')) await proxy.delUserRule(visi.iden, rule) self.notin(rule, visi.info.get('rules')) # Removing a non-existing rule by *rule* has no consequence await proxy.delUserRule(visi.iden, rule) - rule = visi.info.get('rules')[0] - self.isin(rule, visi.info.get('rules')) - await proxy.delUserRule(visi.iden, rule) - self.notin(rule, visi.info.get('rules')) - self.eq(echo.getDmonUser(), echo.auth.rootuser.iden) with self.raises(s_exc.NeedConfValu): await echo.reqAhaProxy() - async def test_cell_drive_perm_migration(self): - async with self.getRegrCore('drive-perm-migr') as core: - item = await core.getDrivePath('driveitemdefaultperms') - self.len(1, item) - self.notin('perm', item) - self.eq(item[0]['permissions'], {'users': {}, 'roles': {}}) - - ldog = await core.auth.getRoleByName('littledog') - bdog = await core.auth.getRoleByName('bigdog') - - louis = await core.auth.getUserByName('lewis') - tim = await core.auth.getUserByName('tim') - mj = await core.auth.getUserByName('mj') - - item = await core.getDrivePath('permfolder/driveitemwithperms') - self.len(2, item) - self.notin('perm', item[0]) - self.notin('perm', item[1]) - self.eq(item[0]['permissions'], {'users': {tim.iden: s_cell.PERM_ADMIN}, 'roles': {}}) - self.eq(item[1]['permissions'], { - 'users': { - mj.iden: s_cell.PERM_ADMIN - }, - 'roles': { - ldog.iden: s_cell.PERM_READ, - bdog.iden: s_cell.PERM_EDIT, - }, - 'default': s_cell.PERM_DENY - }) - - # make sure it's all good with easy perms - self.true(core._hasEasyPerm(item[0], tim, s_cell.PERM_ADMIN)) - self.false(core._hasEasyPerm(item[0], mj, s_cell.PERM_EDIT)) - - self.true(core._hasEasyPerm(item[1], mj, s_cell.PERM_ADMIN)) - self.true(core._hasEasyPerm(item[1], tim, s_cell.PERM_READ)) - self.true(core._hasEasyPerm(item[1], louis, s_cell.PERM_EDIT)) - async def test_cell_unix_sock(self): async with self.getTestCore() as core: @@ -848,7 +754,7 @@ async def test_longpath(self): # but exercises the long-path failure inside of the cell's daemon # instead. with self.getTestDir() as dirn: - extrapath = 108 * 'A' + extrapath = s_const.UNIX_PATH_MAX * 'A' longdirn = s_common.genpath(dirn, extrapath) with self.getAsyncLoggerStream('synapse.lib.cell', 'LOCAL UNIX SOCKET WILL BE UNAVAILABLE') as stream: async with self.getTestCell(s_cell.Cell, dirn=longdirn) as cell: @@ -910,7 +816,6 @@ async def test_cell_getinfo(self): info = await prox.getCellInfo() # Cell information cnfo = info.get('cell') - nxfo = cnfo.get('nexus') snfo = info.get('synapse') self.eq(cnfo.get('commit'), 'mycommit') self.eq(cnfo.get('version'), (1, 2, 3)) @@ -920,15 +825,7 @@ async def test_cell_getinfo(self): self.ge(cnfo.get('nexsindx'), 0) self.true(cnfo.get('active')) self.false(cnfo.get('uplink')) - self.true(cnfo.get('ready')) self.none(cnfo.get('mirror', True)) - # Nexus info - self.ge(nxfo.get('indx'), 0) - self.false(nxfo.get('uplink:ready')) - self.true(nxfo.get('ready')) - self.false(nxfo.get('readonly')) - self.eq(nxfo.get('holds'), []) - # A Cortex populated cellvers self.isin('cortex:defaults', cnfo.get('cellvers', {})) @@ -947,21 +844,6 @@ async def test_cell_getinfo(self): https = netw.get('https') self.eq(https, http_info) - # Write hold information is reflected through cell info - await cell.nexsroot.addWriteHold('boop') - await cell.nexsroot.addWriteHold('beep') - info = await prox.getCellInfo() - nxfo = info.get('cell').get('nexus') - self.true(nxfo.get('readonly')) - self.eq(nxfo.get('holds'), [{'reason': 'beep'}, {'reason': 'boop'}]) - - await cell.nexsroot.delWriteHold('boop') - await cell.nexsroot.delWriteHold('beep') - info = await prox.getCellInfo() - nxfo = info.get('cell').get('nexus') - self.false(nxfo.get('readonly')) - self.eq(nxfo.get('holds'), []) - # Mirrors & ready flags async with self.getTestAha() as aha: # type: s_aha.AhaCell @@ -980,26 +862,18 @@ async def test_cell_getinfo(self): await cell01.sync() cnfo0 = await cell00.getCellInfo() - nxfo0 = cnfo0['cell']['nexus'] cnfo1 = await cell01.getCellInfo() - nxfo1 = cnfo1['cell']['nexus'] self.true(cnfo0['cell']['ready']) self.false(cnfo0['cell']['uplink']) self.none(cnfo0['cell']['mirror']) self.eq(cnfo0['cell']['version'], (1, 2, 3)) - self.false(nxfo0.get('uplink:ready')) - self.true(nxfo0.get('ready')) self.true(cnfo1['cell']['ready']) self.true(cnfo1['cell']['uplink']) self.eq(cnfo1['cell']['mirror'], 'aha://root@cell...') self.eq(cnfo1['cell']['version'], (1, 2, 3)) - self.true(nxfo1.get('uplink:ready')) - self.true(nxfo1.get('ready')) - self.eq(cnfo0['cell']['nexsindx'], cnfo1['cell']['nexsindx']) - self.eq(nxfo0['indx'], nxfo1['indx']) async def test_cell_dyncall(self): @@ -1065,7 +939,7 @@ async def coro(prox, offs): yielded = False async for offset, data in prox.getNexusChanges(offs): yielded = True - nexsiden, act, args, kwargs, meta = data + nexsiden, act, args, kwargs, meta, _ = data if nexsiden == 'auth:auth' and act == 'user:add': retn.append(args) break @@ -1073,7 +947,6 @@ async def coro(prox, offs): conf = { 'nexslog:en': True, - 'nexslog:async': True, 'dmon:listen': 'tcp://127.0.0.1:0/', 'https:port': 0, } @@ -1156,7 +1029,7 @@ async def test_cell_nexuscull(self): self.eq(0, await prox.trimNexsLog()) for i in range(5): - await prox.setHiveKey(('foo', 'bar'), i) + await cell.sync() ind = await prox.getNexsIndx() offs = await prox.rotateNexsLog() @@ -1185,7 +1058,7 @@ async def test_cell_nexuscull(self): self.eq('nexslog:cull', retn[0][1][1]) for i in range(6, 10): - await prox.setHiveKey(('foo', 'bar'), i) + await cell.sync() # trim ind = await prox.getNexsIndx() @@ -1198,7 +1071,7 @@ async def test_cell_nexuscull(self): self.eq(ind + 2, await prox.trimNexsLog()) for i in range(10, 15): - await prox.setHiveKey(('foo', 'bar'), i) + await cell.sync() # nexus log exists but logging is disabled conf['nexslog:en'] = False @@ -1238,8 +1111,8 @@ async def test_cell_nexusrotate(self): } async with await s_cell.Cell.anit(dirn, conf=conf) as cell: - await cell.setHiveKey(('foo', 'bar'), 0) - await cell.setHiveKey(('foo', 'bar'), 1) + await cell.sync() + await cell.sync() await cell.rotateNexsLog() @@ -1251,7 +1124,7 @@ async def test_cell_nexusrotate(self): self.len(2, cell.nexsroot.nexslog._ranges) self.eq(0, cell.nexsroot.nexslog.tailseqn.size) - await cell.setHiveKey(('foo', 'bar'), 2) + await cell.sync() # new item is added to the right log self.len(2, cell.nexsroot.nexslog._ranges) @@ -1316,7 +1189,6 @@ async def test_cell_diag_info(self): self.nn(slab['mapsize']) self.nn(slab['readonly']) self.nn(slab['readahead']) - self.nn(slab['lockmemory']) self.nn(slab['recovering']) async def test_cell_system_info(self): @@ -1345,17 +1217,6 @@ async def test_cell_system_info(self): 'cellapprdisk', 'totalmem', 'availmem'): self.lt(0, info.get(prop)) - async def test_cell_hiveapi(self): - - async with self.getTestCell() as cell: - - await cell.setHiveKey(('foo', 'bar'), 10) - await cell.setHiveKey(('foo', 'baz'), 30) - - async with cell.getLocalProxy() as proxy: - self.eq((), await proxy.getHiveKeys(('lulz',))) - self.eq((('bar', 10), ('baz', 30)), await proxy.getHiveKeys(('foo',))) - async def test_cell_confprint(self): async with self.withSetLoggingMock(): @@ -1377,7 +1238,7 @@ async def test_cell_confprint(self): self.isin('...cell API (https): 0', buf) conf = { - 'dmon:listen': 'tcp://0.0.0.0:0', + 'dmon:listen': None, 'https:port': None, } s_common.yamlsave(conf, dirn, 'cell.yaml') @@ -1387,22 +1248,21 @@ async def test_cell_confprint(self): pass stream.seek(0) buf = stream.read() - self.isin('...cell API (telepath): tcp://0.0.0.0:0', buf) + self.isin(f'...cell API (telepath): tcp://0.0.0.0:27492', buf) self.isin('...cell API (https): disabled', buf) async def test_cell_initargv_conf(self): async with self.withSetLoggingMock(): with self.setTstEnvars(SYN_CELL_NEXSLOG_EN='true', - SYN_CELL_DMON_LISTEN='tcp://0.0.0.0:0', + SYN_CELL_DMON_LISTEN='null', SYN_CELL_HTTPS_PORT='null', SYN_CELL_AUTH_PASSWD='notsecret', ): with self.getTestDir() as dirn: - s_common.yamlsave({'dmon:listen': 'tcp://0.0.0.0:12345/', + s_common.yamlsave({'dmon:listen': 'tcp://0.0.0.0:0/', 'aha:name': 'some:cell'}, dirn, 'cell.yaml') - s_common.yamlsave({'nexslog:async': True}, - dirn, 'cell.mods.yaml') + s_common.yamlsave({}, dirn, 'cell.mods.yaml') async with await s_cell.Cell.initFromArgv([dirn, '--auth-passwd', 'secret']) as cell: # config order for booting from initArgV # 0) cell.mods.yaml @@ -1410,8 +1270,7 @@ async def test_cell_initargv_conf(self): # 2) envars # 3) cell.yaml self.true(cell.conf.req('nexslog:en')) - self.true(cell.conf.req('nexslog:async')) - self.eq(cell.conf.req('dmon:listen'), 'tcp://0.0.0.0:0') + self.none(cell.conf.req('dmon:listen')) self.none(cell.conf.req('https:port')) self.eq(cell.conf.req('aha:name'), 'some:cell') root = cell.auth.rootuser @@ -1483,11 +1342,11 @@ async def test_cell_backup(self): self.none(info['lastexception']) with self.raises(s_exc.BadArg): - await proxy.runBackup('../woot') + await proxy.runBackup(name='../woot') with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 0.1): with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeperProc)): - await self.asyncraises(s_exc.SynErr, proxy.runBackup('_sleeperProc')) + await self.asyncraises(s_exc.SynErr, proxy.runBackup(name='_sleeperProc')) info = await proxy.getBackupInfo() errinfo = info.get('lastexception') @@ -1499,7 +1358,7 @@ async def test_cell_backup(self): with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 8.0): with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeper2Proc)): - await self.asyncraises(s_exc.SynErr, proxy.runBackup('_sleeper2Proc')) + await self.asyncraises(s_exc.SynErr, proxy.runBackup(name='_sleeper2Proc')) info = await proxy.getBackupInfo() laststart2 = info['laststart'] @@ -1509,7 +1368,7 @@ async def test_cell_backup(self): self.eq(errinfo['errinfo']['mesg'], 'backup subprocess start timed out') with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_exiterProc)): - await self.asyncraises(s_exc.SpawnExit, proxy.runBackup('_exiterProc')) + await self.asyncraises(s_exc.SpawnExit, proxy.runBackup(name='_exiterProc')) info = await proxy.getBackupInfo() laststart3 = info['laststart'] @@ -1607,7 +1466,7 @@ async def err(*args, **kwargs): with mock.patch('synapse.lib.coro.executor', err): with self.raises(s_exc.SynErr) as cm: - await proxy.runBackup('partial') + await proxy.runBackup(name='partial') self.eq(cm.exception.get('errx'), 'RuntimeError') self.isin('partial', await proxy.getBackups()) @@ -1659,19 +1518,6 @@ async def test_cell_tls_client(self): async with await s_telepath.openurl(url) as proxy: pass - async def test_cell_auth_ctor(self): - conf = { - 'auth:ctor': 'synapse.tests.test_lib_cell.altAuthCtor', - 'auth:conf': { - 'foo': 'bar', - }, - } - with self.getTestDir() as dirn: - async with await s_cell.Cell.anit(dirn, conf=conf) as cell: - self.eq('faz', cell.conf.get('auth:conf')['baz']) - await cell.auth.addUser('visi') - await cell._storCellAuthMigration() - async def test_cell_auth_userlimit(self): maxusers = 3 conf = { @@ -1896,7 +1742,7 @@ async def _fakeBackup(self, name=None, wait=True): s_common.gendir(os.path.join(backdirn, name)) with mock.patch.object(s_cell.Cell, 'runBackup', _fakeBackup): - arch = s_t_utils.alist(proxy.iterNewBackupArchive('nobkup')) + arch = s_t_utils.alist(proxy.iterNewBackupArchive(name='nobkup')) with self.raises(asyncio.TimeoutError): await asyncio.wait_for(arch, timeout=0.1) @@ -1905,7 +1751,7 @@ async def _slowFakeBackup(self, name=None, wait=True): await asyncio.sleep(3.0) with mock.patch.object(s_cell.Cell, 'runBackup', _slowFakeBackup): - arch = s_t_utils.alist(proxy.iterNewBackupArchive('nobkup2')) + arch = s_t_utils.alist(proxy.iterNewBackupArchive(name='nobkup2')) with self.raises(asyncio.TimeoutError): await asyncio.wait_for(arch, timeout=0.1) @@ -1927,17 +1773,17 @@ async def _iterNewDup(self, user, name=None, remove=False): with mock.patch.object(s_cell.Cell, 'runBackup', _slowFakeBackup2): with mock.patch.object(s_cell.Cell, 'iterNewBackupArchive', _iterNewDup): - arch = s_t_utils.alist(proxy.iterNewBackupArchive('dupbackup', remove=True)) + arch = s_t_utils.alist(proxy.iterNewBackupArchive(name='dupbackup', remove=True)) task = core.schedCoro(arch) await asyncio.wait_for(evt0.wait(), timeout=2) - fail = s_t_utils.alist(proxy.iterNewBackupArchive('alreadystreaming', remove=True)) + fail = s_t_utils.alist(proxy.iterNewBackupArchive(name='alreadystreaming', remove=True)) await self.asyncraises(s_exc.BackupAlreadyRunning, fail) task.cancel() await asyncio.wait_for(evt1.wait(), timeout=2) with self.raises(s_exc.BadArg): - async for msg in proxy.iterNewBackupArchive('bkup'): + async for msg in proxy.iterNewBackupArchive(name='bkup'): pass # Get an existing backup @@ -1950,7 +1796,7 @@ async def _iterNewDup(self, user, name=None, remove=False): self.len(1, nodes) with open(bkuppath2, 'wb') as bkup2: - async for msg in proxy.iterNewBackupArchive('bkup2'): + async for msg in proxy.iterNewBackupArchive(name='bkup2'): bkup2.write(msg) self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups())) @@ -1961,7 +1807,7 @@ async def _iterNewDup(self, user, name=None, remove=False): self.len(1, nodes) with open(bkuppath3, 'wb') as bkup3: - async for msg in proxy.iterNewBackupArchive('bkup3', remove=True): + async for msg in proxy.iterNewBackupArchive(name='bkup3', remove=True): self.true(core.backupstreaming) bkup3.write(msg) @@ -1995,16 +1841,16 @@ async def streamdone(): self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups())) # Start another backup while one is already running - bkup = s_t_utils.alist(proxy.iterNewBackupArchive('runbackup', remove=True)) + bkup = s_t_utils.alist(proxy.iterNewBackupArchive(name='runbackup', remove=True)) task = core.schedCoro(bkup) await asyncio.sleep(0) - fail = s_t_utils.alist(proxy.iterNewBackupArchive('alreadyrunning', remove=True)) + fail = s_t_utils.alist(proxy.iterNewBackupArchive(name='alreadyrunning', remove=True)) await self.asyncraises(s_exc.BackupAlreadyRunning, fail) await asyncio.wait_for(task, 5) with tarfile.open(bkuppath, 'r:gz') as tar: - tar.extractall(path=dirn) + tar.extractall(path=dirn, filter='data') bkupdirn = os.path.join(dirn, 'bkup') async with self.getTestCore(dirn=bkupdirn) as core: @@ -2015,7 +1861,7 @@ async def streamdone(): self.len(0, nodes) with tarfile.open(bkuppath2, 'r:gz') as tar: - tar.extractall(path=dirn) + tar.extractall(path=dirn, filter='data') bkupdirn2 = os.path.join(dirn, 'bkup2') async with self.getTestCore(dirn=bkupdirn2) as core: @@ -2023,7 +1869,7 @@ async def streamdone(): self.len(1, nodes) with tarfile.open(bkuppath3, 'r:gz') as tar: - tar.extractall(path=dirn) + tar.extractall(path=dirn, filter='data') bkupdirn3 = os.path.join(dirn, 'bkup3') async with self.getTestCore(dirn=bkupdirn3) as core: @@ -2032,7 +1878,7 @@ async def streamdone(): with tarfile.open(bkuppath4, 'r:gz') as tar: bkupname = os.path.commonprefix(tar.getnames()) - tar.extractall(path=dirn) + tar.extractall(path=dirn, filter='data') bkupdirn4 = os.path.join(dirn, bkupname) async with self.getTestCore(dirn=bkupdirn4) as core: @@ -2051,11 +1897,11 @@ async def streamdone(): bkup5.write(msg) with mock.patch('synapse.lib.cell._iterBackupProc', _backupEOF): - await s_t_utils.alist(proxy.iterNewBackupArchive('eof', remove=True)) + await s_t_utils.alist(proxy.iterNewBackupArchive(name='eof', remove=True)) with tarfile.open(bkuppath5, 'r:gz') as tar: bkupname = os.path.commonprefix(tar.getnames()) - tar.extractall(path=dirn) + tar.extractall(path=dirn, filter='data') bkupdirn5 = os.path.join(dirn, bkupname) async with self.getTestCore(dirn=bkupdirn5) as core: @@ -2233,7 +2079,7 @@ async def test_backup_restore_base(self): # Make our first backup async with self.getTestCore() as core: - self.len(1, await core.nodes('[inet:ipv4=1.2.3.4]')) + self.len(1, await core.nodes('[inet:ip=1.2.3.4]')) # Punch in a value to the cell.yaml to ensure it persists core.conf['storm:log'] = True @@ -2261,14 +2107,14 @@ async def test_backup_restore_base(self): argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: self.true(await stream.wait(6)) - self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) + self.len(1, await core.nodes('inet:ip=1.2.3.4')) self.true(core.conf.get('storm:log')) # Turning the service back on with the restore URL is fine too. with self.getAsyncLoggerStream('synapse.lib.cell') as stream: argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: - self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) + self.len(1, await core.nodes('inet:ip=1.2.3.4')) # Take a backup of the cell with the restore.done file in place async with await axon.upload() as upfd: @@ -2298,7 +2144,7 @@ async def test_backup_restore_base(self): argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: self.true(await stream.wait(6)) - self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) + self.len(1, await core.nodes('inet:ip=1.2.3.4')) # Restore a backup which has an existing restore.done file in it - that marker file will get overwritten furl2 = f'{url}{s_common.ehex(sha256r)}' @@ -2310,7 +2156,7 @@ async def test_backup_restore_base(self): argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: self.true(await stream.wait(6)) - self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) + self.len(1, await core.nodes('inet:ip=1.2.3.4')) rpath = s_common.genpath(cdir, 'restore.done') with s_common.genfile(rpath) as fd: @@ -2526,73 +2372,6 @@ async def test_backup_restore_double_promote_aha(self): self.len(1, await bcree01.nodes('[inet:asn=8675]')) self.len(1, await bcree00.nodes('inet:asn=8675')) - async def test_passwd_regression(self): - # Backwards compatibility test for shadowv2 - # Cell was created prior to the shadowv2 password change. - with self.getRegrDir('cells', 'passwd-2.109.0') as dirn: - async with self.getTestCell(s_cell.Cell, dirn=dirn) as cell: # type: s_cell.Cell - root = await cell.auth.getUserByName('root') - shadow = root.info.get('passwd') - self.isinstance(shadow, tuple) - self.len(2, shadow) - - # Old password works and is migrated to the new password scheme - self.false(await root.tryPasswd('newp')) - self.true(await root.tryPasswd('root')) - shadow = root.info.get('passwd') - self.isinstance(shadow, dict) - self.eq(shadow.get('type'), s_passwd.DEFAULT_PTYP) - - # Logging back in works - self.true(await root.tryPasswd('root')) - - user = await cell.auth.getUserByName('user') - - # User can login with their regular password. - shadow = user.info.get('passwd') - self.isinstance(shadow, tuple) - self.true(await user.tryPasswd('secret1234')) - shadow = user.info.get('passwd') - self.isinstance(shadow, dict) - - # User has a 10 year duration onepass value available. - onepass = '0f327906fe0221a7f582744ad280e1ca' - self.true(await user.tryPasswd(onepass)) - self.false(await user.tryPasswd(onepass)) - - # Passwords can be changed as well. - await user.setPasswd('hehe') - self.true(await user.tryPasswd('hehe')) - self.false(await user.tryPasswd('secret1234')) - - # Password policies do not prevent live migration of an existing password - with self.getRegrDir('cells', 'passwd-2.109.0') as dirn: - policy = {'complexity': {'length': 5}} - conf = {'auth:passwd:policy': policy} - async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell: # type: s_cell.Cell - root = await cell.auth.getUserByName('root') - shadow = root.info.get('passwd') - self.isinstance(shadow, tuple) - self.len(2, shadow) - - # Old password works and is migrated to the new password scheme - self.false(await root.tryPasswd('newp')) - self.true(await root.tryPasswd('root')) - shadow = root.info.get('passwd') - self.isinstance(shadow, dict) - self.eq(shadow.get('type'), s_passwd.DEFAULT_PTYP) - - # Pre-nexus changes of root via auth:passwd work too. - with self.getRegrDir('cells', 'passwd-2.109.0') as dirn: - conf = {'auth:passwd': 'supersecretpassword'} - async with self.getTestCell(s_cell.Cell, dirn=dirn, conf=conf) as cell: # type: s_cell.Cell - root = await cell.auth.getUserByName('root') - shadow = root.info.get('passwd') - self.isinstance(shadow, dict) - self.eq(shadow.get('type'), s_passwd.DEFAULT_PTYP) - self.false(await root.tryPasswd('root')) - self.true(await root.tryPasswd('supersecretpassword')) - async def test_cell_minspace(self): with self.raises(s_exc.LowSpace): @@ -2658,7 +2437,7 @@ async def wrapDelWriteHold(root, reason): conf = {'limit:disk:free': 0} async with self.getTestCore(dirn=path00, conf=conf) as core00: - await core00.nodes('[ inet:ipv4=1.2.3.4 ]') + await core00.nodes('[ inet:ip=1.2.3.4 ]') s_tools_backup.backup(path00, path01) @@ -2678,21 +2457,21 @@ async def wrapDelWriteHold(root, reason): self.stormIsInErr(errmsg, msgs) msgs = await core01.stormlist('[inet:fqdn=newp.fail]') self.stormIsInErr(errmsg, msgs) - self.len(1, await core00.nodes('[ inet:ipv4=2.3.4.5 ]')) + self.len(1, await core00.nodes('[ inet:ip=2.3.4.5 ]')) offs = await core00.getNexsIndx() self.false(await core01.waitNexsOffs(offs, 1)) - self.len(1, await core01.nodes('inet:ipv4=1.2.3.4')) - self.len(0, await core01.nodes('inet:ipv4=2.3.4.5')) + self.len(1, await core01.nodes('inet:ip=1.2.3.4')) + self.len(0, await core01.nodes('inet:ip=2.3.4.5')) revt.clear() revt.clear() self.true(await asyncio.wait_for(revt.wait(), 1)) await core01.sync() - self.len(1, await core01.nodes('inet:ipv4=1.2.3.4')) - self.len(1, await core01.nodes('inet:ipv4=2.3.4.5')) + self.len(1, await core01.nodes('inet:ip=1.2.3.4')) + self.len(1, await core01.nodes('inet:ip=2.3.4.5')) with mock.patch.object(s_cell.Cell, 'FREE_SPACE_CHECK_FREQ', 600): @@ -2706,9 +2485,9 @@ async def wrapDelWriteHold(root, reason): with mock.patch('shutil.disk_usage', full_disk): opts = {'view': viewiden} - msgs = await core.stormlist('for $x in $lib.range(20000) {[inet:ipv4=$x]}', opts=opts) + msgs = await core.stormlist('for $x in $lib.range(20000) {[inet:ip=([4, $x])]}', opts=opts) self.stormIsInErr(errmsg, msgs) - nodes = await core.nodes('inet:ipv4', opts=opts) + nodes = await core.nodes('inet:ip', opts=opts) self.gt(len(nodes), 0) self.lt(len(nodes), 20000) @@ -2726,7 +2505,7 @@ def spaceexc(self): opts = {'view': viewiden} with self.getAsyncLoggerStream('synapse.lib.lmdbslab', - 'Error during slab resize callback - foo') as stream: + 'Error during slab resize callback - foo') as stream: msgs = await core.stormlist('for $x in $lib.range(200) {[test:int=$x]}', opts=opts) self.true(await stream.wait(timeout=30)) @@ -3050,8 +2829,8 @@ async def test_cell_user_api_key(self): # Verify duration arg for expiration is applied with self.raises(s_exc.BadArg): await cell.addUserApiKey(root, 'newp', duration=0) - rtk1, rtdf1 = await cell.addUserApiKey(root, 'Expiring Token', duration=200) - self.eq(rtdf1.get('expires'), rtdf1.get('updated') + 200) + rtk1, rtdf1 = await cell.addUserApiKey(root, 'Expiring Token', duration=200000) + self.eq(rtdf1.get('expires'), rtdf1.get('updated') + 200000) isok, info = await cell.checkUserApiKey(rtk1) self.true(isok) @@ -3202,105 +2981,6 @@ async def test_cell_iter_slab_data(self): data = await s_t_utils.alist(cell.iterSlabData('hehe', prefix='yup')) self.eq(data, [('wow', 'yes')]) - async def test_cell_nexus_compat(self): - with mock.patch('synapse.lib.cell.NEXUS_VERSION', (0, 0)): - async with self.getRegrCore('hive-migration') as core0: - with mock.patch('synapse.lib.cell.NEXUS_VERSION', (2, 177)): - conf = {'mirror': core0.getLocalUrl()} - async with self.getRegrCore('hive-migration', conf=conf) as core1: - await core1.sync() - - await core1.nodes('$lib.user.vars.set(foo, bar)') - self.eq('bar', await core0.callStorm('return($lib.user.vars.get(foo))')) - - await core1.nodes('$lib.user.vars.pop(foo)') - self.none(await core0.callStorm('return($lib.user.vars.get(foo))')) - - await core1.nodes('$lib.user.profile.set(bar, baz)') - self.eq('baz', await core0.callStorm('return($lib.user.profile.get(bar))')) - - await core1.nodes('$lib.user.profile.pop(bar)') - self.none(await core0.callStorm('return($lib.user.profile.get(bar))')) - - self.eq((0, 0), core1.nexsvers) - await core0.setNexsVers((2, 177)) - await core1.sync() - self.eq((2, 177), core1.nexsvers) - - await core1.nodes('$lib.user.vars.set(foo, bar)') - self.eq('bar', await core0.callStorm('return($lib.user.vars.get(foo))')) - - await core1.nodes('$lib.user.vars.pop(foo)') - self.none(await core0.callStorm('return($lib.user.vars.get(foo))')) - - await core1.nodes('$lib.user.profile.set(bar, baz)') - self.eq('baz', await core0.callStorm('return($lib.user.profile.get(bar))')) - - await core1.nodes('$lib.user.profile.pop(bar)') - self.none(await core0.callStorm('return($lib.user.profile.get(bar))')) - - async def test_cell_hive_migration(self): - - with self.getAsyncLoggerStream('synapse.lib.cell') as stream: - - async with self.getRegrCore('hive-migration') as core: - visi = await core.auth.getUserByName('visi') - asvisi = {'user': visi.iden} - - valu = await core.callStorm('return($lib.user.vars.get(foovar))', opts=asvisi) - self.eq('barvalu', valu) - - valu = await core.callStorm('return($lib.user.profile.get(fooprof))', opts=asvisi) - self.eq('barprof', valu) - - msgs = await core.stormlist('cron.list') - self.stormIsInPrint(' visi 8437c35a.. ', msgs) - self.stormIsInPrint('[tel:mob:telem=*]', msgs) - - msgs = await core.stormlist('dmon.list') - self.stormIsInPrint('0973342044469bc40b577969028c5079: (foodmon ): running', msgs) - - msgs = await core.stormlist('trigger.list') - self.stormIsInPrint('visi 27f5dc524e7c3ee8685816ddf6ca1326', msgs) - self.stormIsInPrint('[ +#count test:str=$tag ]', msgs) - - msgs = await core.stormlist('testcmd0 foo') - self.stormIsInPrint('foo haha', msgs) - - msgs = await core.stormlist('testcmd1') - self.stormIsInPrint('hello', msgs) - - msgs = await core.stormlist('model.deprecated.locks') - self.stormIsInPrint('ou:hasalias', msgs) - - nodes = await core.nodes('_visi:int') - self.len(1, nodes) - node = nodes[0] - self.eq(node.get('tick'), 1577836800000,) - self.eq(node.get('._woot'), 5) - self.nn(node.getTagProp('test', 'score'), 6) - - self.maxDiff = None - roles = s_t_utils.deguidify('[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}]') - users = s_t_utils.deguidify('[{"type":"user","iden":"a357138db50780b62093a6ce0d057fd8","name":"root","rules":[],"roles":[],"admin":true,"email":null,"locked":false,"archived":false,"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"admin":true},"4d50eb257549436414643a71e057091a":{"admin":true}}},{"type":"user","iden":"f77ac6744671a845c27e571071877827","name":"visi","rules":[[true,["cron","add"]],[true,["dmon","add"]],[true,["trigger","add"]]],"roles":[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}],"admin":false,"email":null,"locked":false,"archived":false,"authgates":{"f21b7ae79c2dacb89484929a8409e5d8":{"admin":true},"d7d0380dd4e743e35af31a20d014ed48":{"admin":true}}}]') - gates = s_t_utils.deguidify('[{"iden":"46cfde2c1682566602860f8df7d0cc83","type":"layer","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["layer","read"]]],"admin":false}]},{"iden":"d7d0380dd4e743e35af31a20d014ed48","type":"trigger","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"f21b7ae79c2dacb89484929a8409e5d8","type":"cronjob","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"4d50eb257549436414643a71e057091a","type":"view","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["view","read"]]],"admin":false}]},{"iden":"cortex","type":"cortex","users":[],"roles":[]}]') - - self.eq(roles, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.roles.list())')).decode())) - self.eq(users, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.users.list())')).decode())) - self.eq(gates, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.gates.list())')).decode())) - - with self.raises(s_exc.BadTypeValu): - await core.nodes('[ it:dev:str=foo +#test.newp ]') - - stream.seek(0) - data = stream.getvalue() - newprole = s_common.guid('newprole') - newpuser = s_common.guid('newpuser') - - self.isin(f'Unknown user {newpuser} on gate', data) - self.isin(f'Unknown role {newprole} on gate', data) - self.isin(f'Unknown role {newprole} on user', data) - async def test_cell_check_sysctl(self): sysctls = s_linux.getSysctls() @@ -3524,11 +3204,15 @@ async def test_lib_cell_sadaha(self): async with self.getTestCell() as cell: self.none(await cell.getAhaProxy()) - cell.ahaclient = await cell.enter_context(await s_telepath.Client.anit('cell:///tmp/newp')) - # coverage for failure of aha client to connect - with self.raises(TimeoutError): - self.none(await cell.getAhaProxy(timeout=0.1)) + class MockClient: + async def proxy(self, timeout=None): + raise s_exc.LinkShutDown(mesg='client connection failed') + + cell.ahaclient = MockClient() + + with self.raises(s_exc.LinkShutDown): + self.none(await cell.getAhaProxy()) async def test_stream_backup_exception(self): @@ -3577,7 +3261,7 @@ async def mock_runBackup(*args, **kwargs): with mock.patch.object(s_cell.Cell, 'runBackup', mock_runBackup): with self.getAsyncLoggerStream('synapse.lib.cell', 'Removing') as stream: with self.raises(s_exc.SynErr) as cm: - async for _ in proxy.iterNewBackupArchive('failedbackup', remove=True): + async for _ in proxy.iterNewBackupArchive(name='failedbackup', remove=True): pass self.isin('backup failed', str(cm.exception)) @@ -3590,7 +3274,7 @@ async def mock_runBackup(*args, **kwargs): core.backupstreaming = True with self.raises(s_exc.BackupAlreadyRunning): - async for _ in proxy.iterNewBackupArchive('newbackup', remove=True): + async for _ in proxy.iterNewBackupArchive(name='newbackup', remove=True): pass async def test_cell_peer_noaha(self): @@ -3644,47 +3328,3 @@ async def sleep99(cell): self.none(await cell00.getTask(task01)) self.false(await cell00.killTask(task01)) - - async def test_cell_fini_order(self): - - with self.getTestDir() as dirn: - - data = [] - conf = {'nexslog:en': True} - - async with self.getTestCell(dirn=dirn, conf=conf) as cell: - - event00 = asyncio.Event() - - async def coro(): - try: - event00.set() - await asyncio.sleep(100000) - except asyncio.CancelledError: - # nexus txn can run in a activeTask handler - await cell.sync() - data.append('activetask_cancelled') - - async def bg_coro(): - self.true(await cell.waitfini(timeout=12)) - data.append('cell_fini') - return True - - bg_task = s_coro.create_task(bg_coro()) - cell.runActiveTask(coro()) - self.true(await asyncio.wait_for(event00.wait(), timeout=6)) - - # Perform a non-sync nexus txn then teardown the cell via __aexit__ - self.nn(await cell.addUser('someuser')) - - self.true(await asyncio.wait_for(bg_task, timeout=6)) - - self.eq(data, ['activetask_cancelled', 'cell_fini']) - - async with self.getTestCell(dirn=dirn, conf=conf) as cell: - offs = await cell.getNexsIndx() - items = [] - async for offs, item in cell.getNexusChanges(offs - 1, wait=False): - items.append(item) - self.len(1, items) - self.eq('sync', items[0][1]) From de0542505905dd6f6c17bf9677bf2f482acc7978 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:16:29 -0500 Subject: [PATCH 02/15] updates --- synapse/lib/base.py | 30 +- synapse/lib/cell.py | 598 +++++++++++++++++++++++++-------- synapse/lib/const.py | 4 +- synapse/lib/spawner.py | 4 +- synapse/tests/test_lib_base.py | 54 +-- synapse/tests/test_lib_cell.py | 518 +++++++++++++++++++++++----- 6 files changed, 913 insertions(+), 295 deletions(-) diff --git a/synapse/lib/base.py b/synapse/lib/base.py index 2ca48abd6b0..0dd73850ab9 100644 --- a/synapse/lib/base.py +++ b/synapse/lib/base.py @@ -41,7 +41,7 @@ def _fini_atexit(): # pragma: no cover if __debug__: logger.debug(f'At exit: Missing fini for {item}') for depth, call in enumerate(item.call_stack[:-2]): - logger.debug(f'{depth + 1:3}: {call.strip()}') + logger.debug(f'{depth+1:3}: {call.strip()}') continue try: @@ -347,12 +347,16 @@ async def dist(self, mesg): try: ret.append(await s_coro.ornot(func, mesg)) + except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only + raise except Exception: logger.exception('base %s error with mesg %s', self, mesg) for func in self._syn_links: try: ret.append(await s_coro.ornot(func, mesg)) + except asyncio.CancelledError: # pragma: no cover TODO: remove once >= py 3.8 only + raise except Exception: logger.exception('base %s error with mesg %s', self, mesg) @@ -396,6 +400,8 @@ async def fini(self): for base in list(self.tofini): await base.fini() + # Do not continue to hold a reference to the last item we iterated on. + base = None # NOQA await self._kill_active_tasks() @@ -449,25 +455,9 @@ def onWith(self, evnt, func): finally: self.off(evnt, func) - @contextlib.contextmanager - def onWithMulti(self, evnts, func): - ''' - A context manager which can be used to add a callbacks and remove them when - using a ``with`` statement. - - Args: - evnts (list): A list of event names - func (function): A callback function to receive event tufo - ''' - for evnt in evnts: - self.on(evnt, func) - # Allow exceptions to propagate during the context manager - # but ensure we cleanup our temporary callback - try: - yield self - finally: - for evnt in evnts: - self.off(evnt, func) + def _wouldfini(self): + '''Check if a Base would be fini() if fini() was called on it.''' + return self._syn_refs == 1 async def waitfini(self, timeout=None): ''' diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 22511d75bad..d7cda728924 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -2,7 +2,6 @@ import os import ssl import copy -import stat import time import fcntl import shutil @@ -34,6 +33,7 @@ import synapse.lib.base as s_base import synapse.lib.boss as s_boss import synapse.lib.coro as s_coro +import synapse.lib.hive as s_hive import synapse.lib.link as s_link import synapse.lib.task as s_task import synapse.lib.cache as s_cache @@ -55,6 +55,7 @@ import synapse.lib.version as s_version import synapse.lib.lmdbslab as s_lmdbslab import synapse.lib.thisplat as s_thisplat +import synapse.lib.processpool as s_processpool import synapse.lib.crypto.passwd as s_passwd @@ -148,7 +149,8 @@ async def _doIterBackup(path, chunksize=1024): link0, file1 = await s_link.linkfile() def dowrite(fd): - with tarfile.open(output_filename, 'w|gz', fileobj=fd, compresslevel=1) as tar: + # TODO: When we are 3.12+ convert this back to w|gz - see https://github.com/python/cpython/pull/2962 + with tarfile.open(output_filename, 'w:gz', fileobj=fd, compresslevel=1) as tar: tar.add(path, arcname=os.path.basename(path)) fd.close() @@ -209,18 +211,18 @@ async def initCellApi(self): pass @adminapi(log=True) - async def shutdown(self, *, timeout=None): + async def shutdown(self, timeout=None): return await self.cell.shutdown(timeout=timeout) @adminapi(log=True) - async def freeze(self, *, timeout=30): + async def freeze(self, timeout=30): return await self.cell.freeze(timeout=timeout) @adminapi(log=True) async def resume(self): return await self.cell.resume() - async def allowed(self, perm, *, default=None): + async def allowed(self, perm, default=None): ''' Check if the user has the requested permission. @@ -344,7 +346,7 @@ async def rotateNexsLog(self): return await self.cell.rotateNexsLog() @adminapi(log=True) - async def trimNexsLog(self, *, consumers=None, timeout=60): + async def trimNexsLog(self, consumers=None, timeout=60): ''' Rotate and cull the Nexus log (and those of any consumers) at the current offset. @@ -364,7 +366,7 @@ async def trimNexsLog(self, *, consumers=None, timeout=60): return await self.cell.trimNexsLog(consumers=consumers, timeout=timeout) @adminapi() - async def waitNexsOffs(self, offs, *, timeout=None): + async def waitNexsOffs(self, offs, timeout=None): ''' Wait for the Nexus log to write an offset. @@ -378,11 +380,11 @@ async def waitNexsOffs(self, offs, *, timeout=None): return await self.cell.waitNexsOffs(offs, timeout=timeout) @adminapi(log=True) - async def promote(self, *, graceful=False): + async def promote(self, graceful=False): return await self.cell.promote(graceful=graceful) @adminapi(log=True) - async def handoff(self, turl, *, timeout=30): + async def handoff(self, turl, timeout=30): return await self.cell.handoff(turl, timeout=timeout) @adminapi(log=True) @@ -406,7 +408,7 @@ async def getSystemInfo(self): - volfree - Volume where cell is running free space - backupvolsize - Backup directory volume total space - backupvolfree - Backup directory volume free space - - celluptime - Cell uptime in microseconds + - celluptime - Cell uptime in milliseconds - cellrealdisk - Cell's use of disk, equivalent to du - cellapprdisk - Cell's apparent use of disk, equivalent to ls -l - osversion - OS version/architecture @@ -446,16 +448,16 @@ async def kill(self, iden): return await self.cell.kill(self.user, iden) @adminapi() - async def getTasks(self, *, peers=True, timeout=None): + async def getTasks(self, peers=True, timeout=None): async for task in self.cell.getTasks(peers=peers, timeout=timeout): yield task @adminapi() - async def getTask(self, iden, *, peers=True, timeout=None): + async def getTask(self, iden, peers=True, timeout=None): return await self.cell.getTask(iden, peers=peers, timeout=timeout) @adminapi() - async def killTask(self, iden, *, peers=True, timeout=None): + async def killTask(self, iden, peers=True, timeout=None): return await self.cell.killTask(iden, peers=peers, timeout=timeout) @adminapi(log=True) @@ -467,7 +469,7 @@ async def behold(self): yield mesg @adminapi(log=True) - async def addUser(self, name, *, passwd=None, email=None, iden=None): + async def addUser(self, name, passwd=None, email=None, iden=None): return await self.cell.addUser(name, passwd=passwd, email=email, iden=iden) @adminapi(log=True) @@ -475,14 +477,14 @@ async def delUser(self, iden): return await self.cell.delUser(iden) @adminapi(log=True) - async def addRole(self, name, *, iden=None): + async def addRole(self, name, iden=None): return await self.cell.addRole(name, iden=iden) @adminapi(log=True) async def delRole(self, iden): return await self.cell.delRole(iden) - async def addUserApiKey(self, name, *, duration=None, useriden=None): + async def addUserApiKey(self, name, duration=None, useriden=None): if useriden is None: useriden = self.user.iden @@ -493,7 +495,7 @@ async def addUserApiKey(self, name, *, duration=None, useriden=None): return await self.cell.addUserApiKey(useriden, name, duration=duration) - async def listUserApiKeys(self, *, useriden=None): + async def listUserApiKeys(self, useriden=None): if useriden is None: useriden = self.user.iden @@ -520,16 +522,16 @@ async def delUserApiKey(self, iden): return await self.cell.delUserApiKey(iden) @adminapi() - async def dyncall(self, iden, todo, *, gatekeys=()): + async def dyncall(self, iden, todo, gatekeys=()): return await self.cell.dyncall(iden, todo, gatekeys=gatekeys) @adminapi() - async def dyniter(self, iden, todo, *, gatekeys=()): + async def dyniter(self, iden, todo, gatekeys=()): async for item in self.cell.dyniter(iden, todo, gatekeys=gatekeys): yield item @adminapi() - async def issue(self, nexsiden: str, event: str, args, kwargs, *, meta=None, wait=True): + async def issue(self, nexsiden: str, event: str, args, kwargs, meta=None, wait=True): return await self.cell.nexsroot.issue(nexsiden, event, args, kwargs, meta, wait=wait) @adminapi(log=True) @@ -546,7 +548,7 @@ async def delAuthRole(self, name): await self.cell.auth.delRole(name) @adminapi() - async def getAuthUsers(self, *, archived=False): + async def getAuthUsers(self, archived=False): ''' Args: archived (bool): If true, list all users, else list non-archived users @@ -558,33 +560,76 @@ async def getAuthRoles(self): return await self.cell.getAuthRoles() @adminapi(log=True) - async def addUserRule(self, iden, rule, *, indx=None, gateiden=None): + async def addUserRule(self, iden, rule, indx=None, gateiden=None): return await self.cell.addUserRule(iden, rule, indx=indx, gateiden=gateiden) @adminapi(log=True) - async def setUserRules(self, iden, rules, *, gateiden=None): + async def setUserRules(self, iden, rules, gateiden=None): return await self.cell.setUserRules(iden, rules, gateiden=gateiden) @adminapi(log=True) - async def setRoleRules(self, iden, rules, *, gateiden=None): + async def setRoleRules(self, iden, rules, gateiden=None): return await self.cell.setRoleRules(iden, rules, gateiden=gateiden) @adminapi(log=True) - async def addRoleRule(self, iden, rule, *, indx=None, gateiden=None): + async def addRoleRule(self, iden, rule, indx=None, gateiden=None): return await self.cell.addRoleRule(iden, rule, indx=indx, gateiden=gateiden) @adminapi(log=True) - async def delUserRule(self, iden, rule, *, gateiden=None): + async def delUserRule(self, iden, rule, gateiden=None): return await self.cell.delUserRule(iden, rule, gateiden=gateiden) @adminapi(log=True) - async def delRoleRule(self, iden, rule, *, gateiden=None): + async def delRoleRule(self, iden, rule, gateiden=None): return await self.cell.delRoleRule(iden, rule, gateiden=gateiden) @adminapi(log=True) - async def setUserAdmin(self, iden, admin, *, gateiden=None): + async def setUserAdmin(self, iden, admin, gateiden=None): return await self.cell.setUserAdmin(iden, admin, gateiden=gateiden) + @adminapi() + async def getAuthInfo(self, name): + '''This API is deprecated.''' + s_common.deprecated('CellApi.getAuthInfo') + user = await self.cell.auth.getUserByName(name) + if user is not None: + info = user.pack() + info['roles'] = [self.cell.auth.role(r).name for r in info['roles']] + return info + + role = await self.cell.auth.getRoleByName(name) + if role is not None: + return role.pack() + + raise s_exc.NoSuchName(name=name) + + @adminapi(log=True) + async def addAuthRule(self, name, rule, indx=None, gateiden=None): + '''This API is deprecated.''' + s_common.deprecated('CellApi.addAuthRule') + item = await self.cell.auth.getUserByName(name) + if item is None: + item = await self.cell.auth.getRoleByName(name) + await item.addRule(rule, indx=indx, gateiden=gateiden) + + @adminapi(log=True) + async def delAuthRule(self, name, rule, gateiden=None): + '''This API is deprecated.''' + s_common.deprecated('CellApi.delAuthRule') + item = await self.cell.auth.getUserByName(name) + if item is None: + item = await self.cell.auth.getRoleByName(name) + await item.delRule(rule, gateiden=gateiden) + + @adminapi(log=True) + async def setAuthAdmin(self, name, isadmin): + '''This API is deprecated.''' + s_common.deprecated('CellApi.setAuthAdmin') + item = await self.cell.auth.getUserByName(name) + if item is None: + item = await self.cell.auth.getRoleByName(name) + await item.setAdmin(isadmin) + async def setUserPasswd(self, iden, passwd): await self.cell.auth.reqUser(iden) @@ -597,7 +642,7 @@ async def setUserPasswd(self, iden, passwd): return await self.cell.setUserPasswd(iden, passwd) @adminapi() - async def genUserOnepass(self, iden, *, duration=60000): + async def genUserOnepass(self, iden, duration=60000): return await self.cell.genUserOnepass(iden, duration) @adminapi(log=True) @@ -613,7 +658,7 @@ async def setUserEmail(self, useriden, email): return await self.cell.setUserEmail(useriden, email) @adminapi(log=True) - async def addUserRole(self, useriden, roleiden, *, indx=None): + async def addUserRole(self, useriden, roleiden, indx=None): return await self.cell.addUserRole(useriden, roleiden, indx=indx) @adminapi(log=True) @@ -643,7 +688,7 @@ async def getRoleInfo(self, name): raise s_exc.AuthDeny(mesg=mesg, user=self.user.iden, username=self.user.name) @adminapi() - async def getUserDef(self, iden, *, packroles=True): + async def getUserDef(self, iden, packroles=True): return await self.cell.getUserDef(iden, packroles=packroles) @adminapi() @@ -675,11 +720,11 @@ async def getRoleDefs(self): return await self.cell.getRoleDefs() @adminapi() - async def isUserAllowed(self, iden, perm, *, gateiden=None, default=False): + async def isUserAllowed(self, iden, perm, gateiden=None, default=False): return await self.cell.isUserAllowed(iden, perm, gateiden=gateiden, default=default) @adminapi() - async def isRoleAllowed(self, iden, perm, *, gateiden=None): + async def isRoleAllowed(self, iden, perm, gateiden=None): return await self.cell.isRoleAllowed(iden, perm, gateiden=gateiden) @adminapi() @@ -699,7 +744,7 @@ async def setUserProfInfo(self, iden, name, valu): return await self.cell.setUserProfInfo(iden, name, valu) @adminapi() - async def popUserProfInfo(self, iden, name, *, default=None): + async def popUserProfInfo(self, iden, name, default=None): return await self.cell.popUserProfInfo(iden, name, default=default) @adminapi() @@ -715,12 +760,42 @@ async def getDmonSessions(self): return await self.cell.getDmonSessions() @adminapi() - async def getNexusChanges(self, offs, *, tellready=False, wait=True): + async def listHiveKey(self, path=None): + s_common.deprecated('CellApi.listHiveKey', curv='2.167.0') + return await self.cell.listHiveKey(path=path) + + @adminapi(log=True) + async def getHiveKeys(self, path): + s_common.deprecated('CellApi.getHiveKeys', curv='2.167.0') + return await self.cell.getHiveKeys(path) + + @adminapi(log=True) + async def getHiveKey(self, path): + s_common.deprecated('CellApi.getHiveKey', curv='2.167.0') + return await self.cell.getHiveKey(path) + + @adminapi(log=True) + async def setHiveKey(self, path, valu): + s_common.deprecated('CellApi.setHiveKey', curv='2.167.0') + return await self.cell.setHiveKey(path, valu) + + @adminapi(log=True) + async def popHiveKey(self, path): + s_common.deprecated('CellApi.popHiveKey', curv='2.167.0') + return await self.cell.popHiveKey(path) + + @adminapi(log=True) + async def saveHiveTree(self, path=()): + s_common.deprecated('CellApi.saveHiveTree', curv='2.167.0') + return await self.cell.saveHiveTree(path=path) + + @adminapi() + async def getNexusChanges(self, offs, tellready=False, wait=True): async for item in self.cell.getNexusChanges(offs, tellready=tellready, wait=wait): yield item @adminapi() - async def runBackup(self, *, name=None, wait=True): + async def runBackup(self, name=None, wait=True): ''' Run a new backup. @@ -741,8 +816,8 @@ async def getBackupInfo(self): Returns: (dict) It has the following keys: - currduration - If backup currently running, time in ms since backup started, otherwise None - - laststart - Last time (in epoch microseconds) a backup started - - lastend - Last time (in epoch microseconds) a backup ended + - laststart - Last time (in epoch milliseconds) a backup started + - lastend - Last time (in epoch milliseconds) a backup ended - lastduration - How long last backup took in ms - lastsize - Disk usage of last backup completed - lastupload - Time a backup was last completed being uploaded via iter(New)BackupArchive @@ -789,7 +864,7 @@ async def iterBackupArchive(self, name): yield @adminapi() - async def iterNewBackupArchive(self, *, name=None, remove=False): + async def iterNewBackupArchive(self, name=None, remove=False): ''' Run a new backup and return it as a compressed stream of bytes. @@ -812,7 +887,7 @@ async def getDiagInfo(self): } @adminapi() - async def runGcCollect(self, *, generation=2): + async def runGcCollect(self, generation=2): ''' For diagnostic purposes only! @@ -837,7 +912,7 @@ async def getReloadableSystems(self): return self.cell.getReloadableSystems() @adminapi(log=True) - async def reload(self, *, subsystem=None): + async def reload(self, subsystem=None): return await self.cell.reload(subsystem=subsystem) class Cell(s_nexus.Pusher, s_telepath.Aware): @@ -911,6 +986,13 @@ class Cell(s_nexus.Pusher, s_telepath.Aware): 'description': 'Record all changes to a stream file on disk. Required for mirroring (on both sides).', 'type': 'boolean', }, + 'nexslog:async': { + 'default': True, + 'description': 'Deprecated. This option ignored.', + 'type': 'boolean', + 'hidedocs': True, + 'hidecmdl': True, + }, 'dmon:listen': { 'description': 'A config-driven way to specify the telepath bind URL.', 'type': ['string', 'null'], @@ -974,11 +1056,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware): 'aha:registry': { 'description': 'The telepath URL of the aha service registry.', 'type': ['string', 'array'], - 'items': { - 'type': 'string', - 'pattern': '^ssl://.+$' - }, - 'pattern': '^ssl://.+$' + 'items': {'type': 'string'}, }, 'aha:provision': { 'description': 'The telepath URL of the aha provisioning service.', @@ -1098,13 +1176,11 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): if conf is None: conf = {} - self.starttime = time.monotonic_ns() // 1000 # Used for uptime calc - self.startmicros = s_common.now() # Used to report start time + self.starttime = time.monotonic() # Used for uptime calc + self.startms = s_common.now() # Used to report start time s_telepath.Aware.__init__(self) self.dirn = s_common.gendir(dirn) - self.sockdirn = s_common.gendir(dirn, 'sockets') - self.runid = s_common.guid() self.auth = None @@ -1129,6 +1205,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): 'tellready': 1, 'dynmirror': 1, 'tasks': 1, + 'issuewait': 1 } self.safemode = self.conf.req('safemode') @@ -1239,12 +1316,17 @@ async def fini(): self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE) + self.hive = await self._initCellHive() + self.cellinfo = self.slab.getSafeKeyVal('cell:info') self.cellvers = self.slab.getSafeKeyVal('cell:vers') + await self._bumpCellVers('cell:storage', ( + (1, self._storCellHiveMigration), + ), nexs=False) + if self.inaugural: self.cellinfo.set('nexus:version', NEXUS_VERSION) - self.cellvers.set('cell:storage', 1) # Check the cell version didn't regress if (lastver := self.cellinfo.get('cell:version')) is not None and self.VERSION < lastver: @@ -1265,6 +1347,10 @@ async def fini(): self.nexsvers = self.cellinfo.get('nexus:version', (0, 0)) self.nexspatches = () + await self._bumpCellVers('cell:storage', ( + (2, self._storCellAuthMigration), + ), nexs=False) + self.auth = await self._initCellAuth() auth_passwd = self.conf.get('auth:passwd') @@ -1312,6 +1398,138 @@ async def fini(): # phase 5 - service networking await self.initServiceNetwork() + async def _storCellHiveMigration(self): + logger.warning(f'migrating Cell ({self.getCellType()}) info out of hive') + + async with await self.hive.open(('cellvers',)) as versnode: + versdict = await versnode.dict() + for key, valu in versdict.items(): + self.cellvers.set(key, valu) + + async with await self.hive.open(('cellinfo',)) as infonode: + infodict = await infonode.dict() + for key, valu in infodict.items(): + self.cellinfo.set(key, valu) + + logger.warning(f'...Cell ({self.getCellType()}) info migration complete!') + + async def _storCellAuthMigration(self): + if self.conf.get('auth:ctor') is not None: + return + + logger.warning(f'migrating Cell ({self.getCellType()}) auth out of hive') + + authkv = self.slab.getSafeKeyVal('auth') + + async with await self.hive.open(('auth',)) as rootnode: + + rolekv = authkv.getSubKeyVal('role:info:') + rolenamekv = authkv.getSubKeyVal('role:name:') + + async with await rootnode.open(('roles',)) as roles: + for iden, node in roles: + roledict = await node.dict() + roleinfo = roledict.pack() + + roleinfo['iden'] = iden + roleinfo['name'] = node.valu + roleinfo['authgates'] = {} + roleinfo.setdefault('admin', False) + roleinfo.setdefault('rules', ()) + + rolekv.set(iden, roleinfo) + rolenamekv.set(node.valu, iden) + + userkv = authkv.getSubKeyVal('user:info:') + usernamekv = authkv.getSubKeyVal('user:name:') + + async with await rootnode.open(('users',)) as users: + for iden, node in users: + userdict = await node.dict() + userinfo = userdict.pack() + + userinfo['iden'] = iden + userinfo['name'] = node.valu + userinfo['authgates'] = {} + userinfo.setdefault('admin', False) + userinfo.setdefault('rules', ()) + userinfo.setdefault('locked', False) + userinfo.setdefault('passwd', None) + userinfo.setdefault('archived', False) + + realroles = [] + for userrole in userinfo.get('roles', ()): + if rolekv.get(userrole) is None: + mesg = f'Unknown role {userrole} on user {iden} during migration, ignoring.' + logger.warning(mesg) + continue + + realroles.append(userrole) + + userinfo['roles'] = tuple(realroles) + + userkv.set(iden, userinfo) + usernamekv.set(node.valu, iden) + + varskv = authkv.getSubKeyVal(f'user:{iden}:vars:') + async with await node.open(('vars',)) as varnodes: + for name, varnode in varnodes: + varskv.set(name, varnode.valu) + + profkv = authkv.getSubKeyVal(f'user:{iden}:profile:') + async with await node.open(('profile',)) as profnodes: + for name, profnode in profnodes: + profkv.set(name, profnode.valu) + + gatekv = authkv.getSubKeyVal('gate:info:') + async with await rootnode.open(('authgates',)) as authgates: + for gateiden, node in authgates: + gateinfo = { + 'iden': gateiden, + 'type': node.valu + } + gatekv.set(gateiden, gateinfo) + + async with await node.open(('users',)) as usernodes: + for useriden, usernode in usernodes: + if (user := userkv.get(useriden)) is None: + mesg = f'Unknown user {useriden} on gate {gateiden} during migration, ignoring.' + logger.warning(mesg) + continue + + userinfo = await usernode.dict() + userdict = userinfo.pack() + authkv.set(f'gate:{gateiden}:user:{useriden}', userdict) + + user['authgates'][gateiden] = userdict + userkv.set(useriden, user) + + async with await node.open(('roles',)) as rolenodes: + for roleiden, rolenode in rolenodes: + if (role := rolekv.get(roleiden)) is None: + mesg = f'Unknown role {roleiden} on gate {gateiden} during migration, ignoring.' + logger.warning(mesg) + continue + + roleinfo = await rolenode.dict() + roledict = roleinfo.pack() + authkv.set(f'gate:{gateiden}:role:{roleiden}', roledict) + + role['authgates'][gateiden] = roledict + rolekv.set(roleiden, role) + + logger.warning(f'...Cell ({self.getCellType()}) auth migration complete!') + + async def _drivePermMigration(self): + for lkey, lval in self.slab.scanByPref(s_drive.LKEY_INFO, db=self.drive.dbname): + info = s_msgpack.un(lval) + perm = info.pop('perm', None) + if perm is not None: + perm.setdefault('users', {}) + perm.setdefault('roles', {}) + info['permissions'] = perm + self.slab.put(lkey, s_msgpack.en(info), db=self.drive.dbname) + def getPermDef(self, perm): perm = tuple(perm) if self.permlook is None: @@ -1346,7 +1564,13 @@ def getDmonUser(self): async def fini(self): '''Fini override that ensures locking teardown order.''' - # we inherit from Pusher to make the Cell a Base subclass + + # First we teardown our activebase if it is set. This allows those tasks to be + # cancelled and do any cleanup that they may need to perform. + if self._wouldfini() and self.activebase: + await self.activebase.fini() + + # we inherit from Pusher to make the Cell a Base subclass, so we tear it down through that. retn = await s_nexus.Pusher.fini(self) if retn == 0: self._onFiniCellGuid() @@ -1426,7 +1650,7 @@ async def _onBootOptimize(self): for i, lmdbpath in enumerate(lmdbs): - logger.warning(f'... {i + 1}/{size} {lmdbpath}') + logger.warning(f'... {i+1}/{size} {lmdbpath}') with self.getTempDir() as backpath: @@ -1448,33 +1672,22 @@ def _delTmpFiles(self): tdir = s_common.gendir(self.dirn, 'tmp') names = os.listdir(tdir) - if names: - logger.info(f'Removing {len(names)} temporary files/folders in: {tdir}') - - for name in names: + if not names: + return - path = os.path.join(tdir, name) + logger.warning(f'Removing {len(names)} temporary files/folders in: {tdir}') - if os.path.isfile(path): - os.unlink(path) - continue + for name in names: - if os.path.isdir(path): - shutil.rmtree(path, ignore_errors=True) - continue + path = os.path.join(tdir, name) - names = os.listdir(self.sockdirn) - if names: - logger.info(f'Removing {len(names)} old sockets in: {self.sockdirn}') - for name in names: - path = os.path.join(self.sockdirn, name) - try: - if stat.S_ISSOCK(os.stat(path).st_mode): - os.unlink(path) - except OSError: # pragma: no cover - pass + if os.path.isfile(path): + os.unlink(path) + continue - # FIXME - recursively remove sockets dir here? + if os.path.isdir(path): + shutil.rmtree(path, ignore_errors=True) + continue async def _execCellUpdates(self): # implement to apply updates to a fully initialized active cell @@ -1631,6 +1844,10 @@ async def onlink(proxy): elif isinstance(oldurls, list): oldurls = tuple(oldurls) if newurls and newurls != oldurls: + if oldurls[0].startswith('tcp://'): + s_common.deprecated('aha:registry: tcp:// client values.') + return + self.modCellConf({'aha:registry': newurls}) self.ahaclient.setBootUrls(newurls) @@ -1682,16 +1899,10 @@ async def initServiceEarly(self): pass async def initCellStorage(self): - - path = s_common.gendir(self.dirn, 'slabs', 'drive.lmdb') - sockpath = s_common.genpath(self.sockdirn, 'drive') - - if len(sockpath) > s_const.UNIX_PATH_MAX: - sockpath = None - - spawner = s_drive.FileDrive.spawner(base=self, sockpath=sockpath) - - self.drive = await spawner(path) + self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') + await self._bumpCellVers('drive:storage', ( + (1, self._drivePermMigration), + ), nexs=False) self.onfini(self.drive.fini) @@ -1711,7 +1922,7 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): # replay safety... iden = info.get('iden') - if await self.drive.hasItemInfo(iden): # pragma: no cover + if self.drive.hasItemInfo(iden): # pragma: no cover return await self.drive.getItemPath(iden) # TODO: Remove this in synapse-3xx @@ -1724,10 +1935,10 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): return await self.drive.addItemInfo(info, path=path, reldir=reldir) async def getDriveInfo(self, iden, typename=None): - return await self.drive.getItemInfo(iden, typename=typename) + return self.drive.getItemInfo(iden, typename=typename) - async def reqDriveInfo(self, iden, typename=None): - return await self.drive.reqItemInfo(iden, typename=typename) + def reqDriveInfo(self, iden, typename=None): + return self.drive.reqItemInfo(iden, typename=typename) async def getDrivePath(self, path, reldir=s_drive.rootdir): ''' @@ -1750,16 +1961,14 @@ async def addDrivePath(self, path, perm=None, reldir=s_drive.rootdir): ''' tick = s_common.now() user = self.auth.rootuser.iden - path = await self.drive.getPathNorm(path) + path = self.drive.getPathNorm(path) if perm is None: perm = {'users': {}, 'roles': {}} for name in path: - info = await self.drive.getStepInfo(reldir, name) - - # we could skip this now ;) + info = self.drive.getStepInfo(reldir, name) await asyncio.sleep(0) if info is not None: @@ -1783,7 +1992,7 @@ async def getDriveData(self, iden, vers=None): Return the data associated with the drive item by iden. If vers is specified, return that specific version. ''' - return await self.drive.getItemData(iden, vers=vers) + return self.drive.getItemData(iden, vers=vers) async def getDriveDataVersions(self, iden): async for item in self.drive.getItemDataVersions(iden): @@ -1791,12 +2000,12 @@ async def getDriveDataVersions(self, iden): @s_nexus.Pusher.onPushAuto('drive:del') async def delDriveInfo(self, iden): - if await self.drive.getItemInfo(iden) is not None: + if self.drive.getItemInfo(iden) is not None: await self.drive.delItemInfo(iden) @s_nexus.Pusher.onPushAuto('drive:set:perm') async def setDriveInfoPerm(self, iden, perm): - return await self.drive.setItemPerm(iden, perm) + return self.drive.setItemPerm(iden, perm) @s_nexus.Pusher.onPushAuto('drive:data:path:set') async def setDriveItemProp(self, iden, vers, path, valu): @@ -1845,7 +2054,7 @@ async def delDriveItemProp(self, iden, vers, path): @s_nexus.Pusher.onPushAuto('drive:set:path') async def setDriveInfoPath(self, iden, path): - path = await self.drive.getPathNorm(path) + path = self.drive.getPathNorm(path) pathinfo = await self.drive.getItemPath(iden) if path == [p.get('name') for p in pathinfo]: return pathinfo @@ -1858,13 +2067,13 @@ async def setDriveData(self, iden, versinfo, data): async def delDriveData(self, iden, vers=None): if vers is None: - info = await self.drive.reqItemInfo(iden) + info = self.drive.reqItemInfo(iden) vers = info.get('version') return await self._push('drive:data:del', iden, vers) @s_nexus.Pusher.onPush('drive:data:del') async def _delDriveData(self, iden, vers): - return await self.drive.delItemData(iden, vers) + return self.drive.delItemData(iden, vers) async def getDriveKids(self, iden): async for info in self.drive.getItemKids(iden): @@ -1899,11 +2108,11 @@ async def _bindDmonListen(self): logger.error('LOCAL UNIX SOCKET WILL BE UNAVAILABLE') except Exception: # pragma: no cover logging.exception('Unknown dmon listen error.') - else: - turl = self._getDmonListen() - if turl is not None: - logger.info(f'dmon listening: {turl}') - self.sockaddr = await self.dmon.listen(turl) + + turl = self._getDmonListen() + if turl is not None: + logger.info(f'dmon listening: {turl}') + self.sockaddr = await self.dmon.listen(turl) async def initServiceNetwork(self): @@ -1958,7 +2167,6 @@ async def getAhaInfo(self): 'leader': ahalead, 'urlinfo': urlinfo, 'ready': ready, - 'isleader': self.isactive, 'promotable': self.conf.get('aha:promotable'), } @@ -1993,9 +2201,9 @@ async def _runAhaRegLoop(): proxy = await self.ahaclient.proxy() info = await self.getAhaInfo() - await proxy.addAhaSvc(f'{ahaname}...', info) + await proxy.addAhaSvc(ahaname, info, network=ahanetw) if self.isactive and ahalead is not None: - await proxy.addAhaSvc(f'{ahalead}...', info) + await proxy.addAhaSvc(ahalead, info, network=ahanetw) except Exception as e: logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}') @@ -2077,6 +2285,10 @@ async def waitFor(turl_sani, prox_): return cullat + @s_nexus.Pusher.onPushAuto('nexslog:setindex') + async def setNexsIndx(self, indx): + return await self.nexsroot.setindex(indx) + def getMyUrl(self, user='root'): host = self.conf.req('aha:name') network = self.conf.req('aha:network') @@ -2267,21 +2479,38 @@ async def reqAhaProxy(self, timeout=None, feats=None): return proxy - async def _bumpAhaProxy(self): + async def _setAhaActive(self): if self.ahaclient is None: return - # force a reconnect to AHA to update service info + ahainfo = await self.getAhaInfo() + if ahainfo is None: + return + + ahalead = self.conf.get('aha:leader') + if ahalead is None: + return + try: - proxy = await self.ahaclient.proxy(timeout=5) - if proxy is not None: - await proxy.fini() + proxy = await self.ahaclient.proxy(timeout=2) - except Exception as e: - extra = await self.getLogExtra(name=self.conf.get('aha:name')) - logger.exception('Error forcing AHA reconnect.', extra=extra) + except TimeoutError: # pragma: no cover + return None + + # if we went inactive, bump the aha proxy + if not self.isactive: + await proxy.fini() + return + + ahanetw = self.conf.req('aha:network') + try: + await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw) + except asyncio.CancelledError: # pragma: no cover + raise + except Exception as e: # pragma: no cover + logger.warning(f'_setAhaActive failed: {e}') def isActiveCoro(self, iden): return self.activecoros.get(iden) is not None @@ -2393,7 +2622,7 @@ async def setCellActive(self, active): self.activebase = None await self.initServicePassive() - await self._bumpAhaProxy() + await self._setAhaActive() def runActiveTask(self, coro): # an API for active coroutines to use when running an @@ -2761,10 +2990,18 @@ async def getUserProfInfo(self, iden, name, default=None): user = await self.auth.reqUser(iden) return user.profile.get(name, defv=default) + async def _setUserProfInfoV0(self, iden, name, valu): + path = ('auth', 'users', iden, 'profile', name) + return await self.hive._push('hive:set', path, valu) + async def setUserProfInfo(self, iden, name, valu): user = await self.auth.reqUser(iden) return await user.setProfileValu(name, valu) + async def _popUserProfInfoV0(self, iden, name, default=None): + path = ('auth', 'users', iden, 'profile', name) + return await self.hive._push('hive:pop', path) + async def popUserProfInfo(self, iden, name, default=None): user = await self.auth.reqUser(iden) return await user.popProfileValu(name, default=default) @@ -2785,10 +3022,18 @@ async def getUserVarValu(self, iden, name, default=None): user = await self.auth.reqUser(iden) return user.vars.get(name, defv=default) + async def _setUserVarValuV0(self, iden, name, valu): + path = ('auth', 'users', iden, 'vars', name) + return await self.hive._push('hive:set', path, valu) + async def setUserVarValu(self, iden, name, valu): user = await self.auth.reqUser(iden) return await user.setVarValu(name, valu) + async def _popUserVarValuV0(self, iden, name, default=None): + path = ('auth', 'users', iden, 'vars', name) + return await self.hive._push('hive:pop', path) + async def popUserVarValu(self, iden, name, default=None): user = await self.auth.reqUser(iden) return await user.popVarValu(name, default=default) @@ -3353,6 +3598,13 @@ async def _initCellDmon(self): self.onfini(self.dmon.fini) + async def _initCellHive(self): + db = self.slab.initdb('hive') + hive = await s_hive.SlabHive.anit(self.slab, db=db, nexsroot=self.getCellNexsRoot(), cell=self) + self.onfini(hive) + + return hive + async def _initSlabFile(self, path, readonly=False, ephemeral=False): slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly) slab.addResizeCallback(self.checkFreeSpace) @@ -3370,6 +3622,7 @@ async def _initCellSlab(self, readonly=False): if not os.path.exists(path) and readonly: logger.warning('Creating a slab for a readonly cell.') _slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE) + _slab.initdb('hive') await _slab.fini() self.slab = await self._initSlabFile(path) @@ -3380,10 +3633,16 @@ async def _initCellAuth(self): self.on('user:del', self._onUserDelEvnt) self.on('user:lock', self._onUserLockEvnt) + authctor = self.conf.get('auth:ctor') + if authctor is not None: + s_common.deprecated('auth:ctor cell config option', curv='2.157.0') + ctor = s_dyndeps.getDynLocal(authctor) + return await ctor(self) + maxusers = self.conf.get('max:users') policy = self.conf.get('auth:passwd:policy') - seed = s_common.guid((self.iden, 'auth')) + seed = s_common.guid((self.iden, 'hive', 'auth')) auth = await s_auth.Auth.anit( self.slab, @@ -3876,7 +4135,7 @@ async def _initBootRestore(cls, dirn): content_length = int(resp.headers.get('content-length', 0)) if content_length > 100: - logger.warning(f'Downloading {content_length / s_const.megabyte:.3f} MB of data.') + logger.warning(f'Downloading {content_length/s_const.megabyte:.3f} MB of data.') pvals = [int((content_length * 0.01) * i) for i in range(1, 100)] else: # pragma: no cover logger.warning(f'Odd content-length encountered: {content_length}') @@ -3892,7 +4151,7 @@ async def _initBootRestore(cls, dirn): if pvals and tsize > pvals[0]: pvals.pop(0) percentage = (tsize / content_length) * 100 - logger.warning(f'Downloaded {tsize / s_const.megabyte:.3f} MB, {percentage:.3f}%') + logger.warning(f'Downloaded {tsize/s_const.megabyte:.3f} MB, {percentage:.3f}%') logger.warning(f'Extracting {tarpath} to {dirn}') @@ -3902,7 +4161,7 @@ async def _initBootRestore(cls, dirn): continue memb.name = memb.name.split('/', 1)[1] logger.warning(f'Extracting {memb.name}') - tgz.extract(memb, dirn, filter='data') + tgz.extract(memb, dirn) # and record the rurliden with s_common.genfile(donepath) as fd: @@ -4103,7 +4362,7 @@ async def _initCloneCell(self, proxy): if memb.name.find('/') == -1: continue memb.name = memb.name.split('/', 1)[1] - tgz.extract(memb, self.dirn, filter='data') + tgz.extract(memb, self.dirn) finally: @@ -4144,7 +4403,6 @@ async def _bootCellMirror(self, pnfo): logger.warning(f'Bootstrap mirror from: {murl} DONE!') async def getMirrorUrls(self): - if self.ahaclient is None: raise s_exc.BadConfValu(mesg='Enumerating mirror URLs is only supported when AHA is configured') @@ -4156,7 +4414,7 @@ async def getMirrorUrls(self): mesg = 'Service must be configured with AHA to enumerate mirror URLs' raise s_exc.NoSuchName(mesg=mesg, name=self.ahasvcname) - return [f'aha://{svc["name"]}' for svc in mirrors] + return [f'aha://{svc["svcname"]}.{svc["svcnetw"]}' for svc in mirrors] @classmethod async def initFromArgv(cls, argv, outp=None): @@ -4208,7 +4466,7 @@ async def initFromArgv(cls, argv, outp=None): logger.exception(f'Error while bootstrapping cell config.') raise - s_coro.set_pool_logging(logger, logconf=conf['_log_conf']) + s_processpool.set_pool_logging(logger, logconf=conf['_log_conf']) try: cell = await cls.anit(opts.dirn, conf=conf) @@ -4338,6 +4596,59 @@ async def _cellHealth(self, health): async def getDmonSessions(self): return await self.dmon.getSessInfo() + # ----- Change distributed Auth methods ---- + + async def listHiveKey(self, path=None): + if path is None: + path = () + items = self.hive.dir(path) + if items is None: + return None + return [item[0] for item in items] + + async def getHiveKeys(self, path): + ''' + Return a list of (name, value) tuples for nodes under the path. + ''' + items = self.hive.dir(path) + if items is None: + return () + + return [(i[0], i[1]) for i in items] + + async def getHiveKey(self, path): + ''' + Get the value of a key in the cell default hive + ''' + return await self.hive.get(path) + + async def setHiveKey(self, path, valu): + ''' + Set or change the value of a key in the cell default hive + ''' + return await self.hive.set(path, valu, nexs=True) + + async def popHiveKey(self, path): + ''' + Remove and return the value of a key in the cell default hive. + + Note: this is for expert emergency use only. + ''' + return await self.hive.pop(path, nexs=True) + + async def saveHiveTree(self, path=()): + return await self.hive.saveHiveTree(path=path) + + async def loadHiveTree(self, tree, path=(), trim=False): + ''' + Note: this is for expert emergency use only. + ''' + return await self._push('hive:loadtree', tree, path, trim) + + @s_nexus.Pusher.onPush('hive:loadtree') + async def _onLoadHiveTree(self, tree, path, trim): + return await self.hive.loadHiveTree(tree, path=path, trim=trim) + async def iterSlabData(self, name, prefix=''): slabkv = self.slab.getSafeKeyVal(name, prefix=prefix, create=False) for key, valu in slabkv.items(): @@ -4511,6 +4822,8 @@ async def getCellInfo(self): if mirror is not None: mirror = s_urlhelp.sanitizeUrl(mirror) + nxfo = await self.nexsroot.getNexsInfo() + ret = { 'synapse': { 'commit': s_version.commit, @@ -4523,15 +4836,15 @@ async def getCellInfo(self): 'iden': self.getCellIden(), 'paused': self.paused, 'active': self.isactive, - 'started': self.startmicros, 'safemode': self.safemode, - 'ready': self.nexsroot.ready.is_set(), + 'started': self.startms, + 'ready': nxfo['ready'], # TODO: Remove in 3.x.x 'commit': self.COMMIT, 'version': self.VERSION, 'verstring': self.VERSTRING, 'cellvers': dict(self.cellvers.items()), - 'nexsindx': await self.getNexsIndx(), - 'uplink': self.nexsroot.miruplink.is_set(), + 'nexsindx': nxfo['indx'], # TODO: Remove in 3.x.x + 'uplink': nxfo['uplink:ready'], # TODO: Remove in 3.x.x 'mirror': mirror, 'aha': { 'name': self.conf.get('aha:name'), @@ -4540,7 +4853,8 @@ async def getCellInfo(self): }, 'network': { 'https': self.https_listeners, - } + }, + 'nexus': nxfo, }, 'features': self.features, } @@ -4559,8 +4873,8 @@ async def getSystemInfo(self): - volfree - Volume where cell is running free space - backupvolsize - Backup directory volume total space - backupvolfree - Backup directory volume free space - - cellstarttime - Cell start time in epoch microseconds - - celluptime - Cell uptime in microseconds + - cellstarttime - Cell start time in epoch milliseconds + - celluptime - Cell uptime in milliseconds - cellrealdisk - Cell's use of disk, equivalent to du - cellapprdisk - Cell's apparent use of disk, equivalent to ls -l - osversion - OS version/architecture @@ -4570,7 +4884,7 @@ async def getSystemInfo(self): - cpucount - Number of CPUs on system - tmpdir - The temporary directory interpreted by the Python runtime. ''' - uptime = time.monotonic_ns() // 1000 - self.starttime + uptime = int((time.monotonic() - self.starttime) * 1000) disk = shutil.disk_usage(self.dirn) if self.backdirn: @@ -4592,8 +4906,8 @@ async def getSystemInfo(self): 'volfree': disk.free, # Volume where cell is running free bytes 'backupvolsize': backupvolsize, # Cell's backup directory volume total bytes 'backupvolfree': backupvolfree, # Cell's backup directory volume free bytes - 'cellstarttime': self.startmicros, # Cell's start time in epoch micros - 'celluptime': uptime, # Cell's uptime in micros + 'cellstarttime': self.startms, # cell start time in epoch millis + 'celluptime': uptime, # cell uptime in ms 'cellrealdisk': myusage, # Cell's use of disk, equivalent to du 'cellapprdisk': myappusage, # Cell's apparent use of disk, equivalent to ls -l 'osversion': platform.platform(), # OS version/architecture @@ -4686,7 +5000,7 @@ async def addUserApiKey(self, useriden, name, duration=None): Args: useriden (str): User iden value. name (str): Name of the API key. - duration (int or None): Duration of time for the API key to be valid ( in microseconds ). + duration (int or None): Duration of time for the API key to be valid ( in milliseconds ). Returns: tuple: A tuple of the secret API key value and the API key metadata information. @@ -4723,9 +5037,9 @@ async def addUserApiKey(self, useriden, name, duration=None): async def _genUserApiKey(self, kdef): iden = s_common.uhex(kdef.get('iden')) user = s_common.uhex(kdef.get('user')) - await self.slab.put(iden, user, db=self.apikeydb) + self.slab.put(iden, user, db=self.apikeydb) lkey = user + b'apikey' + iden - await self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) + self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) async def getUserApiKey(self, iden): ''' @@ -4881,7 +5195,7 @@ async def _setUserApiKey(self, user, iden, vals): raise s_exc.NoSuchIden(mesg=f'User API key does not exist: {iden}') kdef = s_msgpack.un(buf) kdef.update(vals) - await self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) + self.slab.put(lkey, s_msgpack.en(kdef), db=self.usermetadb) return kdef async def delUserApiKey(self, iden): @@ -4955,8 +5269,6 @@ def _makeCachedSslCtx(self, opts): else: sslctx = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH) - sslctx.verify_flags &= ~ssl.VERIFY_X509_STRICT - if not opts['verify']: sslctx.check_hostname = False sslctx.verify_mode = ssl.CERT_NONE @@ -5000,12 +5312,14 @@ def _makeCachedSslCtx(self, opts): return sslctx - def getCachedSslCtx(self, opts=None): + def getCachedSslCtx(self, opts=None, verify=None): - # Default to verifying SSL/TLS certificates if opts is None: opts = {} + if verify is not None: + opts['verify'] = verify + opts = s_schemas.reqValidSslCtxOpts(opts) key = tuple(sorted(opts.items())) diff --git a/synapse/lib/const.py b/synapse/lib/const.py index 655af704aaa..217bbba731a 100644 --- a/synapse/lib/const.py +++ b/synapse/lib/const.py @@ -31,8 +31,8 @@ zebibyte = 1024 * exbibyte yobibyte = 1024 * zebibyte -# time (in micros) constants -second = 1000000 +# time (in millis) constants +second = 1000 minute = second * 60 hour = minute * 60 day = hour * 24 diff --git a/synapse/lib/spawner.py b/synapse/lib/spawner.py index bd386f12a7c..fa322b7fd2a 100644 --- a/synapse/lib/spawner.py +++ b/synapse/lib/spawner.py @@ -8,8 +8,8 @@ import synapse.telepath as s_telepath import synapse.lib.base as s_base -import synapse.lib.coro as s_coro import synapse.lib.link as s_link +import synapse.lib.process as s_process logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ async def _spawn(cls, args, kwargs, base=None, sockpath=None): if base is None: base = await s_base.Base.anit() - base.schedCoro(s_coro.spawn((_ioWorkProc, (todo, sockpath), {}))) + base.schedCoro(s_process.spawn((_ioWorkProc, (todo, sockpath), {}))) await s_link.unixwait(sockpath) diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index 88090ada089..faa8a4df7f1 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -10,7 +10,6 @@ import synapse.lib.base as s_base import synapse.lib.coro as s_coro import synapse.lib.scope as s_scope -import synapse.lib.spawner as s_spawner import synapse.tests.utils as s_t_utils @@ -46,14 +45,6 @@ async def postAnit(self): if self.foo == -1: raise s_exc.BadArg(mesg='boom') -class Haha(s_base.Base, s_spawner.SpawnerMixin): - - async def __anit__(self): - await s_base.Base.__anit__(self) - - async def fini(self): - await s_base.Base.fini(self) - class BaseTest(s_t_utils.SynTest): async def test_base_basics(self): @@ -305,14 +296,18 @@ async def callfini(): async def test_base_refcount(self): base = await s_base.Base.anit() + self.true(base._wouldfini()) self.eq(base.incref(), 2) + self.false(base._wouldfini()) self.eq(await base.fini(), 1) self.false(base.isfini) + self.true(base._wouldfini()) self.eq(await base.fini(), 0) self.true(base.isfini) + self.false(base._wouldfini()) async def test_baseref_gen(self): @@ -441,21 +436,6 @@ def onHehe1(mesg): self.len(2, l0) self.len(1, l1) - # set the 'hehe' and haha callback with onWithMulti - with base.onWithMulti(('hehe', 'haha'), onHehe1) as e: - self.true(e is base) - await base.fire('hehe') - self.len(3, l0) - self.len(2, l1) - - await base.fire('haha') - self.len(3, l0) - self.len(3, l1) - - await base.fire('hehe') - self.len(4, l0) - self.len(3, l1) - async def test_base_mixin(self): data = [] @@ -590,29 +570,3 @@ async def func3(bobj, key, valu): # The scope data set in the task is not present outside of it. self.none(s_scope.get('hehe')) - - async def test_base_spawner_fini(self): - - # Test proxy fini, base should fini - base = await s_base.Base.anit() - spawner = Haha.spawner(base=base) - proxy = await spawner() - - self.false(proxy.isfini) - self.false(base.isfini) - - await proxy.fini() - self.true(proxy.isfini) - self.true(base.isfini) - - # Test base fini, proxy should fini - base = await s_base.Base.anit() - spawner = Haha.spawner(base=base) - proxy = await spawner() - - self.false(proxy.isfini) - self.false(base.isfini) - - await base.fini() - self.true(base.isfini) - self.true(proxy.isfini) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 80c165c0680..353d6793695 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -28,7 +28,6 @@ import synapse.lib.coro as s_coro import synapse.lib.json as s_json import synapse.lib.link as s_link -import synapse.lib.const as s_const import synapse.lib.drive as s_drive import synapse.lib.nexus as s_nexus import synapse.lib.config as s_config @@ -57,11 +56,6 @@ def _exiterProc(pipe, srcdir, dstdir, lmdbpaths, logconf): def _backupSleep(path, linkinfo): time.sleep(3.0) -async def migrate_v1(info, versinfo, data, curv): - assert curv == 1 - data['woot'] = 'woot' - return data - async def _doEOFBackup(path): return @@ -149,6 +143,32 @@ async def stream(self, doraise=False): if doraise: raise s_exc.BadTime(mesg='call again later') +async def altAuthCtor(cell): + authconf = cell.conf.get('auth:conf') + assert authconf['foo'] == 'bar' + authconf['baz'] = 'faz' + + maxusers = cell.conf.get('max:users') + + seed = s_common.guid((cell.iden, 'hive', 'auth')) + + auth = await s_auth.Auth.anit( + cell.slab, + 'auth', + seed=seed, + nexsroot=cell.getCellNexsRoot(), + maxusers=maxusers + ) + + auth.link(cell.dist) + + def finilink(): + auth.unlink(cell.dist) + + cell.onfini(finilink) + cell.onfini(auth.fini) + return auth + testDataSchema_v0 = { 'type': 'object', 'properties': { @@ -254,9 +274,9 @@ async def test_cell_drive(self): neatrole = await cell.auth.addRole('neatrole') await fooser.grant(neatrole.iden) - # with self.raises(s_exc.SchemaViolation): - # versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - # await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) + with self.raises(s_exc.SchemaViolation): + versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20, 'stuff': 12}) @@ -312,10 +332,11 @@ async def test_cell_drive(self): self.eq(data[1]['stuff'], 1234) # This will be done by the cell in a cell storage version migration... - callback = 'synapse.tests.test_lib_cell.migrate_v1' - await cell.drive.setTypeSchema('woot', testDataSchema_v1, callback=callback) + async def migrate_v1(info, versinfo, data): + data['woot'] = 'woot' + return data - await cell.setDriveItemProp(iden, versinfo, 'woot', 'woot') + await cell.drive.setTypeSchema('woot', testDataSchema_v1, migrate_v1) versinfo['version'] = (1, 1, 1) await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829) @@ -417,7 +438,7 @@ async def test_cell_drive(self): with self.raises(s_exc.DupName): iden = pathinfo[-2].get('iden') name = pathinfo[-1].get('name') - await cell.drive.reqFreeStep(iden, name) + cell.drive.reqFreeStep(iden, name) walks = [item async for item in cell.drive.walkPathInfo('hehe')] self.len(3, walks) @@ -433,10 +454,10 @@ async def test_cell_drive(self): self.eq('haha', walks[1].get('name')) self.eq('hehe', walks[2].get('name')) - self.none(await cell.drive.getTypeSchema('newp')) + self.none(cell.drive.getTypeSchema('newp')) - # cell.drive.validators.pop('woot') - # self.nn(cell.drive.getTypeValidator('woot')) + cell.drive.validators.pop('woot') + self.nn(cell.drive.getTypeValidator('woot')) # move to root dir pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop') @@ -451,8 +472,7 @@ async def test_cell_drive(self): # explicitly clear out the cache JsValidators, otherwise we get the cached, pre-msgpack # version of the validator, which will be correct and skip the point of this test. s_config._JsValidators.clear() - # FIXME - # await cell.drive.reqValidData('woot', data) + cell.drive.reqValidData('woot', data) async def test_cell_auth(self): @@ -548,6 +568,30 @@ async def test_cell_auth(self): self.true(await proxy.icando('foo', 'bar')) await self.asyncraises(s_exc.AuthDeny, proxy.icando('foo', 'newp')) + # happy path perms + await visi.addRule((True, ('hive:set', 'foo', 'bar'))) + await visi.addRule((True, ('hive:get', 'foo', 'bar'))) + await visi.addRule((True, ('hive:pop', 'foo', 'bar'))) + + val = await echo.setHiveKey(('foo', 'bar'), 'thefirstval') + self.eq(None, val) + + # check that we get the old val back + val = await echo.setHiveKey(('foo', 'bar'), 'wootisetit') + self.eq('thefirstval', val) + + val = await echo.getHiveKey(('foo', 'bar')) + self.eq('wootisetit', val) + + val = await echo.popHiveKey(('foo', 'bar')) + self.eq('wootisetit', val) + + val = await echo.setHiveKey(('foo', 'bar', 'baz'), 'a') + val = await echo.setHiveKey(('foo', 'bar', 'faz'), 'b') + val = await echo.setHiveKey(('foo', 'bar', 'haz'), 'c') + val = await echo.listHiveKey(('foo', 'bar')) + self.eq(('baz', 'faz', 'haz'), val) + # visi user can change visi user pass await proxy.setUserPasswd(visi.iden, 'foobar') # non admin visi user cannot change root user pass @@ -639,20 +683,70 @@ async def test_cell_auth(self): await self.asyncraises(s_exc.AuthDeny, s_telepath.openurl(visi_url)) + await echo.setHiveKey(('foo', 'bar'), [1, 2, 3, 4]) + self.eq([1, 2, 3, 4], await echo.getHiveKey(('foo', 'bar'))) + self.isin('foo', await echo.listHiveKey()) + self.eq(['bar'], await echo.listHiveKey(('foo',))) + await echo.popHiveKey(('foo', 'bar')) + self.eq([], await echo.listHiveKey(('foo',))) + # Ensure we can delete a rule by its item and index position async with echo.getLocalProxy() as proxy: # type: EchoAuthApi - rule = (True, ('foo', 'bar')) + rule = (True, ('hive:set', 'foo', 'bar')) self.isin(rule, visi.info.get('rules')) await proxy.delUserRule(visi.iden, rule) self.notin(rule, visi.info.get('rules')) # Removing a non-existing rule by *rule* has no consequence await proxy.delUserRule(visi.iden, rule) + rule = visi.info.get('rules')[0] + self.isin(rule, visi.info.get('rules')) + await proxy.delUserRule(visi.iden, rule) + self.notin(rule, visi.info.get('rules')) + self.eq(echo.getDmonUser(), echo.auth.rootuser.iden) with self.raises(s_exc.NeedConfValu): await echo.reqAhaProxy() + async def test_cell_drive_perm_migration(self): + async with self.getRegrCore('drive-perm-migr') as core: + item = await core.getDrivePath('driveitemdefaultperms') + self.len(1, item) + self.notin('perm', item) + self.eq(item[0]['permissions'], {'users': {}, 'roles': {}}) + + ldog = await core.auth.getRoleByName('littledog') + bdog = await core.auth.getRoleByName('bigdog') + + louis = await core.auth.getUserByName('lewis') + tim = await core.auth.getUserByName('tim') + mj = await core.auth.getUserByName('mj') + + item = await core.getDrivePath('permfolder/driveitemwithperms') + self.len(2, item) + self.notin('perm', item[0]) + self.notin('perm', item[1]) + self.eq(item[0]['permissions'], {'users': {tim.iden: s_cell.PERM_ADMIN}, 'roles': {}}) + self.eq(item[1]['permissions'], { + 'users': { + mj.iden: s_cell.PERM_ADMIN + }, + 'roles': { + ldog.iden: s_cell.PERM_READ, + bdog.iden: s_cell.PERM_EDIT, + }, + 'default': s_cell.PERM_DENY + }) + + # make sure it's all good with easy perms + self.true(core._hasEasyPerm(item[0], tim, s_cell.PERM_ADMIN)) + self.false(core._hasEasyPerm(item[0], mj, s_cell.PERM_EDIT)) + + self.true(core._hasEasyPerm(item[1], mj, s_cell.PERM_ADMIN)) + self.true(core._hasEasyPerm(item[1], tim, s_cell.PERM_READ)) + self.true(core._hasEasyPerm(item[1], louis, s_cell.PERM_EDIT)) + async def test_cell_unix_sock(self): async with self.getTestCore() as core: @@ -754,7 +848,7 @@ async def test_longpath(self): # but exercises the long-path failure inside of the cell's daemon # instead. with self.getTestDir() as dirn: - extrapath = s_const.UNIX_PATH_MAX * 'A' + extrapath = 108 * 'A' longdirn = s_common.genpath(dirn, extrapath) with self.getAsyncLoggerStream('synapse.lib.cell', 'LOCAL UNIX SOCKET WILL BE UNAVAILABLE') as stream: async with self.getTestCell(s_cell.Cell, dirn=longdirn) as cell: @@ -816,6 +910,7 @@ async def test_cell_getinfo(self): info = await prox.getCellInfo() # Cell information cnfo = info.get('cell') + nxfo = cnfo.get('nexus') snfo = info.get('synapse') self.eq(cnfo.get('commit'), 'mycommit') self.eq(cnfo.get('version'), (1, 2, 3)) @@ -825,7 +920,15 @@ async def test_cell_getinfo(self): self.ge(cnfo.get('nexsindx'), 0) self.true(cnfo.get('active')) self.false(cnfo.get('uplink')) + self.true(cnfo.get('ready')) self.none(cnfo.get('mirror', True)) + # Nexus info + self.ge(nxfo.get('indx'), 0) + self.false(nxfo.get('uplink:ready')) + self.true(nxfo.get('ready')) + self.false(nxfo.get('readonly')) + self.eq(nxfo.get('holds'), []) + # A Cortex populated cellvers self.isin('cortex:defaults', cnfo.get('cellvers', {})) @@ -844,6 +947,21 @@ async def test_cell_getinfo(self): https = netw.get('https') self.eq(https, http_info) + # Write hold information is reflected through cell info + await cell.nexsroot.addWriteHold('boop') + await cell.nexsroot.addWriteHold('beep') + info = await prox.getCellInfo() + nxfo = info.get('cell').get('nexus') + self.true(nxfo.get('readonly')) + self.eq(nxfo.get('holds'), [{'reason': 'beep'}, {'reason': 'boop'}]) + + await cell.nexsroot.delWriteHold('boop') + await cell.nexsroot.delWriteHold('beep') + info = await prox.getCellInfo() + nxfo = info.get('cell').get('nexus') + self.false(nxfo.get('readonly')) + self.eq(nxfo.get('holds'), []) + # Mirrors & ready flags async with self.getTestAha() as aha: # type: s_aha.AhaCell @@ -862,18 +980,26 @@ async def test_cell_getinfo(self): await cell01.sync() cnfo0 = await cell00.getCellInfo() + nxfo0 = cnfo0['cell']['nexus'] cnfo1 = await cell01.getCellInfo() + nxfo1 = cnfo1['cell']['nexus'] self.true(cnfo0['cell']['ready']) self.false(cnfo0['cell']['uplink']) self.none(cnfo0['cell']['mirror']) self.eq(cnfo0['cell']['version'], (1, 2, 3)) + self.false(nxfo0.get('uplink:ready')) + self.true(nxfo0.get('ready')) self.true(cnfo1['cell']['ready']) self.true(cnfo1['cell']['uplink']) self.eq(cnfo1['cell']['mirror'], 'aha://root@cell...') self.eq(cnfo1['cell']['version'], (1, 2, 3)) + self.true(nxfo1.get('uplink:ready')) + self.true(nxfo1.get('ready')) + self.eq(cnfo0['cell']['nexsindx'], cnfo1['cell']['nexsindx']) + self.eq(nxfo0['indx'], nxfo1['indx']) async def test_cell_dyncall(self): @@ -939,7 +1065,7 @@ async def coro(prox, offs): yielded = False async for offset, data in prox.getNexusChanges(offs): yielded = True - nexsiden, act, args, kwargs, meta, _ = data + nexsiden, act, args, kwargs, meta = data if nexsiden == 'auth:auth' and act == 'user:add': retn.append(args) break @@ -947,6 +1073,7 @@ async def coro(prox, offs): conf = { 'nexslog:en': True, + 'nexslog:async': True, 'dmon:listen': 'tcp://127.0.0.1:0/', 'https:port': 0, } @@ -1029,7 +1156,7 @@ async def test_cell_nexuscull(self): self.eq(0, await prox.trimNexsLog()) for i in range(5): - await cell.sync() + await prox.setHiveKey(('foo', 'bar'), i) ind = await prox.getNexsIndx() offs = await prox.rotateNexsLog() @@ -1058,7 +1185,7 @@ async def test_cell_nexuscull(self): self.eq('nexslog:cull', retn[0][1][1]) for i in range(6, 10): - await cell.sync() + await prox.setHiveKey(('foo', 'bar'), i) # trim ind = await prox.getNexsIndx() @@ -1071,7 +1198,7 @@ async def test_cell_nexuscull(self): self.eq(ind + 2, await prox.trimNexsLog()) for i in range(10, 15): - await cell.sync() + await prox.setHiveKey(('foo', 'bar'), i) # nexus log exists but logging is disabled conf['nexslog:en'] = False @@ -1111,8 +1238,8 @@ async def test_cell_nexusrotate(self): } async with await s_cell.Cell.anit(dirn, conf=conf) as cell: - await cell.sync() - await cell.sync() + await cell.setHiveKey(('foo', 'bar'), 0) + await cell.setHiveKey(('foo', 'bar'), 1) await cell.rotateNexsLog() @@ -1124,7 +1251,7 @@ async def test_cell_nexusrotate(self): self.len(2, cell.nexsroot.nexslog._ranges) self.eq(0, cell.nexsroot.nexslog.tailseqn.size) - await cell.sync() + await cell.setHiveKey(('foo', 'bar'), 2) # new item is added to the right log self.len(2, cell.nexsroot.nexslog._ranges) @@ -1189,6 +1316,7 @@ async def test_cell_diag_info(self): self.nn(slab['mapsize']) self.nn(slab['readonly']) self.nn(slab['readahead']) + self.nn(slab['lockmemory']) self.nn(slab['recovering']) async def test_cell_system_info(self): @@ -1217,6 +1345,17 @@ async def test_cell_system_info(self): 'cellapprdisk', 'totalmem', 'availmem'): self.lt(0, info.get(prop)) + async def test_cell_hiveapi(self): + + async with self.getTestCell() as cell: + + await cell.setHiveKey(('foo', 'bar'), 10) + await cell.setHiveKey(('foo', 'baz'), 30) + + async with cell.getLocalProxy() as proxy: + self.eq((), await proxy.getHiveKeys(('lulz',))) + self.eq((('bar', 10), ('baz', 30)), await proxy.getHiveKeys(('foo',))) + async def test_cell_confprint(self): async with self.withSetLoggingMock(): @@ -1238,7 +1377,7 @@ async def test_cell_confprint(self): self.isin('...cell API (https): 0', buf) conf = { - 'dmon:listen': None, + 'dmon:listen': 'tcp://0.0.0.0:0', 'https:port': None, } s_common.yamlsave(conf, dirn, 'cell.yaml') @@ -1248,21 +1387,22 @@ async def test_cell_confprint(self): pass stream.seek(0) buf = stream.read() - self.isin(f'...cell API (telepath): tcp://0.0.0.0:27492', buf) + self.isin('...cell API (telepath): tcp://0.0.0.0:0', buf) self.isin('...cell API (https): disabled', buf) async def test_cell_initargv_conf(self): async with self.withSetLoggingMock(): with self.setTstEnvars(SYN_CELL_NEXSLOG_EN='true', - SYN_CELL_DMON_LISTEN='null', + SYN_CELL_DMON_LISTEN='tcp://0.0.0.0:0', SYN_CELL_HTTPS_PORT='null', SYN_CELL_AUTH_PASSWD='notsecret', ): with self.getTestDir() as dirn: - s_common.yamlsave({'dmon:listen': 'tcp://0.0.0.0:0/', + s_common.yamlsave({'dmon:listen': 'tcp://0.0.0.0:12345/', 'aha:name': 'some:cell'}, dirn, 'cell.yaml') - s_common.yamlsave({}, dirn, 'cell.mods.yaml') + s_common.yamlsave({'nexslog:async': True}, + dirn, 'cell.mods.yaml') async with await s_cell.Cell.initFromArgv([dirn, '--auth-passwd', 'secret']) as cell: # config order for booting from initArgV # 0) cell.mods.yaml @@ -1270,7 +1410,8 @@ async def test_cell_initargv_conf(self): # 2) envars # 3) cell.yaml self.true(cell.conf.req('nexslog:en')) - self.none(cell.conf.req('dmon:listen')) + self.true(cell.conf.req('nexslog:async')) + self.eq(cell.conf.req('dmon:listen'), 'tcp://0.0.0.0:0') self.none(cell.conf.req('https:port')) self.eq(cell.conf.req('aha:name'), 'some:cell') root = cell.auth.rootuser @@ -1342,11 +1483,11 @@ async def test_cell_backup(self): self.none(info['lastexception']) with self.raises(s_exc.BadArg): - await proxy.runBackup(name='../woot') + await proxy.runBackup('../woot') with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 0.1): with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeperProc)): - await self.asyncraises(s_exc.SynErr, proxy.runBackup(name='_sleeperProc')) + await self.asyncraises(s_exc.SynErr, proxy.runBackup('_sleeperProc')) info = await proxy.getBackupInfo() errinfo = info.get('lastexception') @@ -1358,7 +1499,7 @@ async def test_cell_backup(self): with mock.patch.object(s_cell.Cell, 'BACKUP_SPAWN_TIMEOUT', 8.0): with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_sleeper2Proc)): - await self.asyncraises(s_exc.SynErr, proxy.runBackup(name='_sleeper2Proc')) + await self.asyncraises(s_exc.SynErr, proxy.runBackup('_sleeper2Proc')) info = await proxy.getBackupInfo() laststart2 = info['laststart'] @@ -1368,7 +1509,7 @@ async def test_cell_backup(self): self.eq(errinfo['errinfo']['mesg'], 'backup subprocess start timed out') with mock.patch.object(s_cell.Cell, '_backupProc', staticmethod(_exiterProc)): - await self.asyncraises(s_exc.SpawnExit, proxy.runBackup(name='_exiterProc')) + await self.asyncraises(s_exc.SpawnExit, proxy.runBackup('_exiterProc')) info = await proxy.getBackupInfo() laststart3 = info['laststart'] @@ -1466,7 +1607,7 @@ async def err(*args, **kwargs): with mock.patch('synapse.lib.coro.executor', err): with self.raises(s_exc.SynErr) as cm: - await proxy.runBackup(name='partial') + await proxy.runBackup('partial') self.eq(cm.exception.get('errx'), 'RuntimeError') self.isin('partial', await proxy.getBackups()) @@ -1518,6 +1659,19 @@ async def test_cell_tls_client(self): async with await s_telepath.openurl(url) as proxy: pass + async def test_cell_auth_ctor(self): + conf = { + 'auth:ctor': 'synapse.tests.test_lib_cell.altAuthCtor', + 'auth:conf': { + 'foo': 'bar', + }, + } + with self.getTestDir() as dirn: + async with await s_cell.Cell.anit(dirn, conf=conf) as cell: + self.eq('faz', cell.conf.get('auth:conf')['baz']) + await cell.auth.addUser('visi') + await cell._storCellAuthMigration() + async def test_cell_auth_userlimit(self): maxusers = 3 conf = { @@ -1742,7 +1896,7 @@ async def _fakeBackup(self, name=None, wait=True): s_common.gendir(os.path.join(backdirn, name)) with mock.patch.object(s_cell.Cell, 'runBackup', _fakeBackup): - arch = s_t_utils.alist(proxy.iterNewBackupArchive(name='nobkup')) + arch = s_t_utils.alist(proxy.iterNewBackupArchive('nobkup')) with self.raises(asyncio.TimeoutError): await asyncio.wait_for(arch, timeout=0.1) @@ -1751,7 +1905,7 @@ async def _slowFakeBackup(self, name=None, wait=True): await asyncio.sleep(3.0) with mock.patch.object(s_cell.Cell, 'runBackup', _slowFakeBackup): - arch = s_t_utils.alist(proxy.iterNewBackupArchive(name='nobkup2')) + arch = s_t_utils.alist(proxy.iterNewBackupArchive('nobkup2')) with self.raises(asyncio.TimeoutError): await asyncio.wait_for(arch, timeout=0.1) @@ -1773,17 +1927,17 @@ async def _iterNewDup(self, user, name=None, remove=False): with mock.patch.object(s_cell.Cell, 'runBackup', _slowFakeBackup2): with mock.patch.object(s_cell.Cell, 'iterNewBackupArchive', _iterNewDup): - arch = s_t_utils.alist(proxy.iterNewBackupArchive(name='dupbackup', remove=True)) + arch = s_t_utils.alist(proxy.iterNewBackupArchive('dupbackup', remove=True)) task = core.schedCoro(arch) await asyncio.wait_for(evt0.wait(), timeout=2) - fail = s_t_utils.alist(proxy.iterNewBackupArchive(name='alreadystreaming', remove=True)) + fail = s_t_utils.alist(proxy.iterNewBackupArchive('alreadystreaming', remove=True)) await self.asyncraises(s_exc.BackupAlreadyRunning, fail) task.cancel() await asyncio.wait_for(evt1.wait(), timeout=2) with self.raises(s_exc.BadArg): - async for msg in proxy.iterNewBackupArchive(name='bkup'): + async for msg in proxy.iterNewBackupArchive('bkup'): pass # Get an existing backup @@ -1796,7 +1950,7 @@ async def _iterNewDup(self, user, name=None, remove=False): self.len(1, nodes) with open(bkuppath2, 'wb') as bkup2: - async for msg in proxy.iterNewBackupArchive(name='bkup2'): + async for msg in proxy.iterNewBackupArchive('bkup2'): bkup2.write(msg) self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups())) @@ -1807,7 +1961,7 @@ async def _iterNewDup(self, user, name=None, remove=False): self.len(1, nodes) with open(bkuppath3, 'wb') as bkup3: - async for msg in proxy.iterNewBackupArchive(name='bkup3', remove=True): + async for msg in proxy.iterNewBackupArchive('bkup3', remove=True): self.true(core.backupstreaming) bkup3.write(msg) @@ -1841,16 +1995,16 @@ async def streamdone(): self.eq(('bkup', 'bkup2'), sorted(await proxy.getBackups())) # Start another backup while one is already running - bkup = s_t_utils.alist(proxy.iterNewBackupArchive(name='runbackup', remove=True)) + bkup = s_t_utils.alist(proxy.iterNewBackupArchive('runbackup', remove=True)) task = core.schedCoro(bkup) await asyncio.sleep(0) - fail = s_t_utils.alist(proxy.iterNewBackupArchive(name='alreadyrunning', remove=True)) + fail = s_t_utils.alist(proxy.iterNewBackupArchive('alreadyrunning', remove=True)) await self.asyncraises(s_exc.BackupAlreadyRunning, fail) await asyncio.wait_for(task, 5) with tarfile.open(bkuppath, 'r:gz') as tar: - tar.extractall(path=dirn, filter='data') + tar.extractall(path=dirn) bkupdirn = os.path.join(dirn, 'bkup') async with self.getTestCore(dirn=bkupdirn) as core: @@ -1861,7 +2015,7 @@ async def streamdone(): self.len(0, nodes) with tarfile.open(bkuppath2, 'r:gz') as tar: - tar.extractall(path=dirn, filter='data') + tar.extractall(path=dirn) bkupdirn2 = os.path.join(dirn, 'bkup2') async with self.getTestCore(dirn=bkupdirn2) as core: @@ -1869,7 +2023,7 @@ async def streamdone(): self.len(1, nodes) with tarfile.open(bkuppath3, 'r:gz') as tar: - tar.extractall(path=dirn, filter='data') + tar.extractall(path=dirn) bkupdirn3 = os.path.join(dirn, 'bkup3') async with self.getTestCore(dirn=bkupdirn3) as core: @@ -1878,7 +2032,7 @@ async def streamdone(): with tarfile.open(bkuppath4, 'r:gz') as tar: bkupname = os.path.commonprefix(tar.getnames()) - tar.extractall(path=dirn, filter='data') + tar.extractall(path=dirn) bkupdirn4 = os.path.join(dirn, bkupname) async with self.getTestCore(dirn=bkupdirn4) as core: @@ -1897,11 +2051,11 @@ async def streamdone(): bkup5.write(msg) with mock.patch('synapse.lib.cell._iterBackupProc', _backupEOF): - await s_t_utils.alist(proxy.iterNewBackupArchive(name='eof', remove=True)) + await s_t_utils.alist(proxy.iterNewBackupArchive('eof', remove=True)) with tarfile.open(bkuppath5, 'r:gz') as tar: bkupname = os.path.commonprefix(tar.getnames()) - tar.extractall(path=dirn, filter='data') + tar.extractall(path=dirn) bkupdirn5 = os.path.join(dirn, bkupname) async with self.getTestCore(dirn=bkupdirn5) as core: @@ -2079,7 +2233,7 @@ async def test_backup_restore_base(self): # Make our first backup async with self.getTestCore() as core: - self.len(1, await core.nodes('[inet:ip=1.2.3.4]')) + self.len(1, await core.nodes('[inet:ipv4=1.2.3.4]')) # Punch in a value to the cell.yaml to ensure it persists core.conf['storm:log'] = True @@ -2107,14 +2261,14 @@ async def test_backup_restore_base(self): argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: self.true(await stream.wait(6)) - self.len(1, await core.nodes('inet:ip=1.2.3.4')) + self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) self.true(core.conf.get('storm:log')) # Turning the service back on with the restore URL is fine too. with self.getAsyncLoggerStream('synapse.lib.cell') as stream: argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: - self.len(1, await core.nodes('inet:ip=1.2.3.4')) + self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) # Take a backup of the cell with the restore.done file in place async with await axon.upload() as upfd: @@ -2144,7 +2298,7 @@ async def test_backup_restore_base(self): argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: self.true(await stream.wait(6)) - self.len(1, await core.nodes('inet:ip=1.2.3.4')) + self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) # Restore a backup which has an existing restore.done file in it - that marker file will get overwritten furl2 = f'{url}{s_common.ehex(sha256r)}' @@ -2156,7 +2310,7 @@ async def test_backup_restore_base(self): argv = [cdir, '--https', '0', '--telepath', 'tcp://127.0.0.1:0'] async with await s_cortex.Cortex.initFromArgv(argv) as core: self.true(await stream.wait(6)) - self.len(1, await core.nodes('inet:ip=1.2.3.4')) + self.len(1, await core.nodes('inet:ipv4=1.2.3.4')) rpath = s_common.genpath(cdir, 'restore.done') with s_common.genfile(rpath) as fd: @@ -2372,6 +2526,73 @@ async def test_backup_restore_double_promote_aha(self): self.len(1, await bcree01.nodes('[inet:asn=8675]')) self.len(1, await bcree00.nodes('inet:asn=8675')) + async def test_passwd_regression(self): + # Backwards compatibility test for shadowv2 + # Cell was created prior to the shadowv2 password change. + with self.getRegrDir('cells', 'passwd-2.109.0') as dirn: + async with self.getTestCell(s_cell.Cell, dirn=dirn) as cell: # type: s_cell.Cell + root = await cell.auth.getUserByName('root') + shadow = root.info.get('passwd') + self.isinstance(shadow, tuple) + self.len(2, shadow) + + # Old password works and is migrated to the new password scheme + self.false(await root.tryPasswd('newp')) + self.true(await root.tryPasswd('root')) + shadow = root.info.get('passwd') + self.isinstance(shadow, dict) + self.eq(shadow.get('type'), s_passwd.DEFAULT_PTYP) + + # Logging back in works + self.true(await root.tryPasswd('root')) + + user = await cell.auth.getUserByName('user') + + # User can login with their regular password. + shadow = user.info.get('passwd') + self.isinstance(shadow, tuple) + self.true(await user.tryPasswd('secret1234')) + shadow = user.info.get('passwd') + self.isinstance(shadow, dict) + + # User has a 10 year duration onepass value available. + onepass = '0f327906fe0221a7f582744ad280e1ca' + self.true(await user.tryPasswd(onepass)) + self.false(await user.tryPasswd(onepass)) + + # Passwords can be changed as well. + await user.setPasswd('hehe') + self.true(await user.tryPasswd('hehe')) + self.false(await user.tryPasswd('secret1234')) + + # Password policies do not prevent live migration of an existing password + with self.getRegrDir('cells', 'passwd-2.109.0') as dirn: + policy = {'complexity': {'length': 5}} + conf = {'auth:passwd:policy': policy} + async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell: # type: s_cell.Cell + root = await cell.auth.getUserByName('root') + shadow = root.info.get('passwd') + self.isinstance(shadow, tuple) + self.len(2, shadow) + + # Old password works and is migrated to the new password scheme + self.false(await root.tryPasswd('newp')) + self.true(await root.tryPasswd('root')) + shadow = root.info.get('passwd') + self.isinstance(shadow, dict) + self.eq(shadow.get('type'), s_passwd.DEFAULT_PTYP) + + # Pre-nexus changes of root via auth:passwd work too. + with self.getRegrDir('cells', 'passwd-2.109.0') as dirn: + conf = {'auth:passwd': 'supersecretpassword'} + async with self.getTestCell(s_cell.Cell, dirn=dirn, conf=conf) as cell: # type: s_cell.Cell + root = await cell.auth.getUserByName('root') + shadow = root.info.get('passwd') + self.isinstance(shadow, dict) + self.eq(shadow.get('type'), s_passwd.DEFAULT_PTYP) + self.false(await root.tryPasswd('root')) + self.true(await root.tryPasswd('supersecretpassword')) + async def test_cell_minspace(self): with self.raises(s_exc.LowSpace): @@ -2437,7 +2658,7 @@ async def wrapDelWriteHold(root, reason): conf = {'limit:disk:free': 0} async with self.getTestCore(dirn=path00, conf=conf) as core00: - await core00.nodes('[ inet:ip=1.2.3.4 ]') + await core00.nodes('[ inet:ipv4=1.2.3.4 ]') s_tools_backup.backup(path00, path01) @@ -2457,21 +2678,21 @@ async def wrapDelWriteHold(root, reason): self.stormIsInErr(errmsg, msgs) msgs = await core01.stormlist('[inet:fqdn=newp.fail]') self.stormIsInErr(errmsg, msgs) - self.len(1, await core00.nodes('[ inet:ip=2.3.4.5 ]')) + self.len(1, await core00.nodes('[ inet:ipv4=2.3.4.5 ]')) offs = await core00.getNexsIndx() self.false(await core01.waitNexsOffs(offs, 1)) - self.len(1, await core01.nodes('inet:ip=1.2.3.4')) - self.len(0, await core01.nodes('inet:ip=2.3.4.5')) + self.len(1, await core01.nodes('inet:ipv4=1.2.3.4')) + self.len(0, await core01.nodes('inet:ipv4=2.3.4.5')) revt.clear() revt.clear() self.true(await asyncio.wait_for(revt.wait(), 1)) await core01.sync() - self.len(1, await core01.nodes('inet:ip=1.2.3.4')) - self.len(1, await core01.nodes('inet:ip=2.3.4.5')) + self.len(1, await core01.nodes('inet:ipv4=1.2.3.4')) + self.len(1, await core01.nodes('inet:ipv4=2.3.4.5')) with mock.patch.object(s_cell.Cell, 'FREE_SPACE_CHECK_FREQ', 600): @@ -2485,9 +2706,9 @@ async def wrapDelWriteHold(root, reason): with mock.patch('shutil.disk_usage', full_disk): opts = {'view': viewiden} - msgs = await core.stormlist('for $x in $lib.range(20000) {[inet:ip=([4, $x])]}', opts=opts) + msgs = await core.stormlist('for $x in $lib.range(20000) {[inet:ipv4=$x]}', opts=opts) self.stormIsInErr(errmsg, msgs) - nodes = await core.nodes('inet:ip', opts=opts) + nodes = await core.nodes('inet:ipv4', opts=opts) self.gt(len(nodes), 0) self.lt(len(nodes), 20000) @@ -2505,7 +2726,7 @@ def spaceexc(self): opts = {'view': viewiden} with self.getAsyncLoggerStream('synapse.lib.lmdbslab', - 'Error during slab resize callback - foo') as stream: + 'Error during slab resize callback - foo') as stream: msgs = await core.stormlist('for $x in $lib.range(200) {[test:int=$x]}', opts=opts) self.true(await stream.wait(timeout=30)) @@ -2829,8 +3050,8 @@ async def test_cell_user_api_key(self): # Verify duration arg for expiration is applied with self.raises(s_exc.BadArg): await cell.addUserApiKey(root, 'newp', duration=0) - rtk1, rtdf1 = await cell.addUserApiKey(root, 'Expiring Token', duration=200000) - self.eq(rtdf1.get('expires'), rtdf1.get('updated') + 200000) + rtk1, rtdf1 = await cell.addUserApiKey(root, 'Expiring Token', duration=200) + self.eq(rtdf1.get('expires'), rtdf1.get('updated') + 200) isok, info = await cell.checkUserApiKey(rtk1) self.true(isok) @@ -2981,6 +3202,105 @@ async def test_cell_iter_slab_data(self): data = await s_t_utils.alist(cell.iterSlabData('hehe', prefix='yup')) self.eq(data, [('wow', 'yes')]) + async def test_cell_nexus_compat(self): + with mock.patch('synapse.lib.cell.NEXUS_VERSION', (0, 0)): + async with self.getRegrCore('hive-migration') as core0: + with mock.patch('synapse.lib.cell.NEXUS_VERSION', (2, 177)): + conf = {'mirror': core0.getLocalUrl()} + async with self.getRegrCore('hive-migration', conf=conf) as core1: + await core1.sync() + + await core1.nodes('$lib.user.vars.set(foo, bar)') + self.eq('bar', await core0.callStorm('return($lib.user.vars.get(foo))')) + + await core1.nodes('$lib.user.vars.pop(foo)') + self.none(await core0.callStorm('return($lib.user.vars.get(foo))')) + + await core1.nodes('$lib.user.profile.set(bar, baz)') + self.eq('baz', await core0.callStorm('return($lib.user.profile.get(bar))')) + + await core1.nodes('$lib.user.profile.pop(bar)') + self.none(await core0.callStorm('return($lib.user.profile.get(bar))')) + + self.eq((0, 0), core1.nexsvers) + await core0.setNexsVers((2, 177)) + await core1.sync() + self.eq((2, 177), core1.nexsvers) + + await core1.nodes('$lib.user.vars.set(foo, bar)') + self.eq('bar', await core0.callStorm('return($lib.user.vars.get(foo))')) + + await core1.nodes('$lib.user.vars.pop(foo)') + self.none(await core0.callStorm('return($lib.user.vars.get(foo))')) + + await core1.nodes('$lib.user.profile.set(bar, baz)') + self.eq('baz', await core0.callStorm('return($lib.user.profile.get(bar))')) + + await core1.nodes('$lib.user.profile.pop(bar)') + self.none(await core0.callStorm('return($lib.user.profile.get(bar))')) + + async def test_cell_hive_migration(self): + + with self.getAsyncLoggerStream('synapse.lib.cell') as stream: + + async with self.getRegrCore('hive-migration') as core: + visi = await core.auth.getUserByName('visi') + asvisi = {'user': visi.iden} + + valu = await core.callStorm('return($lib.user.vars.get(foovar))', opts=asvisi) + self.eq('barvalu', valu) + + valu = await core.callStorm('return($lib.user.profile.get(fooprof))', opts=asvisi) + self.eq('barprof', valu) + + msgs = await core.stormlist('cron.list') + self.stormIsInPrint(' visi 8437c35a.. ', msgs) + self.stormIsInPrint('[tel:mob:telem=*]', msgs) + + msgs = await core.stormlist('dmon.list') + self.stormIsInPrint('0973342044469bc40b577969028c5079: (foodmon ): running', msgs) + + msgs = await core.stormlist('trigger.list') + self.stormIsInPrint('visi 27f5dc524e7c3ee8685816ddf6ca1326', msgs) + self.stormIsInPrint('[ +#count test:str=$tag ]', msgs) + + msgs = await core.stormlist('testcmd0 foo') + self.stormIsInPrint('foo haha', msgs) + + msgs = await core.stormlist('testcmd1') + self.stormIsInPrint('hello', msgs) + + msgs = await core.stormlist('model.deprecated.locks') + self.stormIsInPrint('ou:hasalias', msgs) + + nodes = await core.nodes('_visi:int') + self.len(1, nodes) + node = nodes[0] + self.eq(node.get('tick'), 1577836800000,) + self.eq(node.get('._woot'), 5) + self.nn(node.getTagProp('test', 'score'), 6) + + self.maxDiff = None + roles = s_t_utils.deguidify('[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}]') + users = s_t_utils.deguidify('[{"type":"user","iden":"a357138db50780b62093a6ce0d057fd8","name":"root","rules":[],"roles":[],"admin":true,"email":null,"locked":false,"archived":false,"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"admin":true},"4d50eb257549436414643a71e057091a":{"admin":true}}},{"type":"user","iden":"f77ac6744671a845c27e571071877827","name":"visi","rules":[[true,["cron","add"]],[true,["dmon","add"]],[true,["trigger","add"]]],"roles":[{"type":"role","iden":"e1ef725990aa62ae3c4b98be8736d89f","name":"all","rules":[],"authgates":{"46cfde2c1682566602860f8df7d0cc83":{"rules":[[true,["layer","read"]]]},"4d50eb257549436414643a71e057091a":{"rules":[[true,["view","read"]]]}}}],"admin":false,"email":null,"locked":false,"archived":false,"authgates":{"f21b7ae79c2dacb89484929a8409e5d8":{"admin":true},"d7d0380dd4e743e35af31a20d014ed48":{"admin":true}}}]') + gates = s_t_utils.deguidify('[{"iden":"46cfde2c1682566602860f8df7d0cc83","type":"layer","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["layer","read"]]],"admin":false}]},{"iden":"d7d0380dd4e743e35af31a20d014ed48","type":"trigger","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"f21b7ae79c2dacb89484929a8409e5d8","type":"cronjob","users":[{"iden":"f77ac6744671a845c27e571071877827","rules":[],"admin":true}],"roles":[]},{"iden":"4d50eb257549436414643a71e057091a","type":"view","users":[{"iden":"a357138db50780b62093a6ce0d057fd8","rules":[],"admin":true}],"roles":[{"iden":"e1ef725990aa62ae3c4b98be8736d89f","rules":[[true,["view","read"]]],"admin":false}]},{"iden":"cortex","type":"cortex","users":[],"roles":[]}]') + + self.eq(roles, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.roles.list())')).decode())) + self.eq(users, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.users.list())')).decode())) + self.eq(gates, s_t_utils.deguidify(s_json.dumps(await core.callStorm('return($lib.auth.gates.list())')).decode())) + + with self.raises(s_exc.BadTypeValu): + await core.nodes('[ it:dev:str=foo +#test.newp ]') + + stream.seek(0) + data = stream.getvalue() + newprole = s_common.guid('newprole') + newpuser = s_common.guid('newpuser') + + self.isin(f'Unknown user {newpuser} on gate', data) + self.isin(f'Unknown role {newprole} on gate', data) + self.isin(f'Unknown role {newprole} on user', data) + async def test_cell_check_sysctl(self): sysctls = s_linux.getSysctls() @@ -3204,15 +3524,11 @@ async def test_lib_cell_sadaha(self): async with self.getTestCell() as cell: self.none(await cell.getAhaProxy()) + cell.ahaclient = await cell.enter_context(await s_telepath.Client.anit('cell:///tmp/newp')) - class MockClient: - async def proxy(self, timeout=None): - raise s_exc.LinkShutDown(mesg='client connection failed') - - cell.ahaclient = MockClient() - - with self.raises(s_exc.LinkShutDown): - self.none(await cell.getAhaProxy()) + # coverage for failure of aha client to connect + with self.raises(TimeoutError): + self.none(await cell.getAhaProxy(timeout=0.1)) async def test_stream_backup_exception(self): @@ -3261,7 +3577,7 @@ async def mock_runBackup(*args, **kwargs): with mock.patch.object(s_cell.Cell, 'runBackup', mock_runBackup): with self.getAsyncLoggerStream('synapse.lib.cell', 'Removing') as stream: with self.raises(s_exc.SynErr) as cm: - async for _ in proxy.iterNewBackupArchive(name='failedbackup', remove=True): + async for _ in proxy.iterNewBackupArchive('failedbackup', remove=True): pass self.isin('backup failed', str(cm.exception)) @@ -3274,7 +3590,7 @@ async def mock_runBackup(*args, **kwargs): core.backupstreaming = True with self.raises(s_exc.BackupAlreadyRunning): - async for _ in proxy.iterNewBackupArchive(name='newbackup', remove=True): + async for _ in proxy.iterNewBackupArchive('newbackup', remove=True): pass async def test_cell_peer_noaha(self): @@ -3328,3 +3644,47 @@ async def sleep99(cell): self.none(await cell00.getTask(task01)) self.false(await cell00.killTask(task01)) + + async def test_cell_fini_order(self): + + with self.getTestDir() as dirn: + + data = [] + conf = {'nexslog:en': True} + + async with self.getTestCell(dirn=dirn, conf=conf) as cell: + + event00 = asyncio.Event() + + async def coro(): + try: + event00.set() + await asyncio.sleep(100000) + except asyncio.CancelledError: + # nexus txn can run in a activeTask handler + await cell.sync() + data.append('activetask_cancelled') + + async def bg_coro(): + self.true(await cell.waitfini(timeout=12)) + data.append('cell_fini') + return True + + bg_task = s_coro.create_task(bg_coro()) + cell.runActiveTask(coro()) + self.true(await asyncio.wait_for(event00.wait(), timeout=6)) + + # Perform a non-sync nexus txn then teardown the cell via __aexit__ + self.nn(await cell.addUser('someuser')) + + self.true(await asyncio.wait_for(bg_task, timeout=6)) + + self.eq(data, ['activetask_cancelled', 'cell_fini']) + + async with self.getTestCell(dirn=dirn, conf=conf) as cell: + offs = await cell.getNexsIndx() + items = [] + async for offs, item in cell.getNexusChanges(offs - 1, wait=False): + items.append(item) + self.len(1, items) + self.eq('sync', items[0][1]) From 951d8bb06e80fdcd9f2edd23b1eb671e9408a47f Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Thu, 19 Feb 2026 14:29:12 -0500 Subject: [PATCH 03/15] updates --- synapse/lib/link.py | 17 ++++++++++++++ synapse/tests/test_lib_agenda.py | 6 ++--- synapse/tests/test_lib_base.py | 35 ++++++++++++++++++++++++++++ synapse/tests/test_lib_cell.py | 3 ++- synapse/tests/test_lib_stormtypes.py | 7 +++--- 5 files changed, 60 insertions(+), 8 deletions(-) diff --git a/synapse/lib/link.py b/synapse/lib/link.py index daf87ad440c..2d7b141d33c 100644 --- a/synapse/lib/link.py +++ b/synapse/lib/link.py @@ -45,6 +45,23 @@ async def unixconnect(path): info = {'path': path, 'unix': True} return await Link.anit(reader, writer, info=info) +async def unixwait(path): + + while True: + try: + + reader, writer = await asyncio.open_unix_connection(path=path) + + reader._transport.abort() + + writer.close() + await writer.wait_closed() + + return + + except (ConnectionRefusedError, FileNotFoundError): + await asyncio.sleep(0.01) + async def linkfile(mode='wb'): ''' Connect a socketpair to a file-object and return (link, file). diff --git a/synapse/tests/test_lib_agenda.py b/synapse/tests/test_lib_agenda.py index 959b830da85..bb3a49db368 100644 --- a/synapse/tests/test_lib_agenda.py +++ b/synapse/tests/test_lib_agenda.py @@ -165,10 +165,10 @@ def timetime(): def looptime(): return unixtime - MONO_DELT - loop = asyncio.get_running_loop() - with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime), self.getTestDir() as dirn: + async with self.getTestCore() as core: - async with self.getTestCore() as core: + loop = asyncio.get_running_loop() + with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime): visi = await core.auth.addUser('visi') await visi.setAdmin(True) diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index faa8a4df7f1..f8bb0dbc1f5 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -10,6 +10,7 @@ import synapse.lib.base as s_base import synapse.lib.coro as s_coro import synapse.lib.scope as s_scope +import synapse.lib.spawner as s_spawner import synapse.tests.utils as s_t_utils @@ -45,6 +46,14 @@ async def postAnit(self): if self.foo == -1: raise s_exc.BadArg(mesg='boom') +class Haha(s_base.Base, s_spawner.SpawnerMixin): + + async def __anit__(self): + await s_base.Base.__anit__(self) + + async def fini(self): + await s_base.Base.fini(self) + class BaseTest(s_t_utils.SynTest): async def test_base_basics(self): @@ -570,3 +579,29 @@ async def func3(bobj, key, valu): # The scope data set in the task is not present outside of it. self.none(s_scope.get('hehe')) + + async def test_base_spawner_fini(self): + + # Test proxy fini, base should fini + base = await s_base.Base.anit() + spawner = Haha.spawner(base=base) + proxy = await spawner() + + self.false(proxy.isfini) + self.false(base.isfini) + + await proxy.fini() + self.true(proxy.isfini) + self.true(base.isfini) + + # Test base fini, proxy should fini + base = await s_base.Base.anit() + spawner = Haha.spawner(base=base) + proxy = await spawner() + + self.false(proxy.isfini) + self.false(base.isfini) + + await base.fini() + self.true(base.isfini) + self.true(proxy.isfini) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 353d6793695..cec27da4a7f 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -28,6 +28,7 @@ import synapse.lib.coro as s_coro import synapse.lib.json as s_json import synapse.lib.link as s_link +import synapse.lib.const as s_const import synapse.lib.drive as s_drive import synapse.lib.nexus as s_nexus import synapse.lib.config as s_config @@ -848,7 +849,7 @@ async def test_longpath(self): # but exercises the long-path failure inside of the cell's daemon # instead. with self.getTestDir() as dirn: - extrapath = 108 * 'A' + extrapath = s_const.UNIX_PATH_MAX * 'A' longdirn = s_common.genpath(dirn, extrapath) with self.getAsyncLoggerStream('synapse.lib.cell', 'LOCAL UNIX SOCKET WILL BE UNAVAILABLE') as stream: async with self.getTestCell(s_cell.Cell, dirn=longdirn) as cell: diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index af57db570a7..df1a1c6c516 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -5691,11 +5691,10 @@ def timetime(): def looptime(): return unixtime - MONO_DELT - loop = asyncio.get_running_loop() - - with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime): + async with self.getTestCoreAndProxy() as (core, prox): - async with self.getTestCoreAndProxy() as (core, prox): + loop = asyncio.get_running_loop() + with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime): mesgs = await core.stormlist('cron.list') self.stormIsInPrint('No cron jobs found', mesgs) From 2181317d784eca95f806b2918e6e9be4a3768bfc Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:03:39 -0500 Subject: [PATCH 04/15] updates --- synapse/lib/cell.py | 80 ++++++++++++++++++++++------------ synapse/lib/drive.py | 10 ++--- synapse/tests/test_lib_cell.py | 30 ++++++++----- 3 files changed, 75 insertions(+), 45 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index d7cda728924..1117cac7487 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -2,6 +2,7 @@ import os import ssl import copy +import stat import time import fcntl import shutil @@ -1181,6 +1182,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): s_telepath.Aware.__init__(self) self.dirn = s_common.gendir(dirn) + self.sockdirn = s_common.gendir(dirn, 'sockets') + self.runid = s_common.guid() self.auth = None @@ -1672,22 +1675,33 @@ def _delTmpFiles(self): tdir = s_common.gendir(self.dirn, 'tmp') names = os.listdir(tdir) - if not names: - return + if names: + logger.warning(f'Removing {len(names)} temporary files/folders in: {tdir}') - logger.warning(f'Removing {len(names)} temporary files/folders in: {tdir}') + for name in names: - for name in names: + path = os.path.join(tdir, name) - path = os.path.join(tdir, name) + if os.path.isfile(path): + os.unlink(path) + continue - if os.path.isfile(path): - os.unlink(path) - continue + if os.path.isdir(path): + shutil.rmtree(path, ignore_errors=True) + continue - if os.path.isdir(path): - shutil.rmtree(path, ignore_errors=True) - continue + names = os.listdir(self.sockdirn) + if names: + logger.info(f'Removing {len(names)} old sockets in: {self.sockdirn}') + for name in names: + path = os.path.join(self.sockdirn, name) + try: + if stat.S_ISSOCK(os.stat(path).st_mode): + os.unlink(path) + except OSError: # pragma: no cover + pass + + # FIXME - recursively remove sockets dir here? async def _execCellUpdates(self): # implement to apply updates to a fully initialized active cell @@ -1899,13 +1913,24 @@ async def initServiceEarly(self): pass async def initCellStorage(self): - self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') - await self._bumpCellVers('drive:storage', ( - (1, self._drivePermMigration), - ), nexs=False) + path = s_common.gendir(self.dirn, 'slabs', 'drive.lmdb') + sockpath = s_common.genpath(self.sockdirn, 'drive') + + if len(sockpath) > s_const.UNIX_PATH_MAX: + sockpath = None + + spawner = s_drive.FileDrive.spawner(base=self, sockpath=sockpath) + + self.drive = await spawner(path) self.onfini(self.drive.fini) + #TODO: needs migration + #self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') + #await self._bumpCellVers('drive:storage', ( + # (1, self._drivePermMigration), + #), nexs=False) + async def addDriveItem(self, info, path=None, reldir=s_drive.rootdir): iden = info.get('iden') @@ -1922,7 +1947,7 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): # replay safety... iden = info.get('iden') - if self.drive.hasItemInfo(iden): # pragma: no cover + if await self.drive.hasItemInfo(iden): # pragma: no cover return await self.drive.getItemPath(iden) # TODO: Remove this in synapse-3xx @@ -1935,10 +1960,10 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): return await self.drive.addItemInfo(info, path=path, reldir=reldir) async def getDriveInfo(self, iden, typename=None): - return self.drive.getItemInfo(iden, typename=typename) + return await self.drive.getItemInfo(iden, typename=typename) - def reqDriveInfo(self, iden, typename=None): - return self.drive.reqItemInfo(iden, typename=typename) + async def reqDriveInfo(self, iden, typename=None): + return await self.drive.reqItemInfo(iden, typename=typename) async def getDrivePath(self, path, reldir=s_drive.rootdir): ''' @@ -1961,15 +1986,14 @@ async def addDrivePath(self, path, perm=None, reldir=s_drive.rootdir): ''' tick = s_common.now() user = self.auth.rootuser.iden - path = self.drive.getPathNorm(path) + path = await self.drive.getPathNorm(path) if perm is None: perm = {'users': {}, 'roles': {}} for name in path: - info = self.drive.getStepInfo(reldir, name) - await asyncio.sleep(0) + info = await self.drive.getStepInfo(reldir, name) if info is not None: reldir = info.get('iden') @@ -1992,7 +2016,7 @@ async def getDriveData(self, iden, vers=None): Return the data associated with the drive item by iden. If vers is specified, return that specific version. ''' - return self.drive.getItemData(iden, vers=vers) + return await self.drive.getItemData(iden, vers=vers) async def getDriveDataVersions(self, iden): async for item in self.drive.getItemDataVersions(iden): @@ -2000,12 +2024,12 @@ async def getDriveDataVersions(self, iden): @s_nexus.Pusher.onPushAuto('drive:del') async def delDriveInfo(self, iden): - if self.drive.getItemInfo(iden) is not None: + if await self.drive.getItemInfo(iden) is not None: await self.drive.delItemInfo(iden) @s_nexus.Pusher.onPushAuto('drive:set:perm') async def setDriveInfoPerm(self, iden, perm): - return self.drive.setItemPerm(iden, perm) + return await self.drive.setItemPerm(iden, perm) @s_nexus.Pusher.onPushAuto('drive:data:path:set') async def setDriveItemProp(self, iden, vers, path, valu): @@ -2054,7 +2078,7 @@ async def delDriveItemProp(self, iden, vers, path): @s_nexus.Pusher.onPushAuto('drive:set:path') async def setDriveInfoPath(self, iden, path): - path = self.drive.getPathNorm(path) + path = await self.drive.getPathNorm(path) pathinfo = await self.drive.getItemPath(iden) if path == [p.get('name') for p in pathinfo]: return pathinfo @@ -2067,13 +2091,13 @@ async def setDriveData(self, iden, versinfo, data): async def delDriveData(self, iden, vers=None): if vers is None: - info = self.drive.reqItemInfo(iden) + info = await self.drive.reqItemInfo(iden) vers = info.get('version') return await self._push('drive:data:del', iden, vers) @s_nexus.Pusher.onPush('drive:data:del') async def _delDriveData(self, iden, vers): - return self.drive.delItemData(iden, vers) + return await self.drive.delItemData(iden, vers) async def getDriveKids(self, iden): async for info in self.drive.getItemKids(iden): diff --git a/synapse/lib/drive.py b/synapse/lib/drive.py index e051fcaabfd..1a137c3e09c 100644 --- a/synapse/lib/drive.py +++ b/synapse/lib/drive.py @@ -217,7 +217,7 @@ def _setItemPerm(self, bidn, perm): info = self._reqItemInfo(bidn) info['permissions'] = perm s_schemas.reqValidDriveInfo(info) - self.slab._put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) + self.slab.put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) return info async def getPathInfo(self, path, reldir=rootdir): @@ -492,7 +492,7 @@ def _delItemData(self, bidn, vers=None): else: info.update(versinfo) - self.slab._put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) + self.slab.put(LKEY_INFO + bidn, s_msgpack.en(info), db=self.dbname) return info def _getLastDataVers(self, bidn): @@ -548,11 +548,11 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): lkey = LKEY_TYPE + typename.encode() - await self.slab.put(lkey, s_msgpack.en(schema), db=self.dbname) + self.slab.put(lkey, s_msgpack.en(schema), db=self.dbname) if vers is not None: verskey = LKEY_TYPE_VERS + typename.encode() - await self.slab.put(verskey, s_msgpack.en(vers), db=self.dbname) + self.slab.put(verskey, s_msgpack.en(vers), db=self.dbname) if callback is not None: async for info in self.getItemsByType(typename): @@ -562,7 +562,7 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): databyts = self.slab.get(LKEY_DATA + bidn + versindx, db=self.dbname) data = await callback(info, s_msgpack.un(byts), s_msgpack.un(databyts), curv) vtor(data) - await self.slab.put(LKEY_DATA + bidn + versindx, s_msgpack.en(data), db=self.dbname) + self.slab.put(LKEY_DATA + bidn + versindx, s_msgpack.en(data), db=self.dbname) await asyncio.sleep(0) return True diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index cec27da4a7f..2fd462412e2 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -57,6 +57,11 @@ def _exiterProc(pipe, srcdir, dstdir, lmdbpaths, logconf): def _backupSleep(path, linkinfo): time.sleep(3.0) +async def migrate_v1(info, versinfo, data, curv): + assert curv == 1 + data['woot'] = 'woot' + return data + async def _doEOFBackup(path): return @@ -275,9 +280,9 @@ async def test_cell_drive(self): neatrole = await cell.auth.addRole('neatrole') await fooser.grant(neatrole.iden) - with self.raises(s_exc.SchemaViolation): - versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) + # with self.raises(s_exc.SchemaViolation): + # versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + # await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20, 'stuff': 12}) @@ -333,11 +338,10 @@ async def test_cell_drive(self): self.eq(data[1]['stuff'], 1234) # This will be done by the cell in a cell storage version migration... - async def migrate_v1(info, versinfo, data): - data['woot'] = 'woot' - return data + callback = 'synapse.tests.test_lib_cell.migrate_v1' + await cell.drive.setTypeSchema('woot', testDataSchema_v1, callback=callback) - await cell.drive.setTypeSchema('woot', testDataSchema_v1, migrate_v1) + await cell.setDriveItemProp(iden, versinfo, 'woot', 'woot') versinfo['version'] = (1, 1, 1) await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829) @@ -371,6 +375,8 @@ async def migrate_v1(info, versinfo, data): self.none(await cell.delDriveItemProp(iden, versinfo, ('lolnope', 'nopath'))) versinfo, data = await cell.getDriveData(iden, vers=(1, 0, 0)) + print(versinfo) + print(data) self.eq('woot', data.get('woot')) versinfo, data = await cell.getDriveData(iden, vers=(1, 1, 0)) @@ -439,7 +445,7 @@ async def migrate_v1(info, versinfo, data): with self.raises(s_exc.DupName): iden = pathinfo[-2].get('iden') name = pathinfo[-1].get('name') - cell.drive.reqFreeStep(iden, name) + await cell.drive.reqFreeStep(iden, name) walks = [item async for item in cell.drive.walkPathInfo('hehe')] self.len(3, walks) @@ -455,10 +461,10 @@ async def migrate_v1(info, versinfo, data): self.eq('haha', walks[1].get('name')) self.eq('hehe', walks[2].get('name')) - self.none(cell.drive.getTypeSchema('newp')) + self.none(await cell.drive.getTypeSchema('newp')) - cell.drive.validators.pop('woot') - self.nn(cell.drive.getTypeValidator('woot')) + # cell.drive.validators.pop('woot') + # self.nn(cell.drive.getTypeValidator('woot')) # move to root dir pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop') @@ -473,7 +479,7 @@ async def migrate_v1(info, versinfo, data): # explicitly clear out the cache JsValidators, otherwise we get the cached, pre-msgpack # version of the validator, which will be correct and skip the point of this test. s_config._JsValidators.clear() - cell.drive.reqValidData('woot', data) + await cell.drive.reqValidData('woot', data) async def test_cell_auth(self): From 349843e0fb169ba8b1728f03a6b4d576dc58d5b6 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Thu, 19 Feb 2026 16:06:17 -0500 Subject: [PATCH 05/15] lint fix --- synapse/lib/cell.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 1117cac7487..cdc02343415 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1925,11 +1925,11 @@ async def initCellStorage(self): self.onfini(self.drive.fini) - #TODO: needs migration - #self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') - #await self._bumpCellVers('drive:storage', ( + # TODO: needs migration + # self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') + # await self._bumpCellVers('drive:storage', ( # (1, self._drivePermMigration), - #), nexs=False) + # ), nexs=False) async def addDriveItem(self, info, path=None, reldir=s_drive.rootdir): From d43df664c447621adf7b8e522539184699bc80f2 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Fri, 20 Feb 2026 13:25:21 -0500 Subject: [PATCH 06/15] updates --- synapse/lib/cell.py | 46 +++++++++++++++++++++++++++++--------------- synapse/lib/drive.py | 5 ++--- 2 files changed, 33 insertions(+), 18 deletions(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index cdc02343415..74d8f0a2083 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1524,14 +1524,32 @@ async def _storCellAuthMigration(self): logger.warning(f'...Cell ({self.getCellType()}) auth migration complete!') async def _drivePermMigration(self): - for lkey, lval in self.slab.scanByPref(s_drive.LKEY_INFO, db=self.drive.dbname): - info = s_msgpack.un(lval) - perm = info.pop('perm', None) - if perm is not None: - perm.setdefault('users', {}) - perm.setdefault('roles', {}) - info['permissions'] = perm - self.slab.put(lkey, s_msgpack.en(info), db=self.drive.dbname) + async with await s_drive.Drive.anit(self.slab, 'celldrive') as olddrive: + for lkey, lval in self.slab.scanByPref(s_drive.LKEY_INFO, db=olddrive.dbname): + info = s_msgpack.un(lval) + perm = info.pop('perm', None) + if perm is not None: + perm.setdefault('users', {}) + perm.setdefault('roles', {}) + info['permissions'] = perm + self.slab.put(lkey, s_msgpack.en(info), db=olddrive.dbname) + + async def _driveCellMigration(self): + logger.warning('Migrating Drive Slabs') + + self.olddrive = await s_drive.Drive.anit(self.slab, 'celldrive') + + dbname = self.olddrive.dbname + newpath = s_common.gendir(self.dirn, 'slabs', 'drive.lmdb') + + async with await s_lmdbslab.Slab.anit(newpath) as newslab: + rows = await self.slab.copydb(dbname, newslab, dbname) + logger.warning(f"Migrated {rows} rows") + newslab.forcecommit() + + # TODO: trash celldrive + + logger.warning('...Drive migration complete!') def getPermDef(self, perm): perm = tuple(perm) @@ -1913,6 +1931,11 @@ async def initServiceEarly(self): pass async def initCellStorage(self): + await self._bumpCellVers('drive:storage', ( + (1, self._drivePermMigration), + (2, self._driveCellMigration), + ), nexs=False) + path = s_common.gendir(self.dirn, 'slabs', 'drive.lmdb') sockpath = s_common.genpath(self.sockdirn, 'drive') @@ -1920,17 +1943,10 @@ async def initCellStorage(self): sockpath = None spawner = s_drive.FileDrive.spawner(base=self, sockpath=sockpath) - self.drive = await spawner(path) self.onfini(self.drive.fini) - # TODO: needs migration - # self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') - # await self._bumpCellVers('drive:storage', ( - # (1, self._drivePermMigration), - # ), nexs=False) - async def addDriveItem(self, info, path=None, reldir=s_drive.rootdir): iden = info.get('iden') diff --git a/synapse/lib/drive.py b/synapse/lib/drive.py index 1a137c3e09c..2ca7f6204a9 100644 --- a/synapse/lib/drive.py +++ b/synapse/lib/drive.py @@ -10,6 +10,7 @@ import synapse.lib.msgpack as s_msgpack import synapse.lib.schemas as s_schemas import synapse.lib.spawner as s_spawner +import synapse.lib.lmdbslab as s_lmdbslab nameregex = regex.compile(s_schemas.re_drivename) def reqValidName(name): @@ -228,7 +229,6 @@ async def getPathInfo(self, path, reldir=rootdir): This API is designed to allow the caller to retrieve the path info and potentially check permissions on each level to control access. ''' - path = await self.getPathNorm(path) parbidn = s_common.uhex(reldir) @@ -615,6 +615,5 @@ async def reqValidData(self, typename, item): class FileDrive(Drive, s_spawner.SpawnerMixin): async def __anit__(self, path): - import synapse.lib.lmdbslab as s_lmdbslab slab = await s_lmdbslab.Slab.anit(path) - return await Drive.__anit__(self, slab, 'drive') + return await Drive.__anit__(self, slab, 'celldrive') From 3c3c35ea6bbe3dd0c2b8275a2b42ce53b2d293f4 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 25 Feb 2026 11:56:16 -0500 Subject: [PATCH 07/15] updates --- synapse/lib/cell.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 74d8f0a2083..744ed6b7703 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -2096,7 +2096,7 @@ async def setDriveInfoPath(self, iden, path): path = await self.drive.getPathNorm(path) pathinfo = await self.drive.getItemPath(iden) - if path == [p.get('name') for p in pathinfo]: + if [str(p) for p in path] == [p.get('name', '') for p in pathinfo]: return pathinfo return await self.drive.setItemPath(iden, path) From 6aa64bd60e3c4bf249e9ca334d58df8caddc1ea7 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 25 Feb 2026 15:38:33 -0500 Subject: [PATCH 08/15] updates --- .coveragerc | 6 +- synapse/lib/cell.py | 4 +- synapse/tests/test_lib_cell.py | 336 ------------------------------- synapse/tests/test_lib_drive.py | 342 ++++++++++++++++++++++++++++++++ 4 files changed, 350 insertions(+), 338 deletions(-) create mode 100644 synapse/tests/test_lib_drive.py diff --git a/.coveragerc b/.coveragerc index 8af9056b0c6..1f5f0eb46e9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,11 +2,15 @@ omit = */synapse/tests/test_* +[run] +concurrency = multiprocessing +parallel = True +sigterm = True + ; Uncomment this section to enable code coverage of storm files in the ; storm_dirs directory listed below. This is disabled by default right now ; because it's pretty intensive and imposes a large perf hit on the already slow ; tests. -;[run] ;plugins = synapse.utils.stormcov [synapse.utils.stormcov] diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 744ed6b7703..112b58a9f40 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1547,7 +1547,9 @@ async def _driveCellMigration(self): logger.warning(f"Migrated {rows} rows") newslab.forcecommit() - # TODO: trash celldrive + await self.olddrive.fini() + self.slab.dropdb(dbname) + self.olddrive = None logger.warning('...Drive migration complete!') diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index 2fd462412e2..05fd2e144b4 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -29,7 +29,6 @@ import synapse.lib.json as s_json import synapse.lib.link as s_link import synapse.lib.const as s_const -import synapse.lib.drive as s_drive import synapse.lib.nexus as s_nexus import synapse.lib.config as s_config import synapse.lib.certdir as s_certdir @@ -57,11 +56,6 @@ def _exiterProc(pipe, srcdir, dstdir, lmdbpaths, logconf): def _backupSleep(path, linkinfo): time.sleep(3.0) -async def migrate_v1(info, versinfo, data, curv): - assert curv == 1 - data['woot'] = 'woot' - return data - async def _doEOFBackup(path): return @@ -175,43 +169,6 @@ def finilink(): cell.onfini(auth.fini) return auth -testDataSchema_v0 = { - 'type': 'object', - 'properties': { - 'type': {'type': 'string'}, - 'size': {'type': 'number'}, - 'stuff': {'type': ['number', 'null'], 'default': None} - }, - 'required': ['type', 'size', 'stuff'], - 'additionalProperties': False, -} - -testDataSchema_v1 = { - 'type': 'object', - 'properties': { - 'type': {'type': 'string'}, - 'size': {'type': 'number'}, - 'stuff': {'type': ['number', 'null'], 'default': None}, - 'woot': {'type': 'string'}, - 'blorp': { - 'type': 'object', - 'properties': { - 'bleep': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'neato': {'type': 'string'} - } - } - } - } - } - }, - 'required': ['type', 'size', 'woot'], - 'additionalProperties': False, -} - class CellTest(s_t_utils.SynTest): async def test_cell_getLocalUrl(self): @@ -226,261 +183,6 @@ async def test_cell_getLocalUrl(self): url = cell.getLocalUrl(user='lowuser', share='*/view') self.eq(url, f'cell://lowuser@{dirn}:*/view') - async def test_cell_drive(self): - - with self.getTestDir() as dirn: - async with self.getTestCell(dirn=dirn) as cell: - - with self.raises(s_exc.BadName): - s_drive.reqValidName('A' * 512) - - info = {'name': 'users'} - pathinfo = await cell.addDriveItem(info) - - info = {'name': 'root'} - pathinfo = await cell.addDriveItem(info, path='users') - - with self.raises(s_exc.DupIden): - await cell.drive.addItemInfo(pathinfo[-1], path='users') - - rootdir = pathinfo[-1].get('iden') - self.eq(0, pathinfo[-1].get('kids')) - - info = {'name': 'win32k.sys', 'type': 'hehe'} - with self.raises(s_exc.NoSuchType): - info = await cell.addDriveItem(info, reldir=rootdir) - - infos = [i async for i in cell.getDriveKids(s_drive.rootdir)] - self.len(1, infos) - self.eq(1, infos[0].get('kids')) - self.eq('users', infos[0].get('name')) - - # TODO how to handle iden match with additional property mismatch - - self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0)) - self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) - self.false(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) - - with self.raises(s_exc.BadVersion): - await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0) - - info = {'name': 'win32k.sys', 'type': 'woot', 'perm': {'users': {}}} - info = await cell.addDriveItem(info, reldir=rootdir) - self.notin('perm', info) - self.eq(info[0]['permissions'], { - 'users': {}, - 'roles': {} - }) - - iden = info[-1].get('iden') - - tick = s_common.now() - rootuser = cell.auth.rootuser.iden - fooser = await cell.auth.addUser('foo') - neatrole = await cell.auth.addRole('neatrole') - await fooser.grant(neatrole.iden) - - # with self.raises(s_exc.SchemaViolation): - # versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - # await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) - - versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20, 'stuff': 12}) - self.eq(info.get('version'), (1, 1, 0)) - self.eq(versinfo.get('version'), (1, 1, 0)) - - versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 13}) - self.eq(info.get('version'), (1, 1, 0)) - self.eq(versinfo.get('version'), (1, 0, 0)) - - versinfo10, data10 = await cell.getDriveData(iden, vers=(1, 0, 0)) - self.eq(versinfo10.get('updated'), tick) - self.eq(versinfo10.get('updater'), rootuser) - self.eq(versinfo10.get('version'), (1, 0, 0)) - - versinfo11, data11 = await cell.getDriveData(iden, vers=(1, 1, 0)) - self.eq(versinfo11.get('updated'), tick + 10) - self.eq(versinfo11.get('updater'), rootuser) - self.eq(versinfo11.get('version'), (1, 1, 0)) - - versions = [vers async for vers in cell.getDriveDataVersions(iden)] - self.len(2, versions) - self.eq(versions[0], versinfo11) - self.eq(versions[1], versinfo10) - - info = await cell.delDriveData(iden, vers=(0, 0, 0)) - - versions = [vers async for vers in cell.getDriveDataVersions(iden)] - self.len(2, versions) - self.eq(versions[0], versinfo11) - self.eq(versions[1], versinfo10) - - info = await cell.delDriveData(iden, vers=(1, 1, 0)) - self.eq(info.get('updated'), tick) - self.eq(info.get('version'), (1, 0, 0)) - - info = await cell.delDriveData(iden, vers=(1, 0, 0)) - self.eq(info.get('size'), 0) - self.eq(info.get('version'), (0, 0, 0)) - self.none(info.get('updated')) - self.none(info.get('updater')) - - # repopulate a couple data versions to test migration and delete - versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 14}) - versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 17, 'stuff': 15}) - self.eq(versinfo, (await cell.getDriveData(iden))[0]) - - await cell.setDriveItemProp(iden, versinfo, ('stuff',), 1234) - data = await cell.getDriveData(iden) - self.eq(data[1]['stuff'], 1234) - - # This will be done by the cell in a cell storage version migration... - callback = 'synapse.tests.test_lib_cell.migrate_v1' - await cell.drive.setTypeSchema('woot', testDataSchema_v1, callback=callback) - - await cell.setDriveItemProp(iden, versinfo, 'woot', 'woot') - - versinfo['version'] = (1, 1, 1) - await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829) - data = await cell.getDriveData(iden) - self.eq(data[0]['version'], (1, 1, 1)) - self.eq(data[1]['stuff'], 3829) - - await self.asyncraises(s_exc.NoSuchIden, cell.setDriveItemProp(s_common.guid(), versinfo, ('lolnope',), 'not real')) - - await self.asyncraises(s_exc.BadArg, cell.setDriveItemProp(iden, versinfo, ('blorp', 0, 'neato'), 'my special string')) - data[1]['blorp'] = { - 'bleep': [{'neato': 'thing'}] - } - info, versinfo = await cell.setDriveData(iden, versinfo, data[1]) - now = s_common.now() - versinfo['updated'] = now - await cell.setDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato'), 'my special string') - data = await cell.getDriveData(iden) - self.eq(now, data[0]['updated']) - self.eq('my special string', data[1]['blorp']['bleep'][0]['neato']) - - versinfo['version'] = (1, 2, 1) - await cell.delDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato')) - vers, data = await cell.getDriveData(iden) - self.eq((1, 2, 1), vers['version']) - self.nn(data['blorp']['bleep'][0]) - self.notin('neato', data['blorp']['bleep'][0]) - - await self.asyncraises(s_exc.NoSuchIden, cell.delDriveItemProp(s_common.guid(), versinfo, 'blorp')) - - self.none(await cell.delDriveItemProp(iden, versinfo, ('lolnope', 'nopath'))) - - versinfo, data = await cell.getDriveData(iden, vers=(1, 0, 0)) - print(versinfo) - print(data) - self.eq('woot', data.get('woot')) - - versinfo, data = await cell.getDriveData(iden, vers=(1, 1, 0)) - self.eq('woot', data.get('woot')) - - with self.raises(s_exc.NoSuchIden): - await cell.reqDriveInfo('d7d6107b200e2c039540fc627bc5537d') - - with self.raises(s_exc.TypeMismatch): - await cell.getDriveInfo(iden, typename='newp') - - self.nn(await cell.getDriveInfo(iden)) - self.len(4, [vers async for vers in cell.getDriveDataVersions(iden)]) - - await cell.delDriveData(iden) - self.len(3, [vers async for vers in cell.getDriveDataVersions(iden)]) - - await cell.delDriveInfo(iden) - - self.none(await cell.getDriveInfo(iden)) - self.len(0, [vers async for vers in cell.getDriveDataVersions(iden)]) - - with self.raises(s_exc.NoSuchPath): - await cell.getDrivePath('users/root/win32k.sys') - - pathinfo = await cell.addDrivePath('foo/bar/baz') - self.len(3, pathinfo) - self.eq('foo', pathinfo[0].get('name')) - self.eq(1, pathinfo[0].get('kids')) - self.eq('bar', pathinfo[1].get('name')) - self.eq(1, pathinfo[1].get('kids')) - self.eq('baz', pathinfo[2].get('name')) - self.eq(0, pathinfo[2].get('kids')) - - self.eq(pathinfo, await cell.addDrivePath('foo/bar/baz')) - - baziden = pathinfo[2].get('iden') - self.eq(pathinfo, await cell.drive.getItemPath(baziden)) - - info = await cell.setDriveInfoPerm(baziden, {'users': {rootuser: s_cell.PERM_ADMIN}, 'roles': {}}) - # make sure drive perms work with easy perms - self.true(cell._hasEasyPerm(info, cell.auth.rootuser, s_cell.PERM_ADMIN)) - # defaults to READ - self.true(cell._hasEasyPerm(info, fooser, s_cell.PERM_READ)) - self.false(cell._hasEasyPerm(info, fooser, s_cell.PERM_EDIT)) - - with self.raises(s_exc.NoSuchIden): - # s_drive.rootdir is all 00s... ;) - await cell.setDriveInfoPerm(s_drive.rootdir, {'users': {}, 'roles': {}}) - - await cell.addDrivePath('hehe/haha') - pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') - - self.eq('hoho', pathinfo[-1].get('name')) - self.eq(baziden, pathinfo[-1].get('iden')) - - self.true(await cell.drive.hasPathInfo('hehe/haha/hoho')) - self.false(await cell.drive.hasPathInfo('foo/bar/baz')) - - pathinfo = await cell.getDrivePath('foo/bar') - self.eq(0, pathinfo[-1].get('kids')) - - pathinfo = await cell.getDrivePath('hehe/haha') - self.eq(1, pathinfo[-1].get('kids')) - - with self.raises(s_exc.DupName): - iden = pathinfo[-2].get('iden') - name = pathinfo[-1].get('name') - await cell.drive.reqFreeStep(iden, name) - - walks = [item async for item in cell.drive.walkPathInfo('hehe')] - self.len(3, walks) - # confirm walked paths are yielded depth first... - self.eq('hoho', walks[0].get('name')) - self.eq('haha', walks[1].get('name')) - self.eq('hehe', walks[2].get('name')) - - iden = walks[2].get('iden') - walks = [item async for item in cell.drive.walkItemInfo(iden)] - self.len(3, walks) - self.eq('hoho', walks[0].get('name')) - self.eq('haha', walks[1].get('name')) - self.eq('hehe', walks[2].get('name')) - - self.none(await cell.drive.getTypeSchema('newp')) - - # cell.drive.validators.pop('woot') - # self.nn(cell.drive.getTypeValidator('woot')) - - # move to root dir - pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop') - self.len(1, pathinfo) - self.eq(s_drive.rootdir, pathinfo[-1].get('parent')) - - pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') - self.len(3, pathinfo) - - async with self.getTestCell(dirn=dirn) as cell: - data = {'type': 'woot', 'size': 20, 'stuff': 12, 'woot': 'woot'} - # explicitly clear out the cache JsValidators, otherwise we get the cached, pre-msgpack - # version of the validator, which will be correct and skip the point of this test. - s_config._JsValidators.clear() - await cell.drive.reqValidData('woot', data) - async def test_cell_auth(self): with self.getTestDir() as dirn: @@ -716,44 +418,6 @@ async def test_cell_auth(self): with self.raises(s_exc.NeedConfValu): await echo.reqAhaProxy() - async def test_cell_drive_perm_migration(self): - async with self.getRegrCore('drive-perm-migr') as core: - item = await core.getDrivePath('driveitemdefaultperms') - self.len(1, item) - self.notin('perm', item) - self.eq(item[0]['permissions'], {'users': {}, 'roles': {}}) - - ldog = await core.auth.getRoleByName('littledog') - bdog = await core.auth.getRoleByName('bigdog') - - louis = await core.auth.getUserByName('lewis') - tim = await core.auth.getUserByName('tim') - mj = await core.auth.getUserByName('mj') - - item = await core.getDrivePath('permfolder/driveitemwithperms') - self.len(2, item) - self.notin('perm', item[0]) - self.notin('perm', item[1]) - self.eq(item[0]['permissions'], {'users': {tim.iden: s_cell.PERM_ADMIN}, 'roles': {}}) - self.eq(item[1]['permissions'], { - 'users': { - mj.iden: s_cell.PERM_ADMIN - }, - 'roles': { - ldog.iden: s_cell.PERM_READ, - bdog.iden: s_cell.PERM_EDIT, - }, - 'default': s_cell.PERM_DENY - }) - - # make sure it's all good with easy perms - self.true(core._hasEasyPerm(item[0], tim, s_cell.PERM_ADMIN)) - self.false(core._hasEasyPerm(item[0], mj, s_cell.PERM_EDIT)) - - self.true(core._hasEasyPerm(item[1], mj, s_cell.PERM_ADMIN)) - self.true(core._hasEasyPerm(item[1], tim, s_cell.PERM_READ)) - self.true(core._hasEasyPerm(item[1], louis, s_cell.PERM_EDIT)) - async def test_cell_unix_sock(self): async with self.getTestCore() as core: diff --git a/synapse/tests/test_lib_drive.py b/synapse/tests/test_lib_drive.py new file mode 100644 index 00000000000..dc8ee2120bb --- /dev/null +++ b/synapse/tests/test_lib_drive.py @@ -0,0 +1,342 @@ +import synapse.exc as s_exc +import synapse.common as s_common + +import synapse.lib.cell as s_cell +import synapse.lib.drive as s_drive +import synapse.lib.config as s_config + +import synapse.tests.utils as s_t_utils + +async def migrate_v1(info, versinfo, data, curv): + assert curv == 1 + data['woot'] = 'woot' + return data + +testDataSchema_v0 = { + 'type': 'object', + 'properties': { + 'type': {'type': 'string'}, + 'size': {'type': 'number'}, + 'stuff': {'type': ['number', 'null'], 'default': None} + }, + 'required': ['type', 'size', 'stuff'], + 'additionalProperties': False, +} + +testDataSchema_v1 = { + 'type': 'object', + 'properties': { + 'type': {'type': 'string'}, + 'size': {'type': 'number'}, + 'stuff': {'type': ['number', 'null'], 'default': None}, + 'woot': {'type': 'string'}, + 'blorp': { + 'type': 'object', + 'properties': { + 'bleep': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'neato': {'type': 'string'} + } + } + } + } + } + }, + 'required': ['type', 'size', 'woot'], + 'additionalProperties': False, +} + +class DriveTest(s_t_utils.SynTest): + + async def test_drive_base(self): + + with self.getTestDir() as dirn: + async with self.getTestCell(dirn=dirn) as cell: + + with self.raises(s_exc.BadName): + s_drive.reqValidName('A' * 512) + + info = {'name': 'users'} + pathinfo = await cell.addDriveItem(info) + + info = {'name': 'root'} + pathinfo = await cell.addDriveItem(info, path='users') + + with self.raises(s_exc.DupIden): + await cell.drive.addItemInfo(pathinfo[-1], path='users') + + rootdir = pathinfo[-1].get('iden') + self.eq(0, pathinfo[-1].get('kids')) + + info = {'name': 'win32k.sys', 'type': 'hehe'} + with self.raises(s_exc.NoSuchType): + info = await cell.addDriveItem(info, reldir=rootdir) + + infos = [i async for i in cell.getDriveKids(s_drive.rootdir)] + self.len(1, infos) + self.eq(1, infos[0].get('kids')) + self.eq('users', infos[0].get('name')) + + # TODO how to handle iden match with additional property mismatch + + self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0)) + self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) + self.false(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) + + with self.raises(s_exc.BadVersion): + await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0) + + info = {'name': 'win32k.sys', 'type': 'woot', 'perm': {'users': {}}} + info = await cell.addDriveItem(info, reldir=rootdir) + self.notin('perm', info) + self.eq(info[0]['permissions'], { + 'users': {}, + 'roles': {} + }) + + iden = info[-1].get('iden') + + tick = s_common.now() + rootuser = cell.auth.rootuser.iden + fooser = await cell.auth.addUser('foo') + neatrole = await cell.auth.addRole('neatrole') + await fooser.grant(neatrole.iden) + + with self.raises(s_exc.SchemaViolation): + versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) + + versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20, 'stuff': 12}) + self.eq(info.get('version'), (1, 1, 0)) + self.eq(versinfo.get('version'), (1, 1, 0)) + + versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 13}) + self.eq(info.get('version'), (1, 1, 0)) + self.eq(versinfo.get('version'), (1, 0, 0)) + + versinfo10, data10 = await cell.getDriveData(iden, vers=(1, 0, 0)) + self.eq(versinfo10.get('updated'), tick) + self.eq(versinfo10.get('updater'), rootuser) + self.eq(versinfo10.get('version'), (1, 0, 0)) + + versinfo11, data11 = await cell.getDriveData(iden, vers=(1, 1, 0)) + self.eq(versinfo11.get('updated'), tick + 10) + self.eq(versinfo11.get('updater'), rootuser) + self.eq(versinfo11.get('version'), (1, 1, 0)) + + versions = [vers async for vers in cell.getDriveDataVersions(iden)] + self.len(2, versions) + self.eq(versions[0], versinfo11) + self.eq(versions[1], versinfo10) + + info = await cell.delDriveData(iden, vers=(0, 0, 0)) + + versions = [vers async for vers in cell.getDriveDataVersions(iden)] + self.len(2, versions) + self.eq(versions[0], versinfo11) + self.eq(versions[1], versinfo10) + + info = await cell.delDriveData(iden, vers=(1, 1, 0)) + self.eq(info.get('updated'), tick) + self.eq(info.get('version'), (1, 0, 0)) + + info = await cell.delDriveData(iden, vers=(1, 0, 0)) + self.eq(info.get('size'), 0) + self.eq(info.get('version'), (0, 0, 0)) + self.none(info.get('updated')) + self.none(info.get('updater')) + + # repopulate a couple data versions to test migration and delete + versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 14}) + versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 17, 'stuff': 15}) + self.eq(versinfo, (await cell.getDriveData(iden))[0]) + + await cell.setDriveItemProp(iden, versinfo, ('stuff',), 1234) + data = await cell.getDriveData(iden) + self.eq(data[1]['stuff'], 1234) + + # This will be done by the cell in a cell storage version migration... + callback = 'synapse.tests.test_lib_drive.migrate_v1' + await cell.drive.setTypeSchema('woot', testDataSchema_v1, callback=callback) + + await cell.setDriveItemProp(iden, versinfo, 'woot', 'woot') + + versinfo['version'] = (1, 1, 1) + await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829) + data = await cell.getDriveData(iden) + self.eq(data[0]['version'], (1, 1, 1)) + self.eq(data[1]['stuff'], 3829) + + await self.asyncraises(s_exc.NoSuchIden, cell.setDriveItemProp(s_common.guid(), versinfo, ('lolnope',), 'not real')) + + await self.asyncraises(s_exc.BadArg, cell.setDriveItemProp(iden, versinfo, ('blorp', 0, 'neato'), 'my special string')) + data[1]['blorp'] = { + 'bleep': [{'neato': 'thing'}] + } + info, versinfo = await cell.setDriveData(iden, versinfo, data[1]) + now = s_common.now() + versinfo['updated'] = now + await cell.setDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato'), 'my special string') + data = await cell.getDriveData(iden) + self.eq(now, data[0]['updated']) + self.eq('my special string', data[1]['blorp']['bleep'][0]['neato']) + + versinfo['version'] = (1, 2, 1) + await cell.delDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato')) + vers, data = await cell.getDriveData(iden) + self.eq((1, 2, 1), vers['version']) + self.nn(data['blorp']['bleep'][0]) + self.notin('neato', data['blorp']['bleep'][0]) + + await self.asyncraises(s_exc.NoSuchIden, cell.delDriveItemProp(s_common.guid(), versinfo, 'blorp')) + + self.none(await cell.delDriveItemProp(iden, versinfo, ('lolnope', 'nopath'))) + + versinfo, data = await cell.getDriveData(iden, vers=(1, 0, 0)) + print(versinfo) + print(data) + self.eq('woot', data.get('woot')) + + versinfo, data = await cell.getDriveData(iden, vers=(1, 1, 0)) + self.eq('woot', data.get('woot')) + + with self.raises(s_exc.NoSuchIden): + await cell.reqDriveInfo('d7d6107b200e2c039540fc627bc5537d') + + with self.raises(s_exc.TypeMismatch): + await cell.getDriveInfo(iden, typename='newp') + + self.nn(await cell.getDriveInfo(iden)) + self.len(4, [vers async for vers in cell.getDriveDataVersions(iden)]) + + await cell.delDriveData(iden) + self.len(3, [vers async for vers in cell.getDriveDataVersions(iden)]) + + await cell.delDriveInfo(iden) + + self.none(await cell.getDriveInfo(iden)) + self.len(0, [vers async for vers in cell.getDriveDataVersions(iden)]) + + with self.raises(s_exc.NoSuchPath): + await cell.getDrivePath('users/root/win32k.sys') + + pathinfo = await cell.addDrivePath('foo/bar/baz') + self.len(3, pathinfo) + self.eq('foo', pathinfo[0].get('name')) + self.eq(1, pathinfo[0].get('kids')) + self.eq('bar', pathinfo[1].get('name')) + self.eq(1, pathinfo[1].get('kids')) + self.eq('baz', pathinfo[2].get('name')) + self.eq(0, pathinfo[2].get('kids')) + + self.eq(pathinfo, await cell.addDrivePath('foo/bar/baz')) + + baziden = pathinfo[2].get('iden') + self.eq(pathinfo, await cell.drive.getItemPath(baziden)) + + info = await cell.setDriveInfoPerm(baziden, {'users': {rootuser: s_cell.PERM_ADMIN}, 'roles': {}}) + # make sure drive perms work with easy perms + self.true(cell._hasEasyPerm(info, cell.auth.rootuser, s_cell.PERM_ADMIN)) + # defaults to READ + self.true(cell._hasEasyPerm(info, fooser, s_cell.PERM_READ)) + self.false(cell._hasEasyPerm(info, fooser, s_cell.PERM_EDIT)) + + with self.raises(s_exc.NoSuchIden): + # s_drive.rootdir is all 00s... ;) + await cell.setDriveInfoPerm(s_drive.rootdir, {'users': {}, 'roles': {}}) + + await cell.addDrivePath('hehe/haha') + pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') + + self.eq('hoho', pathinfo[-1].get('name')) + self.eq(baziden, pathinfo[-1].get('iden')) + + self.true(await cell.drive.hasPathInfo('hehe/haha/hoho')) + self.false(await cell.drive.hasPathInfo('foo/bar/baz')) + + pathinfo = await cell.getDrivePath('foo/bar') + self.eq(0, pathinfo[-1].get('kids')) + + pathinfo = await cell.getDrivePath('hehe/haha') + self.eq(1, pathinfo[-1].get('kids')) + + with self.raises(s_exc.DupName): + iden = pathinfo[-2].get('iden') + name = pathinfo[-1].get('name') + await cell.drive.reqFreeStep(iden, name) + + walks = [item async for item in cell.drive.walkPathInfo('hehe')] + self.len(3, walks) + # confirm walked paths are yielded depth first... + self.eq('hoho', walks[0].get('name')) + self.eq('haha', walks[1].get('name')) + self.eq('hehe', walks[2].get('name')) + + iden = walks[2].get('iden') + walks = [item async for item in cell.drive.walkItemInfo(iden)] + self.len(3, walks) + self.eq('hoho', walks[0].get('name')) + self.eq('haha', walks[1].get('name')) + self.eq('hehe', walks[2].get('name')) + + self.none(await cell.drive.getTypeSchema('newp')) + + # move to root dir + pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop') + self.len(1, pathinfo) + self.eq(s_drive.rootdir, pathinfo[-1].get('parent')) + + pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') + self.len(3, pathinfo) + + async with self.getTestCell(dirn=dirn) as cell: + data = {'type': 'woot', 'size': 20, 'stuff': 12, 'woot': 'woot'} + # explicitly clear out the cache JsValidators, otherwise we get the cached, pre-msgpack + # version of the validator, which will be correct and skip the point of this test. + s_config._JsValidators.clear() + await cell.drive.reqValidData('woot', data) + + async def test_drive_perm_migration(self): + async with self.getRegrCore('drive-perm-migr') as core: + item = await core.getDrivePath('driveitemdefaultperms') + self.len(1, item) + self.notin('perm', item) + self.eq(item[0]['permissions'], {'users': {}, 'roles': {}}) + + ldog = await core.auth.getRoleByName('littledog') + bdog = await core.auth.getRoleByName('bigdog') + + louis = await core.auth.getUserByName('lewis') + tim = await core.auth.getUserByName('tim') + mj = await core.auth.getUserByName('mj') + + item = await core.getDrivePath('permfolder/driveitemwithperms') + self.len(2, item) + self.notin('perm', item[0]) + self.notin('perm', item[1]) + self.eq(item[0]['permissions'], {'users': {tim.iden: s_cell.PERM_ADMIN}, 'roles': {}}) + self.eq(item[1]['permissions'], { + 'users': { + mj.iden: s_cell.PERM_ADMIN + }, + 'roles': { + ldog.iden: s_cell.PERM_READ, + bdog.iden: s_cell.PERM_EDIT, + }, + 'default': s_cell.PERM_DENY + }) + + # make sure it's all good with easy perms + self.true(core._hasEasyPerm(item[0], tim, s_cell.PERM_ADMIN)) + self.false(core._hasEasyPerm(item[0], mj, s_cell.PERM_EDIT)) + + self.true(core._hasEasyPerm(item[1], mj, s_cell.PERM_ADMIN)) + self.true(core._hasEasyPerm(item[1], tim, s_cell.PERM_READ)) + self.true(core._hasEasyPerm(item[1], louis, s_cell.PERM_EDIT)) From 284ac5e7a126d8ef74874f6903f7c90ddcb5dd89 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 25 Feb 2026 15:55:39 -0500 Subject: [PATCH 09/15] updates --- synapse/tests/test_lib_base.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index f8bb0dbc1f5..26af17116ce 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -605,3 +605,13 @@ async def test_base_spawner_fini(self): await base.fini() self.true(base.isfini) self.true(proxy.isfini) + + async def test_base_spawner_none(self): + + spawner = Haha.spawner() + proxy = await spawner() + + self.false(proxy.isfini) + + await proxy.fini() + self.true(proxy.isfini) From e20412e2ec53aadec86dccc93ae54016c7122223 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 25 Feb 2026 16:49:37 -0500 Subject: [PATCH 10/15] updates --- .circleci/config.yml | 6 ++++-- .coveragerc.main | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 .coveragerc.main diff --git a/.circleci/config.yml b/.circleci/config.yml index 551442a8c74..cf686d574ca 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -458,7 +458,8 @@ jobs: CODECOV_FLAG: linux SYN_REGRESSION_REPO: ~/git/synapse-regression COVERAGE_FILE: test-reports/<< pipeline.git.revision >>/.coverage - COVERAGE_ARGS: --cov synapse --cov-append + COVERAGE_PROCESS_START: .coveragerc + COVERAGE_ARGS: --cov synapse --cov-config=.coveragerc.main --cov-append working_directory: ~/repo @@ -474,7 +475,8 @@ jobs: RUN_SYNTAX: 1 CODECOV_FLAG: linux_replay SYN_REGRESSION_REPO: ~/git/synapse-regression - COVERAGE_ARGS: --cov synapse + COVERAGE_PROCESS_START: .coveragerc + COVERAGE_ARGS: --cov synapse --cov-config=.coveragerc.main SYNDEV_NEXUS_REPLAY: 1 working_directory: ~/repo diff --git a/.coveragerc.main b/.coveragerc.main new file mode 100644 index 00000000000..27d062b081d --- /dev/null +++ b/.coveragerc.main @@ -0,0 +1,6 @@ +[report] +omit = + */synapse/tests/test_* + +[run] +sigterm = True From fe1e7c174b2e2fad98fd4c0502c2e0599b852b22 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:00:13 -0500 Subject: [PATCH 11/15] updates --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf686d574ca..f86bbcd2236 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,7 @@ commands: name: run tests command: | . venv/bin/activate - mkdir test-reports + mkdir -p test-reports circleci tests glob synapse/tests/test_*.py synapse/vendor/**/test_*.py | \ circleci tests run \ --timings-type=name \ From 08c62123cf7a77ad1bde4950e7ff387289e6517e Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:27:51 -0500 Subject: [PATCH 12/15] updates --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 1f5f0eb46e9..c3be21060ff 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,7 @@ [report] omit = */synapse/tests/test_* + /tmp/* [run] concurrency = multiprocessing From 78b76004b7ae7af5e3ff175c5a31213805971025 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 25 Feb 2026 17:57:36 -0500 Subject: [PATCH 13/15] updates --- synapse/lib/drive.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/synapse/lib/drive.py b/synapse/lib/drive.py index 2ca7f6204a9..d7d3bb51e50 100644 --- a/synapse/lib/drive.py +++ b/synapse/lib/drive.py @@ -566,19 +566,6 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): await asyncio.sleep(0) return True - async def getMigrRows(self, typename): - - async for info in self.getItemsByType(typename): - - bidn = s_common.uhex(info.get('iden')) - for lkey, byts in self.slab.scanByPref(LKEY_VERS + bidn, db=self.dbname): - datakey = LKEY_DATA + bidn + lkey[-9:] - databyts = self.slab.get(datakey, db=self.dbname) - yield datakey, s_msgpack.un(databyts) - - async def setMigrRow(self, datakey, data): - self.slab.put(datakey, s_msgpack.en(data), db=self.dbname) - async def getItemsByType(self, typename): tkey = typename.encode() + b'\x00' for lkey in self.slab.scanKeysByPref(LKEY_INFO_BYTYPE + tkey, db=self.dbname): From c764653d21c208c0f494d4170de1df52c0eeec49 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Thu, 26 Feb 2026 10:14:40 -0500 Subject: [PATCH 14/15] updates --- synapse/lib/cell.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 77e75e8f944..ed50884650c 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -1721,7 +1721,8 @@ def _delTmpFiles(self): except OSError: # pragma: no cover pass - # FIXME - recursively remove sockets dir here? + shutil.rmtree(self.sockdirn, ignore_errors=True) + self.sockdirn = s_common.gendir(self.dirn, 'sockets') async def _execCellUpdates(self): # implement to apply updates to a fully initialized active cell From 20e50803ab7af3e9b54ecedfe767d7cf7d7d614a Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Thu, 26 Feb 2026 10:37:11 -0500 Subject: [PATCH 15/15] changelog --- changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml | 6 ++++++ changes/fb1de9c8091f353422a50736baf4d803.yaml | 7 +++++++ 2 files changed, 13 insertions(+) create mode 100644 changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml create mode 100644 changes/fb1de9c8091f353422a50736baf4d803.yaml diff --git a/changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml b/changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml new file mode 100644 index 00000000000..17cab9c32a4 --- /dev/null +++ b/changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml @@ -0,0 +1,6 @@ +--- +desc: Migrated cell drive data into a dedicated slab. +desc:literal: false +prs: [] +type: migration +... diff --git a/changes/fb1de9c8091f353422a50736baf4d803.yaml b/changes/fb1de9c8091f353422a50736baf4d803.yaml new file mode 100644 index 00000000000..8a433508a24 --- /dev/null +++ b/changes/fb1de9c8091f353422a50736baf4d803.yaml @@ -0,0 +1,7 @@ +--- +desc: Added a dedicated IO worker for the drive subsystem to offload operations into + a separate process. +desc:literal: false +prs: [] +type: feat +...