diff --git a/.circleci/config.yml b/.circleci/config.yml index 551442a8c74..f86bbcd2236 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -58,7 +58,7 @@ commands: name: run tests command: | . venv/bin/activate - mkdir test-reports + mkdir -p test-reports circleci tests glob synapse/tests/test_*.py synapse/vendor/**/test_*.py | \ circleci tests run \ --timings-type=name \ @@ -458,7 +458,8 @@ jobs: CODECOV_FLAG: linux SYN_REGRESSION_REPO: ~/git/synapse-regression COVERAGE_FILE: test-reports/<< pipeline.git.revision >>/.coverage - COVERAGE_ARGS: --cov synapse --cov-append + COVERAGE_PROCESS_START: .coveragerc + COVERAGE_ARGS: --cov synapse --cov-config=.coveragerc.main --cov-append working_directory: ~/repo @@ -474,7 +475,8 @@ jobs: RUN_SYNTAX: 1 CODECOV_FLAG: linux_replay SYN_REGRESSION_REPO: ~/git/synapse-regression - COVERAGE_ARGS: --cov synapse + COVERAGE_PROCESS_START: .coveragerc + COVERAGE_ARGS: --cov synapse --cov-config=.coveragerc.main SYNDEV_NEXUS_REPLAY: 1 working_directory: ~/repo diff --git a/.coveragerc b/.coveragerc index 8af9056b0c6..c3be21060ff 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,12 +1,17 @@ [report] omit = */synapse/tests/test_* + /tmp/* + +[run] +concurrency = multiprocessing +parallel = True +sigterm = True ; Uncomment this section to enable code coverage of storm files in the ; storm_dirs directory listed below. This is disabled by default right now ; because it's pretty intensive and imposes a large perf hit on the already slow ; tests. -;[run] ;plugins = synapse.utils.stormcov [synapse.utils.stormcov] diff --git a/.coveragerc.main b/.coveragerc.main new file mode 100644 index 00000000000..27d062b081d --- /dev/null +++ b/.coveragerc.main @@ -0,0 +1,6 @@ +[report] +omit = + */synapse/tests/test_* + +[run] +sigterm = True diff --git a/changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml b/changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml new file mode 100644 index 00000000000..17cab9c32a4 --- /dev/null +++ b/changes/c56856d0929e4a71fde0d98fc77ddeb5.yaml @@ -0,0 +1,6 @@ +--- +desc: Migrated cell drive data into a dedicated slab. +desc:literal: false +prs: [] +type: migration +... diff --git a/changes/fb1de9c8091f353422a50736baf4d803.yaml b/changes/fb1de9c8091f353422a50736baf4d803.yaml new file mode 100644 index 00000000000..8a433508a24 --- /dev/null +++ b/changes/fb1de9c8091f353422a50736baf4d803.yaml @@ -0,0 +1,7 @@ +--- +desc: Added a dedicated IO worker for the drive subsystem to offload operations into + a separate process. +desc:literal: false +prs: [] +type: feat +... diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py index 1c5d4617a95..ed50884650c 100644 --- a/synapse/lib/cell.py +++ b/synapse/lib/cell.py @@ -2,6 +2,7 @@ import os import ssl import copy +import stat import time import fcntl import shutil @@ -1181,6 +1182,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None): s_telepath.Aware.__init__(self) self.dirn = s_common.gendir(dirn) + self.sockdirn = s_common.gendir(dirn, 'sockets') + self.runid = s_common.guid() self.auth = None @@ -1521,14 +1524,34 @@ async def _storCellAuthMigration(self): logger.warning(f'...Cell ({self.getCellType()}) auth migration complete!') async def _drivePermMigration(self): - for lkey, lval in self.slab.scanByPref(s_drive.LKEY_INFO, db=self.drive.dbname): - info = s_msgpack.un(lval) - perm = info.pop('perm', None) - if perm is not None: - perm.setdefault('users', {}) - perm.setdefault('roles', {}) - info['permissions'] = perm - self.slab.put(lkey, s_msgpack.en(info), db=self.drive.dbname) + async with await s_drive.Drive.anit(self.slab, 'celldrive') as olddrive: + for lkey, lval in self.slab.scanByPref(s_drive.LKEY_INFO, db=olddrive.dbname): + info = s_msgpack.un(lval) + perm = info.pop('perm', None) + if perm is not None: + perm.setdefault('users', {}) + perm.setdefault('roles', {}) + info['permissions'] = perm + self.slab.put(lkey, s_msgpack.en(info), db=olddrive.dbname) + + async def _driveCellMigration(self): + logger.warning('Migrating Drive Slabs') + + self.olddrive = await s_drive.Drive.anit(self.slab, 'celldrive') + + dbname = self.olddrive.dbname + newpath = s_common.gendir(self.dirn, 'slabs', 'drive.lmdb') + + async with await s_lmdbslab.Slab.anit(newpath) as newslab: + rows = await self.slab.copydb(dbname, newslab, dbname) + logger.warning(f"Migrated {rows} rows") + newslab.forcecommit() + + await self.olddrive.fini() + self.slab.dropdb(dbname) + self.olddrive = None + + logger.warning('...Drive migration complete!') def getPermDef(self, perm): perm = tuple(perm) @@ -1672,22 +1695,34 @@ def _delTmpFiles(self): tdir = s_common.gendir(self.dirn, 'tmp') names = os.listdir(tdir) - if not names: - return + if names: + logger.warning(f'Removing {len(names)} temporary files/folders in: {tdir}') - logger.warning(f'Removing {len(names)} temporary files/folders in: {tdir}') + for name in names: - for name in names: + path = os.path.join(tdir, name) - path = os.path.join(tdir, name) + if os.path.isfile(path): + os.unlink(path) + continue - if os.path.isfile(path): - os.unlink(path) - continue + if os.path.isdir(path): + shutil.rmtree(path, ignore_errors=True) + continue - if os.path.isdir(path): - shutil.rmtree(path, ignore_errors=True) - continue + names = os.listdir(self.sockdirn) + if names: + logger.info(f'Removing {len(names)} old sockets in: {self.sockdirn}') + for name in names: + path = os.path.join(self.sockdirn, name) + try: + if stat.S_ISSOCK(os.stat(path).st_mode): + os.unlink(path) + except OSError: # pragma: no cover + pass + + shutil.rmtree(self.sockdirn, ignore_errors=True) + self.sockdirn = s_common.gendir(self.dirn, 'sockets') async def _execCellUpdates(self): # implement to apply updates to a fully initialized active cell @@ -1899,11 +1934,20 @@ async def initServiceEarly(self): pass async def initCellStorage(self): - self.drive = await s_drive.Drive.anit(self.slab, 'celldrive') await self._bumpCellVers('drive:storage', ( (1, self._drivePermMigration), + (2, self._driveCellMigration), ), nexs=False) + path = s_common.gendir(self.dirn, 'slabs', 'drive.lmdb') + sockpath = s_common.genpath(self.sockdirn, 'drive') + + if len(sockpath) > s_const.UNIX_PATH_MAX: + sockpath = None + + spawner = s_drive.FileDrive.spawner(base=self, sockpath=sockpath) + self.drive = await spawner(path) + self.onfini(self.drive.fini) async def addDriveItem(self, info, path=None, reldir=s_drive.rootdir): @@ -1922,7 +1966,7 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): # replay safety... iden = info.get('iden') - if self.drive.hasItemInfo(iden): # pragma: no cover + if await self.drive.hasItemInfo(iden): # pragma: no cover return await self.drive.getItemPath(iden) # TODO: Remove this in synapse-3xx @@ -1935,10 +1979,10 @@ async def _addDriveItem(self, info, path=None, reldir=s_drive.rootdir): return await self.drive.addItemInfo(info, path=path, reldir=reldir) async def getDriveInfo(self, iden, typename=None): - return self.drive.getItemInfo(iden, typename=typename) + return await self.drive.getItemInfo(iden, typename=typename) - def reqDriveInfo(self, iden, typename=None): - return self.drive.reqItemInfo(iden, typename=typename) + async def reqDriveInfo(self, iden, typename=None): + return await self.drive.reqItemInfo(iden, typename=typename) async def getDrivePath(self, path, reldir=s_drive.rootdir): ''' @@ -1961,15 +2005,14 @@ async def addDrivePath(self, path, perm=None, reldir=s_drive.rootdir): ''' tick = s_common.now() user = self.auth.rootuser.iden - path = self.drive.getPathNorm(path) + path = await self.drive.getPathNorm(path) if perm is None: perm = {'users': {}, 'roles': {}} for name in path: - info = self.drive.getStepInfo(reldir, name) - await asyncio.sleep(0) + info = await self.drive.getStepInfo(reldir, name) if info is not None: reldir = info.get('iden') @@ -1992,7 +2035,7 @@ async def getDriveData(self, iden, vers=None): Return the data associated with the drive item by iden. If vers is specified, return that specific version. ''' - return self.drive.getItemData(iden, vers=vers) + return await self.drive.getItemData(iden, vers=vers) async def getDriveDataVersions(self, iden): async for item in self.drive.getItemDataVersions(iden): @@ -2000,12 +2043,12 @@ async def getDriveDataVersions(self, iden): @s_nexus.Pusher.onPushAuto('drive:del') async def delDriveInfo(self, iden): - if self.drive.getItemInfo(iden) is not None: + if await self.drive.getItemInfo(iden) is not None: await self.drive.delItemInfo(iden) @s_nexus.Pusher.onPushAuto('drive:set:perm') async def setDriveInfoPerm(self, iden, perm): - return self.drive.setItemPerm(iden, perm) + return await self.drive.setItemPerm(iden, perm) @s_nexus.Pusher.onPushAuto('drive:data:path:set') async def setDriveItemProp(self, iden, vers, path, valu): @@ -2054,9 +2097,9 @@ async def delDriveItemProp(self, iden, vers, path): @s_nexus.Pusher.onPushAuto('drive:set:path') async def setDriveInfoPath(self, iden, path): - path = self.drive.getPathNorm(path) + path = await self.drive.getPathNorm(path) pathinfo = await self.drive.getItemPath(iden) - if path == [p.get('name') for p in pathinfo]: + if [str(p) for p in path] == [p.get('name', '') for p in pathinfo]: return pathinfo return await self.drive.setItemPath(iden, path) @@ -2067,13 +2110,13 @@ async def setDriveData(self, iden, versinfo, data): async def delDriveData(self, iden, vers=None): if vers is None: - info = self.drive.reqItemInfo(iden) + info = await self.drive.reqItemInfo(iden) vers = info.get('version') return await self._push('drive:data:del', iden, vers) @s_nexus.Pusher.onPush('drive:data:del') async def _delDriveData(self, iden, vers): - return self.drive.delItemData(iden, vers) + return await self.drive.delItemData(iden, vers) async def getDriveKids(self, iden): async for info in self.drive.getItemKids(iden): diff --git a/synapse/lib/const.py b/synapse/lib/const.py index 2ae64e83764..217bbba731a 100644 --- a/synapse/lib/const.py +++ b/synapse/lib/const.py @@ -51,3 +51,6 @@ # HTTP header constants MAX_LINE_SIZE = kibibyte * 64 MAX_FIELD_SIZE = kibibyte * 64 + +# Socket constants +UNIX_PATH_MAX = 107 diff --git a/synapse/lib/drive.py b/synapse/lib/drive.py index 375d6f05dee..d7d3bb51e50 100644 --- a/synapse/lib/drive.py +++ b/synapse/lib/drive.py @@ -6,8 +6,11 @@ import synapse.lib.base as s_base import synapse.lib.config as s_config +import synapse.lib.dyndeps as s_dyndeps import synapse.lib.msgpack as s_msgpack import synapse.lib.schemas as s_schemas +import synapse.lib.spawner as s_spawner +import synapse.lib.lmdbslab as s_lmdbslab nameregex = regex.compile(s_schemas.re_drivename) def reqValidName(name): @@ -54,7 +57,7 @@ async def __anit__(self, slab, name): self.dbname = slab.initdb(f'drive:{name}') self.validators = {} - def getPathNorm(self, path): + async def getPathNorm(self, path): if isinstance(path, str): path = path.strip().strip('/').split('/') @@ -67,7 +70,7 @@ def _reqInfoType(self, info, typename): mesg = f'Drive item has the wrong type. Expected: {typename} got {infotype}.' raise s_exc.TypeMismatch(mesg=mesg, expected=typename, got=infotype) - def getItemInfo(self, iden, typename=None): + async def getItemInfo(self, iden, typename=None): info = self._getItemInfo(s_common.uhex(iden)) if not info: return @@ -81,7 +84,7 @@ def _getItemInfo(self, bidn): if byts is not None: return s_msgpack.un(byts) - def reqItemInfo(self, iden, typename=None): + async def reqItemInfo(self, iden, typename=None): return self._reqItemInfo(s_common.uhex(iden), typename=typename) def _reqItemInfo(self, bidn, typename=None): @@ -105,7 +108,7 @@ async def getItemPath(self, iden): pathinfo = [] while iden is not None: - info = self.reqItemInfo(iden) + info = await self.reqItemInfo(iden) pathinfo.append(info) iden = info.get('parent') @@ -117,7 +120,7 @@ async def getItemPath(self, iden): async def _setItemPath(self, bidn, path, reldir=rootdir): - path = self.getPathNorm(path) + path = await self.getPathNorm(path) # new parent iden / bidn parinfo = None @@ -171,7 +174,7 @@ async def _setItemPath(self, bidn, path, reldir=rootdir): def _hasStepItem(self, bidn, name): return self.slab.has(LKEY_DIRN + bidn + name.encode(), db=self.dbname) - def getStepInfo(self, iden, name): + async def getStepInfo(self, iden, name): return self._getStepInfo(s_common.uhex(iden), name) def _getStepInfo(self, bidn, name): @@ -208,7 +211,7 @@ async def _addStepInfo(self, parbidn, parinfo, info): await self.slab.putmulti(rows, db=self.dbname) - def setItemPerm(self, iden, perm): + async def setItemPerm(self, iden, perm): return self._setItemPerm(s_common.uhex(iden), perm) def _setItemPerm(self, bidn, perm): @@ -226,8 +229,7 @@ async def getPathInfo(self, path, reldir=rootdir): This API is designed to allow the caller to retrieve the path info and potentially check permissions on each level to control access. ''' - - path = self.getPathNorm(path) + path = await self.getPathNorm(path) parbidn = s_common.uhex(reldir) pathinfo = [] @@ -244,7 +246,7 @@ async def getPathInfo(self, path, reldir=rootdir): return pathinfo - def hasItemInfo(self, iden): + async def hasItemInfo(self, iden): return self._hasItemInfo(s_common.uhex(iden)) def _hasItemInfo(self, bidn): @@ -254,7 +256,7 @@ async def hasPathInfo(self, path, reldir=rootdir): ''' Check for a path existing relative to reldir. ''' - path = self.getPathNorm(path) + path = await self.getPathNorm(path) parbidn = s_common.uhex(reldir) for part in path: @@ -277,7 +279,7 @@ async def addItemInfo(self, info, path=None, reldir=rootdir): pathinfo = [] if path is not None: - path = self.getPathNorm(path) + path = await self.getPathNorm(path) pathinfo = await self.getPathInfo(path, reldir=reldir) if pathinfo: pariden = pathinfo[-1].get('iden') @@ -300,7 +302,7 @@ async def addItemInfo(self, info, path=None, reldir=rootdir): bidn = s_common.uhex(iden) if typename is not None: - self.reqTypeValidator(typename) + await self.reqTypeValidator(typename) if self._getItemInfo(bidn) is not None: mesg = f'A drive entry with ID {iden} already exists.' @@ -311,7 +313,7 @@ async def addItemInfo(self, info, path=None, reldir=rootdir): pathinfo.append(info) return pathinfo - def reqFreeStep(self, iden, name): + async def reqFreeStep(self, iden, name): return self._reqFreeStep(s_common.uhex(iden), name) def _reqFreeStep(self, bidn, name): @@ -362,7 +364,7 @@ async def _walkItemInfo(self, bidn): async def walkPathInfo(self, path, reldir=rootdir): - path = self.getPathNorm(path) + path = await self.getPathNorm(path) pathinfo = await self.getPathInfo(path, reldir=reldir) bidn = s_common.uhex(pathinfo[-1].get('iden')) @@ -409,7 +411,7 @@ async def _setItemData(self, bidn, versinfo, data): typename = info.get('type') - self.reqValidData(typename, data) + await self.reqValidData(typename, data) byts = s_msgpack.en(data) @@ -439,7 +441,7 @@ async def _setItemData(self, bidn, versinfo, data): return info, versinfo - def getItemData(self, iden, vers=None): + async def getItemData(self, iden, vers=None): ''' Return a (versinfo, data) tuple for the given iden. If version is not specified, the current version is returned. @@ -465,7 +467,7 @@ def _getItemData(self, bidn, vers=None): return s_msgpack.un(versbyts), s_msgpack.un(databyts) - def delItemData(self, iden, vers=None): + async def delItemData(self, iden, vers=None): return self._delItemData(s_common.uhex(iden), vers=vers) def _delItemData(self, bidn, vers=None): @@ -507,12 +509,12 @@ async def getItemDataVersions(self, iden): yield s_msgpack.un(byts) await asyncio.sleep(0) - def getTypeSchema(self, typename): + async def getTypeSchema(self, typename): byts = self.slab.get(LKEY_TYPE + typename.encode(), db=self.dbname) if byts is not None: return s_msgpack.un(byts, use_list=True) - def getTypeSchemaVersion(self, typename): + async def getTypeSchemaVersion(self, typename): verskey = LKEY_TYPE_VERS + typename.encode() byts = self.slab.get(verskey, db=self.dbname) if byts is not None: @@ -522,9 +524,16 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): reqValidName(typename) + if isinstance(callback, str): + callback = s_dyndeps.getDynLocal(callback) + + # if we were invoked via telepath, the schmea needs to be mutable... + schema = s_msgpack.deepcopy(schema, use_list=True) + + curv = await self.getTypeSchemaVersion(typename) + if vers is not None: vers = int(vers) - curv = self.getTypeSchemaVersion(typename) if curv is not None: if vers == curv: return False @@ -551,7 +560,7 @@ async def setTypeSchema(self, typename, schema, callback=None, vers=None): for lkey, byts in self.slab.scanByPref(LKEY_VERS + bidn, db=self.dbname): versindx = lkey[-9:] databyts = self.slab.get(LKEY_DATA + bidn + versindx, db=self.dbname) - data = await callback(info, s_msgpack.un(byts), s_msgpack.un(databyts)) + data = await callback(info, s_msgpack.un(byts), s_msgpack.un(databyts), curv) vtor(data) self.slab.put(LKEY_DATA + bidn + versindx, s_msgpack.en(data), db=self.dbname) await asyncio.sleep(0) @@ -565,12 +574,12 @@ async def getItemsByType(self, typename): if info is not None: yield info - def getTypeValidator(self, typename): + async def getTypeValidator(self, typename): vtor = self.validators.get(typename) if vtor is not None: return vtor - schema = self.getTypeSchema(typename) + schema = await self.getTypeSchema(typename) if schema is None: return None @@ -579,13 +588,19 @@ def getTypeValidator(self, typename): return vtor - def reqTypeValidator(self, typename): - vtor = self.getTypeValidator(typename) + async def reqTypeValidator(self, typename): + vtor = await self.getTypeValidator(typename) if vtor is not None: return vtor mesg = f'No schema registered with name: {typename}' raise s_exc.NoSuchType(mesg=mesg) - def reqValidData(self, typename, item): - self.reqTypeValidator(typename)(item) + async def reqValidData(self, typename, item): + return (await self.reqTypeValidator(typename))(item) + +class FileDrive(Drive, s_spawner.SpawnerMixin): + + async def __anit__(self, path): + slab = await s_lmdbslab.Slab.anit(path) + return await Drive.__anit__(self, slab, 'celldrive') diff --git a/synapse/lib/link.py b/synapse/lib/link.py index e30af4199af..29ef44d78fb 100644 --- a/synapse/lib/link.py +++ b/synapse/lib/link.py @@ -44,6 +44,23 @@ async def unixconnect(path): info = {'path': path, 'unix': True} return await Link.anit(reader, writer, info=info) +async def unixwait(path): + + while True: + try: + + reader, writer = await asyncio.open_unix_connection(path=path) + + reader._transport.abort() + + writer.close() + await writer.wait_closed() + + return + + except (ConnectionRefusedError, FileNotFoundError): + await asyncio.sleep(0.01) + async def linkfile(mode='wb'): ''' Connect a socketpair to a file-object and return (link, file). diff --git a/synapse/lib/spawner.py b/synapse/lib/spawner.py new file mode 100644 index 00000000000..fa322b7fd2a --- /dev/null +++ b/synapse/lib/spawner.py @@ -0,0 +1,80 @@ +import asyncio +import logging +import tempfile + +import synapse.exc as s_exc +import synapse.common as s_common +import synapse.daemon as s_daemon +import synapse.telepath as s_telepath + +import synapse.lib.base as s_base +import synapse.lib.link as s_link +import synapse.lib.process as s_process + +logger = logging.getLogger(__name__) + +def _ioWorkProc(todo, sockpath): + + async def workloop(): + + async with await s_daemon.Daemon.anit() as dmon: + + func, args, kwargs = todo + + item = await func(*args, **kwargs) + + dmon.share('dmon', dmon) + dmon.share('item', item) + + # bind last so we're ready to go... + await dmon.listen(f'unix://{sockpath}') + await item.waitfini() + + asyncio.run(workloop()) + +class SpawnerMixin: + + @classmethod + def spawner(cls, base=None, sockpath=None): + async def _spawn(*args, **kwargs): + return await cls._spawn(args, kwargs, base=base, sockpath=sockpath) + return _spawn + + @classmethod + async def _spawn(cls, args, kwargs, base=None, sockpath=None): + + todo = (cls.anit, args, kwargs) + + iden = s_common.guid() + + if sockpath is None: + tmpdir = tempfile.gettempdir() + sockpath = s_common.genpath(tmpdir, iden) + + if base is None: + base = await s_base.Base.anit() + + base.schedCoro(s_process.spawn((_ioWorkProc, (todo, sockpath), {}))) + + await s_link.unixwait(sockpath) + + proxy = await s_telepath.openurl(f'unix://{sockpath}:item') + + async def fini(): + + try: + async with await s_telepath.openurl(f'unix://{sockpath}:item') as finiproxy: + await finiproxy.task(('fini', (), {})) + except (s_exc.LinkErr, s_exc.NoSuchPath, asyncio.CancelledError): + # This can fail if the subprocess was terminated from outside... + pass + + if not base.isfini: + logger.error(f'IO Worker Socket Closed: {sockpath}') + + await base.fini() + + proxy.onfini(fini) + base.onfini(proxy) + + return proxy diff --git a/synapse/tests/test_lib_agenda.py b/synapse/tests/test_lib_agenda.py index 959b830da85..bb3a49db368 100644 --- a/synapse/tests/test_lib_agenda.py +++ b/synapse/tests/test_lib_agenda.py @@ -165,10 +165,10 @@ def timetime(): def looptime(): return unixtime - MONO_DELT - loop = asyncio.get_running_loop() - with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime), self.getTestDir() as dirn: + async with self.getTestCore() as core: - async with self.getTestCore() as core: + loop = asyncio.get_running_loop() + with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime): visi = await core.auth.addUser('visi') await visi.setAdmin(True) diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py index faa8a4df7f1..26af17116ce 100644 --- a/synapse/tests/test_lib_base.py +++ b/synapse/tests/test_lib_base.py @@ -10,6 +10,7 @@ import synapse.lib.base as s_base import synapse.lib.coro as s_coro import synapse.lib.scope as s_scope +import synapse.lib.spawner as s_spawner import synapse.tests.utils as s_t_utils @@ -45,6 +46,14 @@ async def postAnit(self): if self.foo == -1: raise s_exc.BadArg(mesg='boom') +class Haha(s_base.Base, s_spawner.SpawnerMixin): + + async def __anit__(self): + await s_base.Base.__anit__(self) + + async def fini(self): + await s_base.Base.fini(self) + class BaseTest(s_t_utils.SynTest): async def test_base_basics(self): @@ -570,3 +579,39 @@ async def func3(bobj, key, valu): # The scope data set in the task is not present outside of it. self.none(s_scope.get('hehe')) + + async def test_base_spawner_fini(self): + + # Test proxy fini, base should fini + base = await s_base.Base.anit() + spawner = Haha.spawner(base=base) + proxy = await spawner() + + self.false(proxy.isfini) + self.false(base.isfini) + + await proxy.fini() + self.true(proxy.isfini) + self.true(base.isfini) + + # Test base fini, proxy should fini + base = await s_base.Base.anit() + spawner = Haha.spawner(base=base) + proxy = await spawner() + + self.false(proxy.isfini) + self.false(base.isfini) + + await base.fini() + self.true(base.isfini) + self.true(proxy.isfini) + + async def test_base_spawner_none(self): + + spawner = Haha.spawner() + proxy = await spawner() + + self.false(proxy.isfini) + + await proxy.fini() + self.true(proxy.isfini) diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py index d522e10dd68..93d665690a8 100644 --- a/synapse/tests/test_lib_cell.py +++ b/synapse/tests/test_lib_cell.py @@ -28,7 +28,7 @@ import synapse.lib.coro as s_coro import synapse.lib.json as s_json import synapse.lib.link as s_link -import synapse.lib.drive as s_drive +import synapse.lib.const as s_const import synapse.lib.nexus as s_nexus import synapse.lib.config as s_config import synapse.lib.certdir as s_certdir @@ -172,43 +172,6 @@ def finilink(): cell.onfini(auth.fini) return auth -testDataSchema_v0 = { - 'type': 'object', - 'properties': { - 'type': {'type': 'string'}, - 'size': {'type': 'number'}, - 'stuff': {'type': ['number', 'null'], 'default': None} - }, - 'required': ['type', 'size', 'stuff'], - 'additionalProperties': False, -} - -testDataSchema_v1 = { - 'type': 'object', - 'properties': { - 'type': {'type': 'string'}, - 'size': {'type': 'number'}, - 'stuff': {'type': ['number', 'null'], 'default': None}, - 'woot': {'type': 'string'}, - 'blorp': { - 'type': 'object', - 'properties': { - 'bleep': { - 'type': 'array', - 'items': { - 'type': 'object', - 'properties': { - 'neato': {'type': 'string'} - } - } - } - } - } - }, - 'required': ['type', 'size', 'woot'], - 'additionalProperties': False, -} - class CellTest(s_t_utils.SynTest): async def test_cell_getLocalUrl(self): @@ -223,260 +186,6 @@ async def test_cell_getLocalUrl(self): url = cell.getLocalUrl(user='lowuser', share='*/view') self.eq(url, f'cell://lowuser@{dirn}:*/view') - async def test_cell_drive(self): - - with self.getTestDir() as dirn: - async with self.getTestCell(dirn=dirn) as cell: - - with self.raises(s_exc.BadName): - s_drive.reqValidName('A' * 512) - - info = {'name': 'users'} - pathinfo = await cell.addDriveItem(info) - - info = {'name': 'root'} - pathinfo = await cell.addDriveItem(info, path='users') - - with self.raises(s_exc.DupIden): - await cell.drive.addItemInfo(pathinfo[-1], path='users') - - rootdir = pathinfo[-1].get('iden') - self.eq(0, pathinfo[-1].get('kids')) - - info = {'name': 'win32k.sys', 'type': 'hehe'} - with self.raises(s_exc.NoSuchType): - info = await cell.addDriveItem(info, reldir=rootdir) - - infos = [i async for i in cell.getDriveKids(s_drive.rootdir)] - self.len(1, infos) - self.eq(1, infos[0].get('kids')) - self.eq('users', infos[0].get('name')) - - # TODO how to handle iden match with additional property mismatch - - self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0)) - self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) - self.false(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) - - with self.raises(s_exc.BadVersion): - await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0) - - info = {'name': 'win32k.sys', 'type': 'woot', 'perm': {'users': {}}} - info = await cell.addDriveItem(info, reldir=rootdir) - self.notin('perm', info) - self.eq(info[0]['permissions'], { - 'users': {}, - 'roles': {} - }) - - iden = info[-1].get('iden') - - tick = s_common.now() - rootuser = cell.auth.rootuser.iden - fooser = await cell.auth.addUser('foo') - neatrole = await cell.auth.addRole('neatrole') - await fooser.grant(neatrole.iden) - - with self.raises(s_exc.SchemaViolation): - versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) - - versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20, 'stuff': 12}) - self.eq(info.get('version'), (1, 1, 0)) - self.eq(versinfo.get('version'), (1, 1, 0)) - - versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 13}) - self.eq(info.get('version'), (1, 1, 0)) - self.eq(versinfo.get('version'), (1, 0, 0)) - - versinfo10, data10 = await cell.getDriveData(iden, vers=(1, 0, 0)) - self.eq(versinfo10.get('updated'), tick) - self.eq(versinfo10.get('updater'), rootuser) - self.eq(versinfo10.get('version'), (1, 0, 0)) - - versinfo11, data11 = await cell.getDriveData(iden, vers=(1, 1, 0)) - self.eq(versinfo11.get('updated'), tick + 10) - self.eq(versinfo11.get('updater'), rootuser) - self.eq(versinfo11.get('version'), (1, 1, 0)) - - versions = [vers async for vers in cell.getDriveDataVersions(iden)] - self.len(2, versions) - self.eq(versions[0], versinfo11) - self.eq(versions[1], versinfo10) - - info = await cell.delDriveData(iden, vers=(0, 0, 0)) - - versions = [vers async for vers in cell.getDriveDataVersions(iden)] - self.len(2, versions) - self.eq(versions[0], versinfo11) - self.eq(versions[1], versinfo10) - - info = await cell.delDriveData(iden, vers=(1, 1, 0)) - self.eq(info.get('updated'), tick) - self.eq(info.get('version'), (1, 0, 0)) - - info = await cell.delDriveData(iden, vers=(1, 0, 0)) - self.eq(info.get('size'), 0) - self.eq(info.get('version'), (0, 0, 0)) - self.none(info.get('updated')) - self.none(info.get('updater')) - - # repopulate a couple data versions to test migration and delete - versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 14}) - versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} - info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 17, 'stuff': 15}) - self.eq(versinfo, (await cell.getDriveData(iden))[0]) - - await cell.setDriveItemProp(iden, versinfo, ('stuff',), 1234) - data = await cell.getDriveData(iden) - self.eq(data[1]['stuff'], 1234) - - # This will be done by the cell in a cell storage version migration... - async def migrate_v1(info, versinfo, data): - data['woot'] = 'woot' - return data - - await cell.drive.setTypeSchema('woot', testDataSchema_v1, migrate_v1) - - versinfo['version'] = (1, 1, 1) - await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829) - data = await cell.getDriveData(iden) - self.eq(data[0]['version'], (1, 1, 1)) - self.eq(data[1]['stuff'], 3829) - - await self.asyncraises(s_exc.NoSuchIden, cell.setDriveItemProp(s_common.guid(), versinfo, ('lolnope',), 'not real')) - - await self.asyncraises(s_exc.BadArg, cell.setDriveItemProp(iden, versinfo, ('blorp', 0, 'neato'), 'my special string')) - data[1]['blorp'] = { - 'bleep': [{'neato': 'thing'}] - } - info, versinfo = await cell.setDriveData(iden, versinfo, data[1]) - now = s_common.now() - versinfo['updated'] = now - await cell.setDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato'), 'my special string') - data = await cell.getDriveData(iden) - self.eq(now, data[0]['updated']) - self.eq('my special string', data[1]['blorp']['bleep'][0]['neato']) - - versinfo['version'] = (1, 2, 1) - await cell.delDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato')) - vers, data = await cell.getDriveData(iden) - self.eq((1, 2, 1), vers['version']) - self.nn(data['blorp']['bleep'][0]) - self.notin('neato', data['blorp']['bleep'][0]) - - await self.asyncraises(s_exc.NoSuchIden, cell.delDriveItemProp(s_common.guid(), versinfo, 'blorp')) - - self.none(await cell.delDriveItemProp(iden, versinfo, ('lolnope', 'nopath'))) - - versinfo, data = await cell.getDriveData(iden, vers=(1, 0, 0)) - self.eq('woot', data.get('woot')) - - versinfo, data = await cell.getDriveData(iden, vers=(1, 1, 0)) - self.eq('woot', data.get('woot')) - - with self.raises(s_exc.NoSuchIden): - await cell.reqDriveInfo('d7d6107b200e2c039540fc627bc5537d') - - with self.raises(s_exc.TypeMismatch): - await cell.getDriveInfo(iden, typename='newp') - - self.nn(await cell.getDriveInfo(iden)) - self.len(4, [vers async for vers in cell.getDriveDataVersions(iden)]) - - await cell.delDriveData(iden) - self.len(3, [vers async for vers in cell.getDriveDataVersions(iden)]) - - await cell.delDriveInfo(iden) - - self.none(await cell.getDriveInfo(iden)) - self.len(0, [vers async for vers in cell.getDriveDataVersions(iden)]) - - with self.raises(s_exc.NoSuchPath): - await cell.getDrivePath('users/root/win32k.sys') - - pathinfo = await cell.addDrivePath('foo/bar/baz') - self.len(3, pathinfo) - self.eq('foo', pathinfo[0].get('name')) - self.eq(1, pathinfo[0].get('kids')) - self.eq('bar', pathinfo[1].get('name')) - self.eq(1, pathinfo[1].get('kids')) - self.eq('baz', pathinfo[2].get('name')) - self.eq(0, pathinfo[2].get('kids')) - - self.eq(pathinfo, await cell.addDrivePath('foo/bar/baz')) - - baziden = pathinfo[2].get('iden') - self.eq(pathinfo, await cell.drive.getItemPath(baziden)) - - info = await cell.setDriveInfoPerm(baziden, {'users': {rootuser: s_cell.PERM_ADMIN}, 'roles': {}}) - # make sure drive perms work with easy perms - self.true(cell._hasEasyPerm(info, cell.auth.rootuser, s_cell.PERM_ADMIN)) - # defaults to READ - self.true(cell._hasEasyPerm(info, fooser, s_cell.PERM_READ)) - self.false(cell._hasEasyPerm(info, fooser, s_cell.PERM_EDIT)) - - with self.raises(s_exc.NoSuchIden): - # s_drive.rootdir is all 00s... ;) - await cell.setDriveInfoPerm(s_drive.rootdir, {'users': {}, 'roles': {}}) - - await cell.addDrivePath('hehe/haha') - pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') - - self.eq('hoho', pathinfo[-1].get('name')) - self.eq(baziden, pathinfo[-1].get('iden')) - - self.true(await cell.drive.hasPathInfo('hehe/haha/hoho')) - self.false(await cell.drive.hasPathInfo('foo/bar/baz')) - - pathinfo = await cell.getDrivePath('foo/bar') - self.eq(0, pathinfo[-1].get('kids')) - - pathinfo = await cell.getDrivePath('hehe/haha') - self.eq(1, pathinfo[-1].get('kids')) - - with self.raises(s_exc.DupName): - iden = pathinfo[-2].get('iden') - name = pathinfo[-1].get('name') - cell.drive.reqFreeStep(iden, name) - - walks = [item async for item in cell.drive.walkPathInfo('hehe')] - self.len(3, walks) - # confirm walked paths are yielded depth first... - self.eq('hoho', walks[0].get('name')) - self.eq('haha', walks[1].get('name')) - self.eq('hehe', walks[2].get('name')) - - iden = walks[2].get('iden') - walks = [item async for item in cell.drive.walkItemInfo(iden)] - self.len(3, walks) - self.eq('hoho', walks[0].get('name')) - self.eq('haha', walks[1].get('name')) - self.eq('hehe', walks[2].get('name')) - - self.none(cell.drive.getTypeSchema('newp')) - - cell.drive.validators.pop('woot') - self.nn(cell.drive.getTypeValidator('woot')) - - # move to root dir - pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop') - self.len(1, pathinfo) - self.eq(s_drive.rootdir, pathinfo[-1].get('parent')) - - pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') - self.len(3, pathinfo) - - async with self.getTestCell(dirn=dirn) as cell: - data = {'type': 'woot', 'size': 20, 'stuff': 12, 'woot': 'woot'} - # explicitly clear out the cache JsValidators, otherwise we get the cached, pre-msgpack - # version of the validator, which will be correct and skip the point of this test. - s_config._JsValidators.clear() - cell.drive.reqValidData('woot', data) - async def test_cell_auth(self): with self.getTestDir() as dirn: @@ -712,44 +421,6 @@ async def test_cell_auth(self): with self.raises(s_exc.NeedConfValu): await echo.reqAhaProxy() - async def test_cell_drive_perm_migration(self): - async with self.getRegrCore('drive-perm-migr') as core: - item = await core.getDrivePath('driveitemdefaultperms') - self.len(1, item) - self.notin('perm', item) - self.eq(item[0]['permissions'], {'users': {}, 'roles': {}}) - - ldog = await core.auth.getRoleByName('littledog') - bdog = await core.auth.getRoleByName('bigdog') - - louis = await core.auth.getUserByName('lewis') - tim = await core.auth.getUserByName('tim') - mj = await core.auth.getUserByName('mj') - - item = await core.getDrivePath('permfolder/driveitemwithperms') - self.len(2, item) - self.notin('perm', item[0]) - self.notin('perm', item[1]) - self.eq(item[0]['permissions'], {'users': {tim.iden: s_cell.PERM_ADMIN}, 'roles': {}}) - self.eq(item[1]['permissions'], { - 'users': { - mj.iden: s_cell.PERM_ADMIN - }, - 'roles': { - ldog.iden: s_cell.PERM_READ, - bdog.iden: s_cell.PERM_EDIT, - }, - 'default': s_cell.PERM_DENY - }) - - # make sure it's all good with easy perms - self.true(core._hasEasyPerm(item[0], tim, s_cell.PERM_ADMIN)) - self.false(core._hasEasyPerm(item[0], mj, s_cell.PERM_EDIT)) - - self.true(core._hasEasyPerm(item[1], mj, s_cell.PERM_ADMIN)) - self.true(core._hasEasyPerm(item[1], tim, s_cell.PERM_READ)) - self.true(core._hasEasyPerm(item[1], louis, s_cell.PERM_EDIT)) - async def test_cell_unix_sock(self): async with self.getTestCore() as core: @@ -851,7 +522,7 @@ async def test_longpath(self): # but exercises the long-path failure inside of the cell's daemon # instead. with self.getTestDir() as dirn: - extrapath = 108 * 'A' + extrapath = s_const.UNIX_PATH_MAX * 'A' longdirn = s_common.genpath(dirn, extrapath) with self.getAsyncLoggerStream('synapse.lib.cell', 'LOCAL UNIX SOCKET WILL BE UNAVAILABLE') as stream: async with self.getTestCell(s_cell.Cell, dirn=longdirn) as cell: diff --git a/synapse/tests/test_lib_drive.py b/synapse/tests/test_lib_drive.py new file mode 100644 index 00000000000..dc8ee2120bb --- /dev/null +++ b/synapse/tests/test_lib_drive.py @@ -0,0 +1,342 @@ +import synapse.exc as s_exc +import synapse.common as s_common + +import synapse.lib.cell as s_cell +import synapse.lib.drive as s_drive +import synapse.lib.config as s_config + +import synapse.tests.utils as s_t_utils + +async def migrate_v1(info, versinfo, data, curv): + assert curv == 1 + data['woot'] = 'woot' + return data + +testDataSchema_v0 = { + 'type': 'object', + 'properties': { + 'type': {'type': 'string'}, + 'size': {'type': 'number'}, + 'stuff': {'type': ['number', 'null'], 'default': None} + }, + 'required': ['type', 'size', 'stuff'], + 'additionalProperties': False, +} + +testDataSchema_v1 = { + 'type': 'object', + 'properties': { + 'type': {'type': 'string'}, + 'size': {'type': 'number'}, + 'stuff': {'type': ['number', 'null'], 'default': None}, + 'woot': {'type': 'string'}, + 'blorp': { + 'type': 'object', + 'properties': { + 'bleep': { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'neato': {'type': 'string'} + } + } + } + } + } + }, + 'required': ['type', 'size', 'woot'], + 'additionalProperties': False, +} + +class DriveTest(s_t_utils.SynTest): + + async def test_drive_base(self): + + with self.getTestDir() as dirn: + async with self.getTestCell(dirn=dirn) as cell: + + with self.raises(s_exc.BadName): + s_drive.reqValidName('A' * 512) + + info = {'name': 'users'} + pathinfo = await cell.addDriveItem(info) + + info = {'name': 'root'} + pathinfo = await cell.addDriveItem(info, path='users') + + with self.raises(s_exc.DupIden): + await cell.drive.addItemInfo(pathinfo[-1], path='users') + + rootdir = pathinfo[-1].get('iden') + self.eq(0, pathinfo[-1].get('kids')) + + info = {'name': 'win32k.sys', 'type': 'hehe'} + with self.raises(s_exc.NoSuchType): + info = await cell.addDriveItem(info, reldir=rootdir) + + infos = [i async for i in cell.getDriveKids(s_drive.rootdir)] + self.len(1, infos) + self.eq(1, infos[0].get('kids')) + self.eq('users', infos[0].get('name')) + + # TODO how to handle iden match with additional property mismatch + + self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0)) + self.true(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) + self.false(await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=1)) + + with self.raises(s_exc.BadVersion): + await cell.drive.setTypeSchema('woot', testDataSchema_v0, vers=0) + + info = {'name': 'win32k.sys', 'type': 'woot', 'perm': {'users': {}}} + info = await cell.addDriveItem(info, reldir=rootdir) + self.notin('perm', info) + self.eq(info[0]['permissions'], { + 'users': {}, + 'roles': {} + }) + + iden = info[-1].get('iden') + + tick = s_common.now() + rootuser = cell.auth.rootuser.iden + fooser = await cell.auth.addUser('foo') + neatrole = await cell.auth.addRole('neatrole') + await fooser.grant(neatrole.iden) + + with self.raises(s_exc.SchemaViolation): + versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + await cell.setDriveData(iden, versinfo, {'newp': 'newp'}) + + versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 20, 'stuff': 12}) + self.eq(info.get('version'), (1, 1, 0)) + self.eq(versinfo.get('version'), (1, 1, 0)) + + versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 13}) + self.eq(info.get('version'), (1, 1, 0)) + self.eq(versinfo.get('version'), (1, 0, 0)) + + versinfo10, data10 = await cell.getDriveData(iden, vers=(1, 0, 0)) + self.eq(versinfo10.get('updated'), tick) + self.eq(versinfo10.get('updater'), rootuser) + self.eq(versinfo10.get('version'), (1, 0, 0)) + + versinfo11, data11 = await cell.getDriveData(iden, vers=(1, 1, 0)) + self.eq(versinfo11.get('updated'), tick + 10) + self.eq(versinfo11.get('updater'), rootuser) + self.eq(versinfo11.get('version'), (1, 1, 0)) + + versions = [vers async for vers in cell.getDriveDataVersions(iden)] + self.len(2, versions) + self.eq(versions[0], versinfo11) + self.eq(versions[1], versinfo10) + + info = await cell.delDriveData(iden, vers=(0, 0, 0)) + + versions = [vers async for vers in cell.getDriveDataVersions(iden)] + self.len(2, versions) + self.eq(versions[0], versinfo11) + self.eq(versions[1], versinfo10) + + info = await cell.delDriveData(iden, vers=(1, 1, 0)) + self.eq(info.get('updated'), tick) + self.eq(info.get('version'), (1, 0, 0)) + + info = await cell.delDriveData(iden, vers=(1, 0, 0)) + self.eq(info.get('size'), 0) + self.eq(info.get('version'), (0, 0, 0)) + self.none(info.get('updated')) + self.none(info.get('updater')) + + # repopulate a couple data versions to test migration and delete + versinfo = {'version': (1, 0, 0), 'updated': tick, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'hehe', 'size': 0, 'stuff': 14}) + versinfo = {'version': (1, 1, 0), 'updated': tick + 10, 'updater': rootuser} + info, versinfo = await cell.setDriveData(iden, versinfo, {'type': 'haha', 'size': 17, 'stuff': 15}) + self.eq(versinfo, (await cell.getDriveData(iden))[0]) + + await cell.setDriveItemProp(iden, versinfo, ('stuff',), 1234) + data = await cell.getDriveData(iden) + self.eq(data[1]['stuff'], 1234) + + # This will be done by the cell in a cell storage version migration... + callback = 'synapse.tests.test_lib_drive.migrate_v1' + await cell.drive.setTypeSchema('woot', testDataSchema_v1, callback=callback) + + await cell.setDriveItemProp(iden, versinfo, 'woot', 'woot') + + versinfo['version'] = (1, 1, 1) + await cell.setDriveItemProp(iden, versinfo, 'stuff', 3829) + data = await cell.getDriveData(iden) + self.eq(data[0]['version'], (1, 1, 1)) + self.eq(data[1]['stuff'], 3829) + + await self.asyncraises(s_exc.NoSuchIden, cell.setDriveItemProp(s_common.guid(), versinfo, ('lolnope',), 'not real')) + + await self.asyncraises(s_exc.BadArg, cell.setDriveItemProp(iden, versinfo, ('blorp', 0, 'neato'), 'my special string')) + data[1]['blorp'] = { + 'bleep': [{'neato': 'thing'}] + } + info, versinfo = await cell.setDriveData(iden, versinfo, data[1]) + now = s_common.now() + versinfo['updated'] = now + await cell.setDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato'), 'my special string') + data = await cell.getDriveData(iden) + self.eq(now, data[0]['updated']) + self.eq('my special string', data[1]['blorp']['bleep'][0]['neato']) + + versinfo['version'] = (1, 2, 1) + await cell.delDriveItemProp(iden, versinfo, ('blorp', 'bleep', 0, 'neato')) + vers, data = await cell.getDriveData(iden) + self.eq((1, 2, 1), vers['version']) + self.nn(data['blorp']['bleep'][0]) + self.notin('neato', data['blorp']['bleep'][0]) + + await self.asyncraises(s_exc.NoSuchIden, cell.delDriveItemProp(s_common.guid(), versinfo, 'blorp')) + + self.none(await cell.delDriveItemProp(iden, versinfo, ('lolnope', 'nopath'))) + + versinfo, data = await cell.getDriveData(iden, vers=(1, 0, 0)) + print(versinfo) + print(data) + self.eq('woot', data.get('woot')) + + versinfo, data = await cell.getDriveData(iden, vers=(1, 1, 0)) + self.eq('woot', data.get('woot')) + + with self.raises(s_exc.NoSuchIden): + await cell.reqDriveInfo('d7d6107b200e2c039540fc627bc5537d') + + with self.raises(s_exc.TypeMismatch): + await cell.getDriveInfo(iden, typename='newp') + + self.nn(await cell.getDriveInfo(iden)) + self.len(4, [vers async for vers in cell.getDriveDataVersions(iden)]) + + await cell.delDriveData(iden) + self.len(3, [vers async for vers in cell.getDriveDataVersions(iden)]) + + await cell.delDriveInfo(iden) + + self.none(await cell.getDriveInfo(iden)) + self.len(0, [vers async for vers in cell.getDriveDataVersions(iden)]) + + with self.raises(s_exc.NoSuchPath): + await cell.getDrivePath('users/root/win32k.sys') + + pathinfo = await cell.addDrivePath('foo/bar/baz') + self.len(3, pathinfo) + self.eq('foo', pathinfo[0].get('name')) + self.eq(1, pathinfo[0].get('kids')) + self.eq('bar', pathinfo[1].get('name')) + self.eq(1, pathinfo[1].get('kids')) + self.eq('baz', pathinfo[2].get('name')) + self.eq(0, pathinfo[2].get('kids')) + + self.eq(pathinfo, await cell.addDrivePath('foo/bar/baz')) + + baziden = pathinfo[2].get('iden') + self.eq(pathinfo, await cell.drive.getItemPath(baziden)) + + info = await cell.setDriveInfoPerm(baziden, {'users': {rootuser: s_cell.PERM_ADMIN}, 'roles': {}}) + # make sure drive perms work with easy perms + self.true(cell._hasEasyPerm(info, cell.auth.rootuser, s_cell.PERM_ADMIN)) + # defaults to READ + self.true(cell._hasEasyPerm(info, fooser, s_cell.PERM_READ)) + self.false(cell._hasEasyPerm(info, fooser, s_cell.PERM_EDIT)) + + with self.raises(s_exc.NoSuchIden): + # s_drive.rootdir is all 00s... ;) + await cell.setDriveInfoPerm(s_drive.rootdir, {'users': {}, 'roles': {}}) + + await cell.addDrivePath('hehe/haha') + pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') + + self.eq('hoho', pathinfo[-1].get('name')) + self.eq(baziden, pathinfo[-1].get('iden')) + + self.true(await cell.drive.hasPathInfo('hehe/haha/hoho')) + self.false(await cell.drive.hasPathInfo('foo/bar/baz')) + + pathinfo = await cell.getDrivePath('foo/bar') + self.eq(0, pathinfo[-1].get('kids')) + + pathinfo = await cell.getDrivePath('hehe/haha') + self.eq(1, pathinfo[-1].get('kids')) + + with self.raises(s_exc.DupName): + iden = pathinfo[-2].get('iden') + name = pathinfo[-1].get('name') + await cell.drive.reqFreeStep(iden, name) + + walks = [item async for item in cell.drive.walkPathInfo('hehe')] + self.len(3, walks) + # confirm walked paths are yielded depth first... + self.eq('hoho', walks[0].get('name')) + self.eq('haha', walks[1].get('name')) + self.eq('hehe', walks[2].get('name')) + + iden = walks[2].get('iden') + walks = [item async for item in cell.drive.walkItemInfo(iden)] + self.len(3, walks) + self.eq('hoho', walks[0].get('name')) + self.eq('haha', walks[1].get('name')) + self.eq('hehe', walks[2].get('name')) + + self.none(await cell.drive.getTypeSchema('newp')) + + # move to root dir + pathinfo = await cell.setDriveInfoPath(baziden, 'zipzop') + self.len(1, pathinfo) + self.eq(s_drive.rootdir, pathinfo[-1].get('parent')) + + pathinfo = await cell.setDriveInfoPath(baziden, 'hehe/haha/hoho') + self.len(3, pathinfo) + + async with self.getTestCell(dirn=dirn) as cell: + data = {'type': 'woot', 'size': 20, 'stuff': 12, 'woot': 'woot'} + # explicitly clear out the cache JsValidators, otherwise we get the cached, pre-msgpack + # version of the validator, which will be correct and skip the point of this test. + s_config._JsValidators.clear() + await cell.drive.reqValidData('woot', data) + + async def test_drive_perm_migration(self): + async with self.getRegrCore('drive-perm-migr') as core: + item = await core.getDrivePath('driveitemdefaultperms') + self.len(1, item) + self.notin('perm', item) + self.eq(item[0]['permissions'], {'users': {}, 'roles': {}}) + + ldog = await core.auth.getRoleByName('littledog') + bdog = await core.auth.getRoleByName('bigdog') + + louis = await core.auth.getUserByName('lewis') + tim = await core.auth.getUserByName('tim') + mj = await core.auth.getUserByName('mj') + + item = await core.getDrivePath('permfolder/driveitemwithperms') + self.len(2, item) + self.notin('perm', item[0]) + self.notin('perm', item[1]) + self.eq(item[0]['permissions'], {'users': {tim.iden: s_cell.PERM_ADMIN}, 'roles': {}}) + self.eq(item[1]['permissions'], { + 'users': { + mj.iden: s_cell.PERM_ADMIN + }, + 'roles': { + ldog.iden: s_cell.PERM_READ, + bdog.iden: s_cell.PERM_EDIT, + }, + 'default': s_cell.PERM_DENY + }) + + # make sure it's all good with easy perms + self.true(core._hasEasyPerm(item[0], tim, s_cell.PERM_ADMIN)) + self.false(core._hasEasyPerm(item[0], mj, s_cell.PERM_EDIT)) + + self.true(core._hasEasyPerm(item[1], mj, s_cell.PERM_ADMIN)) + self.true(core._hasEasyPerm(item[1], tim, s_cell.PERM_READ)) + self.true(core._hasEasyPerm(item[1], louis, s_cell.PERM_EDIT)) diff --git a/synapse/tests/test_lib_stormtypes.py b/synapse/tests/test_lib_stormtypes.py index af57db570a7..df1a1c6c516 100644 --- a/synapse/tests/test_lib_stormtypes.py +++ b/synapse/tests/test_lib_stormtypes.py @@ -5691,11 +5691,10 @@ def timetime(): def looptime(): return unixtime - MONO_DELT - loop = asyncio.get_running_loop() - - with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime): + async with self.getTestCoreAndProxy() as (core, prox): - async with self.getTestCoreAndProxy() as (core, prox): + loop = asyncio.get_running_loop() + with mock.patch.object(loop, 'time', looptime), mock.patch('time.time', timetime): mesgs = await core.stormlist('cron.list') self.stormIsInPrint('No cron jobs found', mesgs)