diff --git a/synapse/tests/test_tools_migrate3x.py b/synapse/tests/test_tools_migrate3x.py index 55073404a2..b6a6dbcc83 100644 --- a/synapse/tests/test_tools_migrate3x.py +++ b/synapse/tests/test_tools_migrate3x.py @@ -243,3 +243,55 @@ async def test_migr_layeroffs(self): self.len(1, pulls) pdef = list(pulls.values())[0] self.eq(23, pdef.get('offs', 0)) + + async def test_migr_auth_rules_cron(self): + conf = { + 'src': None, + 'dest': None, + } + + async with self._getTestMigrCore(conf, regrname='cron-creator-to-user') as (migr, dest): + + await migr.migrate() + await migr.fini() + + async with await s_cortex.Cortex.anit(dest, conf=None) as core: + + auth = core.auth + + user = await auth.reqUserByName('testuser') + role = await auth.reqRoleByName('testrole') + + urules = user.getRules() + self.isin((True, ('cron', 'set', 'user')), urules) + + rrules = role.getRules() + self.isin((True, ('cron', 'set', 'user', 'extra')), rrules) + + async def test_migr_auth_rules_profile(self): + conf = { + 'src': None, + 'dest': None, + } + + async with self._getTestMigrCore(conf, regrname='2.192.0-auth-rules-migr') as (migr, dest): + + await migr.migrate() + await migr.fini() + + async with await s_cortex.Cortex.anit(dest, conf=None) as core: + + auth = core.auth + + user = await auth.reqUserByName('visi') + role = await auth.reqRoleByName('visi-role') + + # user rules: auth.user.*.profile.* -> auth.user.profile.*.* + urules = user.getRules() + self.isin((True, ('auth', 'user', 'profile', 'get', 'fullname')), urules) + self.isin((True, ('auth', 'user', 'profile', 'set', 'fullname')), urules) + self.isin((True, ('auth', 'user', 'profile', 'del', 'fullname')), urules) + + # role rules: auth.role.*.profile.* -> auth.role.profile.*.* + rrules = role.getRules() + self.isin((False, ('auth', 'user', 'profile', 'del', 'nickname')), rrules) diff --git a/synapse/tools/cortex/migrate3x.py b/synapse/tools/cortex/migrate3x.py index b044e677fb..a296dfdc2c 100644 --- a/synapse/tools/cortex/migrate3x.py +++ b/synapse/tools/cortex/migrate3x.py @@ -34,6 +34,77 @@ REQ_2X_CORE_VERS = '>=2.180.1,<3.0.0' +class MigrAuth: + ''' + Helper for migrating auth related data during 2.x.x to 3.x.x migration. + ''' + + def __init__(self, migr, authkv): + self.migr = migr + self.authkv = authkv + + def migrate(self): + self._migrUsers() + self._migrRoles() + + def _migrUsers(self): + userkv = self.authkv.getSubKeyVal('user:info:') + + for iden, info in userkv.items(): + updated = False + + valu = info.get('onepass') + if valu is not None and not isinstance(valu, dict): + logger.warning(f'Removing deprecated one time password shadow for user {iden}!') + info.pop('onepass') + updated = True + + valu = info.get('passwd') + if valu is not None and not isinstance(valu, dict): + logger.warning(f'Removing deprecated password shadow for user {iden}!') + info.pop('passwd') + updated = True + + self._migrRules(info) + updated = True + + if updated: + userkv.set(iden, info) + + def _migrRoles(self): + rolekv = self.authkv.getSubKeyVal('role:info:') + for iden, info in rolekv.items(): + self._migrRules(info) + rolekv.set(iden, info) + + def _migrRules(self, info): + rules = [] + for allow, path in info.get('rules', ()): + if (newpath := self._migrRulePath(path)) is not None: + rules.append((allow, newpath)) + + info['rules'] = rules + + for gateiden, gateinfo in list(info.get('authgates').items()): + rules = [] + for allow, path in gateinfo.get('rules', ()): + if (newpath := self._migrRulePath(path)) is not None: + rules.append((allow, newpath)) + + gateinfo['rules'] = rules + + def _migrRulePath(self, path): + path = self.migr._migrRulePath(path) + if path is None: + return None + + if len(path) >= 4 and path[0] == 'auth' and path[1] == 'user' and path[3] == 'profile': + action = path[2] + rest = path[4:] + return ('auth', 'user', 'profile', action, *rest) + + return path + class Migrator(s_base.Base): ''' Standalone tool for migrating Synapse from a source Cortex to a new destination 3.x.x Cortex. @@ -528,6 +599,9 @@ async def _migrDirn(self): return locallyrs def _migrRulePath(self, path): + ''' + Generic rule paths, runs first before auth paths. + ''' for part in path: if '.' in part: @@ -586,26 +660,6 @@ def _migrRulePath(self, path): return path - def _migrRules(self, info): - - rules = [] - for allow, path in info.get('rules', ()): - if (newpath := self._migrRulePath(path)) is not None: - rules.append((allow, newpath)) - - info['rules'] = rules - - for gateiden, gateinfo in list(info.get('authgates').items()): - rules = [] - for allow, path in gateinfo.get('rules', ()): - if (newpath := self._migrRulePath(path)) is not None: - rules.append((allow, newpath)) - - gateinfo['rules'] = rules - - if not rules and not gateinfo.get('admin'): - info['authgates'].pop(gateiden) - async def _migrCell(self): ''' Migrate top-level cell information including the YAML file if it exists to @@ -628,27 +682,11 @@ async def _migrCell(self): self.cellslab.dropdb('hive') authkv = self.cellslab.getSafeKeyVal('auth') - userkv = authkv.getSubKeyVal('user:info:') - - for iden, info in userkv.items(): - if not ((valu := info.get('onepass')) is None or isinstance(valu, dict)): - logger.warning(f'Removing deprecated one time password shadow for user {iden}!') - info.pop('onepass') - - if not ((valu := info.get('passwd')) is None or isinstance(valu, dict)): - logger.warning(f'Removing deprecated password shadow for user {iden}!') - info.pop('passwd') - self._migrRules(info) - - userkv.set(iden, info) - - rolekv = authkv.getSubKeyVal('role:info:') - - for iden, info in rolekv.items(): - self._migrRules(info) + migrauth = MigrAuth(self, authkv) + migrauth.migrate() - rolekv.set(iden, info) + userkv = authkv.getSubKeyVal('user:info:') for viewiden in self.viewdefs.keys(): trigdict = self.cortexdata.getSubKeyVal(f'view:{viewiden}:trigger:') @@ -824,14 +862,16 @@ async def _migrNexslog(self): async with await s_multislabseqn.MultiSlabSeqn.anit(spath) as srclog, \ await s_multislabseqn.MultiSlabSeqn.anit(dpath) as dstlog: + kwargs = {} + meta = None + etime = s_common.now() + # Check for entries in nodeeditlogs with offsets before the start of a trimmed nexus log if (logstrt := srclog.firstindx) > 0: async def wrapgenr(iden, genr): async for offs, realedits in genr: yield offs, realedits, iden - - kwargs = {} genrs = [wrapgenr(iden, seqn.aiter(0, wait=False)) for iden, seqn in editlogs.items()] async for offs, realedits, iden in s_common.merggenr2(genrs):