From bd0421196db58ebc1933c06b14d42e83551b5aeb Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Mon, 2 Mar 2026 15:15:14 -0500 Subject: [PATCH 1/4] migr auth user profile rules --- synapse/tests/test_tools_migrate3x.py | 52 ++++++++ synapse/tools/migrate3x.py | 164 ++++++++++++++++++-------- 2 files changed, 164 insertions(+), 52 deletions(-) diff --git a/synapse/tests/test_tools_migrate3x.py b/synapse/tests/test_tools_migrate3x.py index acdfaff9e78..146a1f37de9 100644 --- a/synapse/tests/test_tools_migrate3x.py +++ b/synapse/tests/test_tools_migrate3x.py @@ -128,3 +128,55 @@ async def test_migr_layeroffs(self): self.len(1, pulls) pdef = list(pulls.values())[0] self.eq(23, pdef.get('offs', 0)) + + async def test_migr_auth_rules_cron(self): + conf = { + 'src': None, + 'dest': None, + } + + async with self._getTestMigrCore(conf, regrname='cron-creator-to-user') as (migr, dest): + + await migr.migrate() + await migr.fini() + + async with await s_cortex.Cortex.anit(dest, conf=None) as core: + + auth = core.auth + + user = await auth.reqUserByName('testuser') + role = await auth.reqRoleByName('testrole') + + urules = user.getRules() + self.isin((True, ('cron', 'set', 'user')), urules) + + rrules = role.getRules() + self.isin((True, ('cron', 'set', 'user', 'extra')), rrules) + + async def test_migr_auth_rules_profile(self): + conf = { + 'src': None, + 'dest': None, + } + + async with self._getTestMigrCore(conf, regrname='2.192.0-auth-rules-migr') as (migr, dest): + + await migr.migrate() + await migr.fini() + + async with await s_cortex.Cortex.anit(dest, conf=None) as core: + + auth = core.auth + + user = await auth.reqUserByName('visi') + role = await auth.reqRoleByName('visi-role') + + # user rules: auth.user.*.profile.* -> auth.user.profile.*.* + urules = user.getRules() + self.isin((True, ('auth', 'user', 'profile', 'get', 'fullname')), urules) + self.isin((True, ('auth', 'user', 'profile', 'set', 'fullname')), urules) + self.isin((True, ('auth', 'user', 'profile', 'pop', 'fullname')), urules) + + # role rules: auth.role.*.profile.* -> auth.role.profile.*.* + rrules = role.getRules() + self.isin((False, ('auth', 'user', 'profile', 'del', 'nickname')), rrules) diff --git a/synapse/tools/migrate3x.py b/synapse/tools/migrate3x.py index f00bac60fca..ab919e57888 100644 --- a/synapse/tools/migrate3x.py +++ b/synapse/tools/migrate3x.py @@ -28,12 +28,96 @@ import synapse.lib.lmdbslab as s_lmdbslab import synapse.lib.multislabseqn as s_multislabseqn -import synapse.tools.backup as s_backup +import synapse.tools.service.backup as s_backup logger = logging.getLogger(__name__) REQ_2X_CORE_VERS = '>=2.180.1,<3.0.0' +class MigrAuth: + ''' + Helper for migrating auth related data during 2.x.x to 3.x.x migration. + ''' + + def __init__(self, authkv): + self.authkv = authkv + self.permmigrs = ( + (('cron', 'set', 'creator'), ('cron', 'set', 'user')), + ) + + def migrate(self): + self._migrUsers() + self._migrRoles() + + def _migrUsers(self): + userkv = self.authkv.getSubKeyVal('user:info:') + + for iden, info in userkv.items(): + update = False + + valu = info.get('onepass') + if valu is not None and not isinstance(valu, dict): + logger.warning(f'Removing deprecated one time password shadow for user {iden}!') + update = True + info.pop('onepass') + + valu = info.get('passwd') + if valu is not None and not isinstance(valu, dict): + logger.warning(f'Removing deprecated password shadow for user {iden}!') + update = True + info.pop('passwd') + + if self._migrRulerInfo(info): + update = True + + if update: + userkv.set(iden, info) + + def _migrRoles(self): + rolekv = self.authkv.getSubKeyVal('role:info:') + for iden, info in rolekv.items(): + if self._migrRulerInfo(info): + rolekv.set(iden, info) + + def _migrRulerInfo(self, info): + changed = False + rules, upd = self._migrRules(info.get('rules', ())) + if upd: + info['rules'] = rules + changed = True + + return changed + + def _migrRules(self, rules): + if not rules: + return rules, False + + changed = False + newrules = [] + + for allow, path in rules: + newpath = self._migrRulePath(path) + if newpath != path: + changed = True + newrules.append((allow, newpath)) + + return newrules, changed + + def _migrRulePath(self, path): + for oldperm, newperm in self.permmigrs: + olen = len(oldperm) + if len(path) >= olen and tuple(path[:olen]) == oldperm: + return tuple(newperm + tuple(path[olen:])) + + if len(path) >= 4: + if path[0] == 'auth' and path[1] == 'user' and path[3] == 'profile': + action = path[2] + rest = path[4:] + newpath = ('auth', 'user', 'profile', action, *rest) + return tuple(newpath) + + return path + class Migrator(s_base.Base): ''' Standalone tool for migrating Synapse from a source Cortex to a new destination 3.x.x Cortex. @@ -509,55 +593,11 @@ async def _migrCell(self): self.cellslab.dropdb('hive') authkv = self.cellslab.getSafeKeyVal('auth') - userkv = authkv.getSubKeyVal('user:info:') - - permmigrs = ( - (('cron', 'set', 'creator'), ('cron', 'set', 'user')), - ) - - for iden, info in userkv.items(): - update = False - if not ((valu := info.get('onepass')) is None or isinstance(valu, dict)): - logger.warning(f'Removing deprecated one time password shadow for user {iden}!') - update = True - info.pop('onepass') - - if not ((valu := info.get('passwd')) is None or isinstance(valu, dict)): - logger.warning(f'Removing deprecated password shadow for user {iden}!') - update = True - info.pop('passwd') - - rules = [] - for allow, path in info.get('rules', ()): - for oldperm, newperm in permmigrs: - if path[:3] == oldperm: - update = True - rules.append((allow, newperm + path[3:])) - continue - rules.append((allow, path)) - - info['rules'] = rules - - if update: - userkv.set(iden, info) - - rolekv = authkv.getSubKeyVal('role:info:') - - for iden, info in rolekv.items(): - update = False - rules = [] - for allow, path in info.get('rules', ()): - for oldperm, newperm in permmigrs: - if path[:3] == oldperm: - update = True - rules.append((allow, newperm + path[3:])) - continue - rules.append((allow, path)) - info['rules'] = rules + migrauth = MigrAuth(authkv) + migrauth.migrate() - if update: - rolekv.set(iden, info) + userkv = authkv.getSubKeyVal('user:info:') for viewiden in self.viewdefs.keys(): trigdict = self.cortexdata.getSubKeyVal(f'view:{viewiden}:trigger:') @@ -607,6 +647,7 @@ async def _migrCell(self): async def _migrNexslog(self): editlogs = {} + migrauth = MigrAuth(self.cellslab.getSafeKeyVal('auth')) for iden, layrinfo in self.layrdefs.items(): path = os.path.join(self.src, 'layers', iden, 'nodeedits.lmdb') @@ -621,7 +662,17 @@ async def _migrNexslog(self): async for offs, item in s_coro.pause(srclog.iter(0)): if item[1] != 'edits': - await dstlog.add(item + (None,), indx=offs) + nexsiden, event, args, kwargs, meta = item + + if event in ('user:info', 'role:info') and len(args) >= 3 and args[1] == 'rules': + rules = args[2] + newrules, upd = migrauth._migrRules(rules) + if upd: + if isinstance(rules, tuple): + newrules = tuple(newrules) + args = args[:2] + (newrules,) + args[3:] + + await dstlog.add((nexsiden, event, args, kwargs, meta, None), indx=offs) continue nexsiden, event, args, kwargs, _ = item @@ -720,8 +771,13 @@ async def _migrDatamodel(self): for name, fdef in self.oldmodel['forms'].items(): if name in self.formmigr: - if (normopts := self.formmigr[name][1]) is not None and 'type' not in normopts: - normopts['type'] = self.model.form(normopts['name']).type + normopts = self.formmigr[name][1] + if normopts is not None and 'type' not in normopts: + form = self.model.form(normopts['name']) + if form is None: + nomigr.append(f'form={name}') + continue + normopts['type'] = form.type continue newname = name @@ -778,6 +834,10 @@ async def _migrDatamodel(self): if (tmigr := self.formmigr.get(oldtname)) is not None: oldtname = tmigr[1]['name'] + if getattr(prop, 'typedef', None) is None: + nomigr.append(f'prop={propfull}') + continue + newtname, newtopts = prop.typedef oldtopts = s_common.flatten(oldtopts) newtopts = s_common.flatten(newtopts) @@ -1208,7 +1268,7 @@ async def _migrlogAdd(self, migrop, logtyp, key, val): lkey = migrop.encode() + b'\x00' + logtyp.encode() + b'\x00' + bkey lval = s_msgpack.en(val) - self.migrslab.put(lkey, lval, overwrite=True, db=self.migrdb) + await self.migrslab.put(lkey, lval, overwrite=True, db=self.migrdb) except asyncio.CancelledError: # pragma: no cover raise From 70e149e24e7329ed191f15a99de3e539ac48be8e Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Tue, 3 Mar 2026 10:31:31 -0500 Subject: [PATCH 2/4] Reset file --- synapse/tests/test_tools_migrate3x.py | 52 --------------------------- 1 file changed, 52 deletions(-) diff --git a/synapse/tests/test_tools_migrate3x.py b/synapse/tests/test_tools_migrate3x.py index 38053cd1be3..55073404a28 100644 --- a/synapse/tests/test_tools_migrate3x.py +++ b/synapse/tests/test_tools_migrate3x.py @@ -243,55 +243,3 @@ async def test_migr_layeroffs(self): self.len(1, pulls) pdef = list(pulls.values())[0] self.eq(23, pdef.get('offs', 0)) - - async def test_migr_auth_rules_cron(self): - conf = { - 'src': None, - 'dest': None, - } - - async with self._getTestMigrCore(conf, regrname='cron-creator-to-user') as (migr, dest): - - await migr.migrate() - await migr.fini() - - async with await s_cortex.Cortex.anit(dest, conf=None) as core: - - auth = core.auth - - user = await auth.reqUserByName('testuser') - role = await auth.reqRoleByName('testrole') - - urules = user.getRules() - self.isin((True, ('cron', 'set', 'user')), urules) - - rrules = role.getRules() - self.isin((True, ('cron', 'set', 'user', 'extra')), rrules) - - async def test_migr_auth_rules_profile(self): - conf = { - 'src': None, - 'dest': None, - } - - async with self._getTestMigrCore(conf, regrname='2.192.0-auth-rules-migr') as (migr, dest): - - await migr.migrate() - await migr.fini() - - async with await s_cortex.Cortex.anit(dest, conf=None) as core: - - auth = core.auth - - user = await auth.reqUserByName('visi') - role = await auth.reqRoleByName('visi-role') - - # user rules: auth.user.*.profile.* -> auth.user.profile.*.* - urules = user.getRules() - self.isin((True, ('auth', 'user', 'profile', 'get', 'fullname')), urules) - self.isin((True, ('auth', 'user', 'profile', 'set', 'fullname')), urules) - self.isin((True, ('auth', 'user', 'profile', 'pop', 'fullname')), urules) - - # role rules: auth.role.*.profile.* -> auth.role.profile.*.* - rrules = role.getRules() - self.isin((False, ('auth', 'user', 'profile', 'del', 'nickname')), rrules) From 82e3c8d02e53e4475569a4c5d3f1a795fafde638 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Tue, 3 Mar 2026 14:41:18 -0500 Subject: [PATCH 3/4] updates --- synapse/tests/test_tools_migrate3x.py | 52 +++++++++++ synapse/tools/cortex/migrate3x.py | 119 +++++++++++++++++--------- 2 files changed, 130 insertions(+), 41 deletions(-) diff --git a/synapse/tests/test_tools_migrate3x.py b/synapse/tests/test_tools_migrate3x.py index 55073404a28..b6a6dbcc839 100644 --- a/synapse/tests/test_tools_migrate3x.py +++ b/synapse/tests/test_tools_migrate3x.py @@ -243,3 +243,55 @@ async def test_migr_layeroffs(self): self.len(1, pulls) pdef = list(pulls.values())[0] self.eq(23, pdef.get('offs', 0)) + + async def test_migr_auth_rules_cron(self): + conf = { + 'src': None, + 'dest': None, + } + + async with self._getTestMigrCore(conf, regrname='cron-creator-to-user') as (migr, dest): + + await migr.migrate() + await migr.fini() + + async with await s_cortex.Cortex.anit(dest, conf=None) as core: + + auth = core.auth + + user = await auth.reqUserByName('testuser') + role = await auth.reqRoleByName('testrole') + + urules = user.getRules() + self.isin((True, ('cron', 'set', 'user')), urules) + + rrules = role.getRules() + self.isin((True, ('cron', 'set', 'user', 'extra')), rrules) + + async def test_migr_auth_rules_profile(self): + conf = { + 'src': None, + 'dest': None, + } + + async with self._getTestMigrCore(conf, regrname='2.192.0-auth-rules-migr') as (migr, dest): + + await migr.migrate() + await migr.fini() + + async with await s_cortex.Cortex.anit(dest, conf=None) as core: + + auth = core.auth + + user = await auth.reqUserByName('visi') + role = await auth.reqRoleByName('visi-role') + + # user rules: auth.user.*.profile.* -> auth.user.profile.*.* + urules = user.getRules() + self.isin((True, ('auth', 'user', 'profile', 'get', 'fullname')), urules) + self.isin((True, ('auth', 'user', 'profile', 'set', 'fullname')), urules) + self.isin((True, ('auth', 'user', 'profile', 'del', 'fullname')), urules) + + # role rules: auth.role.*.profile.* -> auth.role.profile.*.* + rrules = role.getRules() + self.isin((False, ('auth', 'user', 'profile', 'del', 'nickname')), rrules) diff --git a/synapse/tools/cortex/migrate3x.py b/synapse/tools/cortex/migrate3x.py index b044e677fbb..b51f1cae44c 100644 --- a/synapse/tools/cortex/migrate3x.py +++ b/synapse/tools/cortex/migrate3x.py @@ -34,6 +34,74 @@ REQ_2X_CORE_VERS = '>=2.180.1,<3.0.0' +class MigrAuth: + ''' + Helper for migrating auth related data during 2.x.x to 3.x.x migration. + ''' + + def __init__(self, migr, authkv): + self.migr = migr + self.authkv = authkv + + def migrate(self): + self._migrUsers() + self._migrRoles() + + def _migrUsers(self): + userkv = self.authkv.getSubKeyVal('user:info:') + + for iden, info in userkv.items(): + updated = False + + valu = info.get('onepass') + if valu is not None and not isinstance(valu, dict): + logger.warning(f'Removing deprecated one time password shadow for user {iden}!') + info.pop('onepass') + updated = True + + valu = info.get('passwd') + if valu is not None and not isinstance(valu, dict): + logger.warning(f'Removing deprecated password shadow for user {iden}!') + info.pop('passwd') + updated = True + + self._migrRules(info) + updated = True + + if updated: + userkv.set(iden, info) + + def _migrRoles(self): + rolekv = self.authkv.getSubKeyVal('role:info:') + for iden, info in rolekv.items(): + self._migrRules(info) + rolekv.set(iden, info) + + def _migrRules(self, info): + rules = [] + for allow, path in info.get('rules', ()): + if (newpath := self._migrRulePath(path)) is not None: + rules.append((allow, newpath)) + + info['rules'] = rules + + for gateiden, gateinfo in list(info.get('authgates').items()): + rules = [] + for allow, path in gateinfo.get('rules', ()): + if (newpath := self._migrRulePath(path)) is not None: + rules.append((allow, newpath)) + + gateinfo['rules'] = rules + + def _migrRulePath(self, path): + path = self.migr._migrRulePath(path) + if len(path) >= 4 and path[0] == 'auth' and path[1] == 'user' and path[3] == 'profile': + action = path[2] + rest = path[4:] + return ('auth', 'user', 'profile', action, *rest) + + return path + class Migrator(s_base.Base): ''' Standalone tool for migrating Synapse from a source Cortex to a new destination 3.x.x Cortex. @@ -528,6 +596,9 @@ async def _migrDirn(self): return locallyrs def _migrRulePath(self, path): + ''' + Generic rule paths, runs first before auth paths. + ''' for part in path: if '.' in part: @@ -586,26 +657,6 @@ def _migrRulePath(self, path): return path - def _migrRules(self, info): - - rules = [] - for allow, path in info.get('rules', ()): - if (newpath := self._migrRulePath(path)) is not None: - rules.append((allow, newpath)) - - info['rules'] = rules - - for gateiden, gateinfo in list(info.get('authgates').items()): - rules = [] - for allow, path in gateinfo.get('rules', ()): - if (newpath := self._migrRulePath(path)) is not None: - rules.append((allow, newpath)) - - gateinfo['rules'] = rules - - if not rules and not gateinfo.get('admin'): - info['authgates'].pop(gateiden) - async def _migrCell(self): ''' Migrate top-level cell information including the YAML file if it exists to @@ -628,27 +679,11 @@ async def _migrCell(self): self.cellslab.dropdb('hive') authkv = self.cellslab.getSafeKeyVal('auth') - userkv = authkv.getSubKeyVal('user:info:') - - for iden, info in userkv.items(): - if not ((valu := info.get('onepass')) is None or isinstance(valu, dict)): - logger.warning(f'Removing deprecated one time password shadow for user {iden}!') - info.pop('onepass') - if not ((valu := info.get('passwd')) is None or isinstance(valu, dict)): - logger.warning(f'Removing deprecated password shadow for user {iden}!') - info.pop('passwd') - - self._migrRules(info) + migrauth = MigrAuth(self, authkv) + migrauth.migrate() - userkv.set(iden, info) - - rolekv = authkv.getSubKeyVal('role:info:') - - for iden, info in rolekv.items(): - self._migrRules(info) - - rolekv.set(iden, info) + userkv = authkv.getSubKeyVal('user:info:') for viewiden in self.viewdefs.keys(): trigdict = self.cortexdata.getSubKeyVal(f'view:{viewiden}:trigger:') @@ -824,14 +859,16 @@ async def _migrNexslog(self): async with await s_multislabseqn.MultiSlabSeqn.anit(spath) as srclog, \ await s_multislabseqn.MultiSlabSeqn.anit(dpath) as dstlog: + kwargs = {} + meta = None + etime = s_common.now() + # Check for entries in nodeeditlogs with offsets before the start of a trimmed nexus log if (logstrt := srclog.firstindx) > 0: async def wrapgenr(iden, genr): async for offs, realedits in genr: yield offs, realedits, iden - - kwargs = {} genrs = [wrapgenr(iden, seqn.aiter(0, wait=False)) for iden, seqn in editlogs.items()] async for offs, realedits, iden in s_common.merggenr2(genrs): From e644df46b83bf996cefb6f3a7110726662044fa5 Mon Sep 17 00:00:00 2001 From: OCBender <181250370+OCBender@users.noreply.github.com> Date: Wed, 4 Mar 2026 11:01:39 -0500 Subject: [PATCH 4/4] updates --- synapse/tools/cortex/migrate3x.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/synapse/tools/cortex/migrate3x.py b/synapse/tools/cortex/migrate3x.py index b51f1cae44c..a296dfdc2c1 100644 --- a/synapse/tools/cortex/migrate3x.py +++ b/synapse/tools/cortex/migrate3x.py @@ -95,6 +95,9 @@ def _migrRules(self, info): def _migrRulePath(self, path): path = self.migr._migrRulePath(path) + if path is None: + return None + if len(path) >= 4 and path[0] == 'auth' and path[1] == 'user' and path[3] == 'profile': action = path[2] rest = path[4:]