From 933e1aa97fb6999678336b30bf8df785757a6e23 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Thu, 27 Jun 2024 17:31:55 -0400
Subject: [PATCH 01/68] WIP: step1: aha:network default

---
 synapse/lib/aha.py            | 181 ++++++++++++++++++++++------------
 synapse/lib/cell.py           |  37 ++++---
 synapse/tests/test_lib_aha.py | 148 ++++++++++-----------------
 synapse/tools/aha/enroll.py   |   3 +-
 4 files changed, 199 insertions(+), 170 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 0e495bdd8f8..f3738eaf52e 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -380,17 +380,17 @@ async def getUserInfo(self):
         return {
             'aha:urls': self.aha._getAhaUrls(),
             'aha:user': self.userinfo.get('name'),
-            'aha:network': self.aha.conf.get('aha:network'),
+            'aha:network': self.aha._getAhaNetwork(),
         }
 
     async def getCaCert(self):
-        ahanetw = self.aha.conf.get('aha:network')
+        ahanetw = self.aha._getAhaNetwork()
         return self.aha.certdir.getCaCertBytes(ahanetw)
 
     async def signUserCsr(self, byts):
 
         ahauser = self.userinfo.get('name')
-        ahanetw = self.aha.conf.get('aha:network')
+        ahanetw = self.aha._getAhaNetwork()
 
         username = f'{ahauser}@{ahanetw}'
 
@@ -416,7 +416,7 @@ async def getProvInfo(self):
         return self.provinfo
 
     async def getCaCert(self):
-        ahanetw = self.aha.conf.get('aha:network')
+        ahanetw = self.aha._getAhaNetwork()
         return self.aha.certdir.getCaCertBytes(ahanetw)
 
     async def signHostCsr(self, byts):
@@ -464,6 +464,10 @@ class AhaCell(s_cell.Cell):
     confbase['mirror']['hidedocs'] = False  # type: ignore
     confbase['mirror']['hidecmdl'] = False  # type: ignore
     confdefs = {
+        'dns:name': {
+            'description': 'The registered DNS name used to reach the AHA service.',
+            'type': ['string', 'null'],
+        },
         'aha:urls': {
             'description': 'A list of all available AHA server URLs.',
             'type': ['string', 'array'],
@@ -537,41 +541,95 @@ def _initCellHttpApis(self):
         self.addHttpApi('/api/v1/aha/provision/service', AhaProvisionServiceV1, {'cell': self})
 
     async def initServiceRuntime(self):
+
         self.addActiveCoro(self._clearInactiveSessions)
 
         if self.isactive:
+
             # bootstrap a CA for our aha:network
-            netw = self.conf.get('aha:network')
-            if netw is not None:
+            netw = self._getAhaNetwork()
+
+            if self.certdir.getCaCertPath(netw) is None:
+                logger.info(f'Adding CA certificate for {netw}')
+                await self.genCaCert(netw)
+
+            name = self.conf.get('aha:name')
+
+            host = f'{name}.{netw}'
+            if self.certdir.getHostCertPath(host) is None:
+                logger.info(f'Adding server certificate for {host}')
+                await self._genHostCert(host, signas=netw)
+
+            user = self._getAhaAdmin()
+            if user is not None:
+                if self.certdir.getUserCertPath(user) is None:
+                    logger.info(f'Adding user certificate for {user}@{netw}')
+                    await self._genUserCert(user, signas=netw)
+
+    def _getDnsName(self):
+        # emulate the old aha name.network behavior if the
+        # explicit option is not set.
+
+        hostname = self.conf.get('dns:name')
+        if hostname is not None:
+            return hostname
+
+        name = self.conf.get('aha:name')
+        if name is not None:
+            return f'{name}.{self._getAhaNetwork()}'
 
-                if self.certdir.getCaCertPath(netw) is None:
-                    logger.info(f'Adding CA certificate for {netw}')
-                    await self.genCaCert(netw)
+    def _getProvListen(self):
 
-                name = self.conf.get('aha:name')
+        lisn = self.conf.get('provision:listen')
+        if lisn is not None:
+            return lisn
 
-                host = f'{name}.{netw}'
-                if self.certdir.getHostCertPath(host) is None:
-                    logger.info(f'Adding server certificate for {host}')
-                    await self._genHostCert(host, signas=netw)
+        # this may not use _getDnsName() in order to maintain
+        # backward compatibilty with aha name.network configs
+        # that do not intend to listen for provisioning.
+        hostname = self.conf.get('dns:name')
+        if hostname is not None:
+            return f'ssl://{hostname}:27272'
 
-                user = self._getAhaAdmin()
-                if user is not None:
-                    if self.certdir.getUserCertPath(user) is None:
-                        logger.info(f'Adding user certificate for {user}@{netw}')
-                        await self._genUserCert(user, signas=netw)
+    def _getDmonListen(self):
+
+        lisn = self.conf.get('dmon:listen')
+        if lisn is not None:
+            return lisn
+
+        network = self._getAhaNetwork()
+        dnsname = self._getDnsName()
+        if dnsname is not None:
+            return f'ssl://0.0.0.0?hostname={dnsname}&ca={network}'
+
+    def _reqProvListen(self):
+        lisn = self._getProvListen()
+        if lisn is not None:
+            return lisn
+
+        mesg = 'The AHA server is not configured for provisioning.'
+        raise s_exc.NeedConfValu(mesg=mesg)
 
     async def initServiceNetwork(self):
 
+        # bootstrap CA/host certs first
+        network = self._getAhaNetwork()
+        if network is not None:
+            await self._genCaCert(network)
+
+        hostname = self._getDnsName()
+        if hostname is not None and network is not None:
+            await self._genHostCert(hostname, signas=network)
+
         await s_cell.Cell.initServiceNetwork(self)
 
         self.provdmon = None
 
-        provurl = self.conf.get('provision:listen')
+        provurl = self._getProvListen()
         if provurl is not None:
             self.provdmon = await ProvDmon.anit(self)
             self.onfini(self.provdmon)
-            await self.provdmon.listen(provurl)
+            self.provaddr = await self.provdmon.listen(provurl)
 
     async def _clearInactiveSessions(self):
 
@@ -658,11 +716,7 @@ async def addAhaSvc(self, name, info, network=None):
     def _getAhaName(self, name):
         # the modern version of names is absolute or ...
         if name.endswith('...'):
-            netw = self.conf.get('aha:network')
-            if netw is None: # pragma: no cover
-                mesg = 'AHA Server requires aha:network configuration.'
-                raise s_exc.NeedConfValu(mesg=mesg)
-            name = name[:-2] + netw
+            return name[:-2] + self._getAhaNetwork()
         return name
 
     async def getAhaPool(self, name):
@@ -780,6 +834,7 @@ async def _reqAhaSvc(self, svcname):
     @s_nexus.Pusher.onPushAuto('aha:svc:del')
     async def delAhaSvc(self, name, network=None):
 
+        name = self._getAhaName(name)
         svcname, svcnetw, svcfull = self._nameAndNetwork(name, network)
 
         logger.info(f'Deleting service [{svcfull}].', extra=await self.getLogExtra(name=svcname, netw=svcnetw))
@@ -794,6 +849,8 @@ async def delAhaSvc(self, name, network=None):
         await self.fire('aha:svcdel', svcname=svcname, svcnetw=svcnetw)
 
     async def setAhaSvcDown(self, name, linkiden, network=None):
+
+        name = self._getAhaName(name)
         svcname, svcnetw, svcfull = self._nameAndNetwork(name, network)
         path = ('aha', 'services', svcnetw, svcname)
 
@@ -916,8 +973,18 @@ async def genCaCert(self, network):
 
         return cacert
 
+    async def _genCaCert(self, network):
+
+        if os.path.isfile(os.path.join(self.dirn, 'certs', 'cas', f'{network}.crt')):
+            return
+
+        await self.genCaCert(network)
+
     async def _genHostCert(self, hostname, signas=None):
 
+        if signas is not None:
+            await self._genCaCert(signas)
+
         if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', '{hostname}.crt')):
             return
 
@@ -1004,7 +1071,8 @@ async def signUserCsr(self, csrtext, signas=None):
 
         return self.certdir._certToByts(cert).decode()
 
-    def _getAhaUrls(self):
+    def _getAhaUrls(self, user='root'):
+
         urls = self.conf.get('aha:urls')
         if urls is not None:
             if isinstance(urls, str):
@@ -1015,29 +1083,29 @@ def _getAhaUrls(self):
         if ahaname is None:
             return None
 
-        ahanetw = self.conf.get('aha:network')
-        if ahanetw is None:
-            return None
+        ahanetw = self._getAhaNetwork()
 
         if self.sockaddr is None or not isinstance(self.sockaddr, tuple):
             return None
 
+        # FIXME this is the problematic use case
+        # FIXME may need to generate an AHA CA signed server cert for DNS NAME
+
         # TODO this could eventually enumerate others via itself
-        return (f'ssl://{ahaname}.{ahanetw}:{self.sockaddr[1]}',)
+        netloc = self._getDnsName()
+        return (f'ssl://{netloc}:{self.sockaddr[1]}?certname={user}@{ahanetw}',)
 
     async def addAhaSvcProv(self, name, provinfo=None):
 
+        if not name:
+            raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.')
+
         ahaurls = self._getAhaUrls()
         if ahaurls is None:
             mesg = 'AHA server has no configured aha:urls.'
             raise s_exc.NeedConfValu(mesg=mesg)
 
-        if self.conf.get('provision:listen') is None:
-            mesg = 'The AHA server does not have a provision:listen URL!'
-            raise s_exc.NeedConfValu(mesg=mesg)
-
-        if not name:
-            raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.')
+        self._reqProvListen()
 
         if provinfo is None:
             provinfo = {}
@@ -1048,23 +1116,15 @@ async def addAhaSvcProv(self, name, provinfo=None):
 
         conf = provinfo.setdefault('conf', {})
 
-        mynetw = self.conf.get('aha:network')
+        netw = self._getAhaNetwork()
 
         ahaadmin = self.conf.get('aha:admin')
         if ahaadmin is not None: # pragma: no cover
             conf.setdefault('aha:admin', ahaadmin)
 
         conf.setdefault('aha:user', 'root')
-        conf.setdefault('aha:network', mynetw)
 
-        netw = conf.get('aha:network')
-        if netw is None:
-            mesg = 'AHA server has no configured aha:network.'
-            raise s_exc.NeedConfValu(mesg=mesg)
-
-        if netw != mynetw:
-            mesg = f'Provisioning aha:network must be equal to the Aha servers network. Expected {mynetw}, got {netw}'
-            raise s_exc.BadConfValu(mesg=mesg, name='aha:network', expected=mynetw, got=netw)
+        conf['aha:network'] = netw
 
         hostname = f'{name}.{netw}'
 
@@ -1090,13 +1150,14 @@ async def addAhaSvcProv(self, name, provinfo=None):
 
         # allow user to win over leader
         ahauser = conf.get('aha:user')
-        ahaurls = s_telepath.modurl(ahaurls, user=ahauser)
+        certname = f'{ahauser}@{netw}'
+        ahaurls = s_telepath.modurl(ahaurls, certname=certname)
 
         conf.setdefault('aha:registry', ahaurls)
 
         mirname = provinfo.get('mirror')
         if mirname is not None:
-            conf['mirror'] = f'aha://{ahauser}@{mirname}.{netw}'
+            conf['mirror'] = f'aha://{ahauser}@{mirname}...'
 
         user = await self.auth.getUserByName(ahauser)
         if user is None:
@@ -1122,7 +1183,10 @@ async def addAhaSvcProv(self, name, provinfo=None):
 
     def _getProvClientUrl(self, iden):
 
-        provlisn = self.conf.get('provision:listen')
+        provlisn = self._getProvListen()
+
+        provport = self.provaddr[1]
+        provhost = self._getDnsName()
 
         urlinfo = s_telepath.chopurl(provlisn)
 
@@ -1133,8 +1197,8 @@ def _getProvClientUrl(self, iden):
             host = urlinfo.get('host')
 
         newinfo = {
-            'host': host,
-            'port': urlinfo.get('port'),
+            'host': provhost,
+            'port': provport,
             'scheme': scheme,
             'path': '/' + iden,
         }
@@ -1177,19 +1241,12 @@ async def delAhaSvcProv(self, iden):
 
     async def addAhaUserEnroll(self, name, userinfo=None, again=False):
 
-        provurl = self.conf.get('provision:listen')
-        if provurl is None:
-            mesg = 'The AHA server does not have a provision:listen URL!'
-            raise s_exc.NeedConfValu(mesg=mesg)
-
-        ahanetw = self.conf.get('aha:network')
-        if ahanetw is None:
-            mesg = 'AHA server requires aha:network configuration.'
-            raise s_exc.NeedConfValu(mesg=mesg)
-
         if not name:
             raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.')
 
+        provurl = self._reqProvListen()
+        ahanetw = self._getAhaNetwork()
+
         username = f'{name}@{ahanetw}'
 
         if len(username) > 64:
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 35fb5c33f65..db3a66669f5 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -961,7 +961,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'type': 'string',
         },
         'aha:network': {
-            'description': 'The AHA service network. This makes aha:name/aha:leader relative names.',
+            'description': 'The AHA service network. Defaults to "synapse".',
             'type': 'string',
         },
         'aha:registry': {
@@ -1126,8 +1126,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         # we need to know this pretty early...
         self.ahasvcname = None
         ahaname = self.conf.get('aha:name')
-        ahanetw = self.conf.get('aha:network')
-        if ahaname is not None and ahanetw is not None:
+        ahanetw = self._getAhaNetwork()
+        if ahaname is not None:
             self.ahasvcname = f'{ahaname}.{ahanetw}'
 
         # each cell has a guid
@@ -1507,7 +1507,7 @@ async def finiaha():
             self.onfini(finiaha)
 
         ahauser = self.conf.get('aha:user')
-        ahanetw = self.conf.get('aha:network')
+        ahanetw = self._getAhaNetwork()
 
         ahaadmin = self._getAhaAdmin()
         if ahaadmin is not None:
@@ -1516,6 +1516,20 @@ async def finiaha():
         if ahauser is not None:
             await self._addAdminUser(ahauser)
 
+    def _getAhaNetwork(self):
+        return self.conf.get('aha:network', 'synapse')
+
+    def _getDmonListen(self):
+
+        lisn = self.conf.get('dmon:listen')
+        if lisn is not None:
+            return lisn
+
+        network = self._getAhaNetwork()
+        ahaname = self.conf.get('aha:name')
+        if ahaname is not None:
+            return f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}'
+
     async def _addAdminUser(self, username):
         # add the user in a pre-nexus compatible way
         user = await self.auth.getUserByName(username)
@@ -1562,7 +1576,7 @@ async def initServiceNetwork(self):
 
         self.sockaddr = None
 
-        turl = self.conf.get('dmon:listen')
+        turl = self._getDmonListen()
         if turl is not None:
             self.sockaddr = await self.dmon.listen(turl)
             logger.info(f'dmon listening: {turl}')
@@ -1628,7 +1642,7 @@ async def _initAhaService(self):
             return
 
         ahalead = self.conf.get('aha:leader')
-        ahanetw = self.conf.get('aha:network')
+        ahanetw = self._getAhaNetwork()
 
         ahainfo = await self.getAhaInfo()
         if ahainfo is None:
@@ -1747,10 +1761,7 @@ async def promote(self, graceful=False):
                 mesg = 'Cannot gracefully promote without aha:name configured.'
                 raise s_exc.BadArg(mesg=mesg)
 
-            ahanetw = self.conf.get('aha:network')
-            if ahanetw is None: # pragma: no cover
-                mesg = 'Cannot gracefully promote without aha:network configured.'
-                raise s_exc.BadArg(mesg=mesg)
+            ahanetw = self._getAhaNetwork()
 
             myurl = f'aha://{ahaname}.{ahanetw}'
             logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}')
@@ -1848,7 +1859,7 @@ async def _setAhaActive(self):
             await proxy.fini()
             return
 
-        ahanetw = self.conf.get('aha:network')
+        ahanetw = self._getAhaNetwork()
         try:
             await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw)
         except asyncio.CancelledError:  # pragma: no cover
@@ -3876,7 +3887,7 @@ async def _getCellUser(self, link, mesg):
 
             if username.find('@') != -1:
                 userpart, hostpart = username.split('@', 1)
-                if hostpart == self.conf.get('aha:network'):
+                if hostpart == self._getAhaNetwork():
                     user = await self.auth.getUserByName(userpart)
                     if user is not None:
                         if user.isLocked():
@@ -4069,7 +4080,7 @@ async def getCellInfo(self):
                 'aha': {
                     'name': self.conf.get('aha:name'),
                     'leader': self.conf.get('aha:leader'),
-                    'network': self.conf.get('aha:network'),
+                    'network': self._getAhaNetwork(),
                 }
             },
             'features': {
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index eace62dfade..9d2f4a9f06e 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -104,15 +104,14 @@ async def test_lib_aha_offon(self):
 
                 wait00 = aha.waiter(1, 'aha:svcadd')
                 cryo_conf = {
-                    'aha:name': '0.cryo.mynet',
-                    'aha:admin': 'root@cryo.mynet',
+                    'aha:name': '0.cryo',
                     'aha:registry': f'tcp://root:secret@127.0.0.1:{port}',
                     'dmon:listen': 'tcp://0.0.0.0:0/',
                 }
                 async with self.getTestCryo(dirn=cryo0_dirn, conf=cryo_conf) as cryo:
                     self.isin(len(await wait00.wait(timeout=6)), (1, 2))
 
-                    svc = await aha.getAhaSvc('0.cryo.mynet')
+                    svc = await aha.getAhaSvc('0.cryo...')
                     linkiden = svc.get('svcinfo', {}).get('online')
                     self.nn(linkiden)
 
@@ -122,12 +121,12 @@ async def test_lib_aha_offon(self):
             async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha:
                 wait01 = aha.waiter(1, 'aha:svcdown')
                 await wait01.wait(timeout=6)
-                svc = await aha.getAhaSvc('0.cryo.mynet')
+                svc = await aha.getAhaSvc('0.cryo...')
                 self.notin('online', svc.get('svcinfo'))
 
                 # Try setting something down a second time
-                await aha.setAhaSvcDown('0.cryo.mynet', linkiden, network=None)
-                svc = await aha.getAhaSvc('0.cryo.mynet')
+                await aha.setAhaSvcDown('0.cryo', linkiden, network='synapse')
+                svc = await aha.getAhaSvc('0.cryo...')
                 self.notin('online', svc.get('svcinfo'))
 
     async def test_lib_aha(self):
@@ -163,9 +162,9 @@ async def test_lib_aha(self):
 
             wait00 = aha.waiter(1, 'aha:svcadd')
             conf = {
-                'aha:name': '0.cryo.mynet',
-                'aha:leader': 'cryo.mynet',
-                'aha:admin': 'root@cryo.mynet',
+                'aha:name': '0.cryo',
+                'aha:leader': 'cryo',
+                'aha:admin': 'root@synapse',
                 'aha:registry': [f'tcp://root:hehehaha@127.0.0.1:{port}',
                                  f'tcp://root:hehehaha@127.0.0.1:{port}'],
                 'dmon:listen': 'tcp://0.0.0.0:0/',
@@ -174,7 +173,7 @@ async def test_lib_aha(self):
 
                 await cryo.auth.rootuser.setPasswd('secret')
 
-                ahaadmin = await cryo.auth.getUserByName('root@cryo.mynet')
+                ahaadmin = await cryo.auth.getUserByName('root@synapse')
                 self.nn(ahaadmin)
                 self.true(ahaadmin.isAdmin())
 
@@ -183,14 +182,14 @@ async def test_lib_aha(self):
                 with self.raises(s_exc.NoSuchName):
                     await s_telepath.getAhaProxy({'host': 'hehe.haha'})
 
-                async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy:
+                async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 with self.raises(s_exc.BadArg):
                     _proxy = await cryo.ahaclient.proxy(timeout=2)
-                    await _proxy.modAhaSvcInfo('cryo.mynet', {'newp': 'newp'})
+                    await _proxy.modAhaSvcInfo('cryo...', {'newp': 'newp'})
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy:
+                async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 # force a reconnect...
@@ -199,7 +198,7 @@ async def test_lib_aha(self):
                 await proxy.fini()
                 self.nn(await waiter.wait(timeout=6))
 
-                async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy:
+                async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 waiter = aha.waiter(1, 'aha:svcadd')
@@ -207,17 +206,17 @@ async def test_lib_aha(self):
                 await cryo.setCellActive(False)
 
                 with self.raises(s_exc.NoSuchName):
-                    async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy:
+                    async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
                         pass
 
                 self.nn(await waiter.wait(timeout=6))
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy:
+                async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 await cryo.setCellActive(True)
 
-                async with await s_telepath.openurl('aha://root:secret@cryo.mynet') as proxy:
+                async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 # some coverage edge cases...
@@ -234,7 +233,7 @@ async def test_lib_aha(self):
                 self.false(ahaadmin.isAdmin())
 
             async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
-                ahaadmin = await cryo.auth.getUserByName('root@cryo.mynet')
+                ahaadmin = await cryo.auth.getUserByName('root@synapse')
                 # And we should be unlocked and admin now
                 self.false(ahaadmin.isLocked())
                 self.true(ahaadmin.isAdmin())
@@ -322,15 +321,16 @@ async def test_lib_aha(self):
                     self.eq(info.get('status'), 'ok')
                     result = info.get('result')
                     self.len(2, result)
-                    self.eq({'0.cryo.mynet', 'cryo.mynet'},
+                    self.eq({'0.cryo.synapse', 'cryo.synapse'},
                             {svcinfo.get('name') for svcinfo in result})
 
-                async with sess.get(svcsurl, json={'network': 'mynet'}) as resp:
+                async with sess.get(svcsurl, json={'network': 'synapse'}) as resp:
                     info = await resp.json()
                     self.eq(info.get('status'), 'ok')
                     result = info.get('result')
-                    self.len(1, result)
-                    self.eq('cryo.mynet', result[0].get('name'))
+                    self.len(2, result)
+                    self.eq({'0.cryo.synapse', 'cryo.synapse'},
+                            {svcinfo.get('name') for svcinfo in result})
 
                 async with sess.get(svcsurl, json={'network': 'newp'}) as resp:
                     info = await resp.json()
@@ -385,7 +385,7 @@ async def test_lib_aha(self):
                 uinfo = ahainfo.get('urlinfo', {})
                 self.eq(uinfo.get('scheme'), 'tcp')
                 self.gt(uinfo.get('port'), 0)
-                self.eq(aha._getAhaUrls()[0], f'ssl://0.test.foo:{aha.sockaddr[1]}')
+                self.eq(aha._getAhaUrls()[0], f'ssl://0.test.foo:{aha.sockaddr[1]}?certname=root@foo')
 
     async def test_lib_aha_loadenv(self):
 
@@ -441,8 +441,8 @@ async def test_lib_aha_finid_cell(self):
 
             wait00 = aha.waiter(1, 'aha:svcadd')
             conf = {
-                'aha:name': '0.cryo.mynet',
-                'aha:admin': 'root@cryo.mynet',
+                'aha:name': '0.cryo',
+                'aha:admin': 'root@synapse',
                 'aha:registry': aharegistry,
                 'dmon:listen': 'tcp://0.0.0.0:0/',
             }
@@ -450,7 +450,7 @@ async def test_lib_aha_finid_cell(self):
 
                 await cryo.auth.rootuser.setPasswd('secret')
 
-                ahaadmin = await cryo.auth.getUserByName('root@cryo.mynet')
+                ahaadmin = await cryo.auth.getUserByName('root@synapse')
                 self.nn(ahaadmin)
                 self.true(ahaadmin.isAdmin())
 
@@ -458,7 +458,7 @@ async def test_lib_aha_finid_cell(self):
 
                 self.isin(atup, s_telepath.aha_clients)
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy:
+                async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 _ahaclient = s_telepath.aha_clients.get(atup).get('client')
@@ -475,7 +475,7 @@ async def quickproxy(self, timeout):
                 with mock.patch('synapse.telepath.Client.proxy', quickproxy):
                     with self.raises(asyncio.TimeoutError):
 
-                        async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy:
+                        async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
                             self.fail('Should never reach a connection.')
 
     async def test_lib_aha_onlink_fail(self):
@@ -493,8 +493,7 @@ async def test_lib_aha_onlink_fail(self):
 
                 wait00 = aha.waiter(1, 'aha:svcadd')
                 conf = {
-                    'aha:name': '0.cryo.mynet',
-                    'aha:admin': 'root@cryo.mynet',
+                    'aha:name': '0.cryo',
                     'aha:registry': f'tcp://root:secret@127.0.0.1:{port}',
                     'dmon:listen': 'tcp://0.0.0.0:0/',
                 }
@@ -504,7 +503,7 @@ async def test_lib_aha_onlink_fail(self):
 
                     self.none(await wait00.wait(timeout=2))
 
-                    svc = await aha.getAhaSvc('0.cryo.mynet')
+                    svc = await aha.getAhaSvc('0.cryo...')
                     self.none(svc)
 
                     wait01 = aha.waiter(1, 'aha:svcadd')
@@ -512,11 +511,11 @@ async def test_lib_aha_onlink_fail(self):
 
                     self.nn(await wait01.wait(timeout=2))
 
-                    svc = await aha.getAhaSvc('0.cryo.mynet')
+                    svc = await aha.getAhaSvc('0.cryo...')
                     self.nn(svc)
                     self.nn(svc.get('svcinfo', {}).get('online'))
 
-                    async with await s_telepath.openurl('aha://root:secret@0.cryo.mynet') as proxy:
+                    async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
                         self.nn(await proxy.getCellIden())
 
     async def test_lib_aha_bootstrap(self):
@@ -559,34 +558,20 @@ async def test_lib_aha_noconf(self):
             with self.raises(s_exc.NeedConfValu):
                 await aha.addAhaUserEnroll('hehe')
 
-            aha.conf['provision:listen'] = 'tcp://127.0.0.1:27272'
-
-            with self.raises(s_exc.NeedConfValu):
-                await aha.addAhaSvcProv('hehe')
-
             with self.raises(s_exc.NeedConfValu):
                 await aha.addAhaUserEnroll('hehe')
 
-            aha.conf['aha:network'] = 'haha'
-            await aha.addAhaSvcProv('hehe')
-
     async def test_lib_aha_provision(self):
 
         with self.getTestDir() as dirn:
 
             conf = {
                 'aha:name': 'aha',
-                'aha:network': 'loop.vertex.link',
-                'provision:listen': 'ssl://aha.loop.vertex.link:0'
+                'dns:name': 'aha.loop.vertex.link',
             }
             async with self.getTestAha(dirn=dirn, conf=conf) as aha:
 
-                addr, port = aha.provdmon.addr
-                # update the config to reflect the dynamically bound port
-                aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}'
-
-                # do this config ex-post-facto due to port binding...
-                host, ahaport = await aha.dmon.listen('ssl://0.0.0.0:0?hostname=aha.loop.vertex.link&ca=loop.vertex.link')
+                ahaport = aha.sockaddr[1]
                 aha.conf['aha:urls'] = f'ssl://aha.loop.vertex.link:{ahaport}'
 
                 url = aha.getLocalUrl()
@@ -655,19 +640,19 @@ async def test_lib_aha_provision(self):
                     self.none(await axon.auth.getUserByName('axon@loop.vertex.link'))
 
                     self.true(os.path.isfile(s_common.genpath(axon.dirn, 'prov.done')))
-                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'cas', 'loop.vertex.link.crt')))
-                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.loop.vertex.link.crt')))
-                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.loop.vertex.link.key')))
-                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@loop.vertex.link.crt')))
-                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@loop.vertex.link.key')))
+                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'cas', 'synapse.crt')))
+                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.synapse.crt')))
+                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'hosts', '00.axon.synapse.key')))
+                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@synapse.crt')))
+                    self.true(os.path.isfile(s_common.genpath(axon.dirn, 'certs', 'users', 'root@synapse.key')))
 
                     yamlconf = s_common.yamlload(axon.dirn, 'cell.yaml')
                     self.eq('axon', yamlconf.get('aha:leader'))
                     self.eq('00.axon', yamlconf.get('aha:name'))
-                    self.eq('loop.vertex.link', yamlconf.get('aha:network'))
+                    self.eq('synapse', yamlconf.get('aha:network'))
                     self.none(yamlconf.get('aha:admin'))
-                    self.eq((f'ssl://root@aha.loop.vertex.link:{ahaport}',), yamlconf.get('aha:registry'))
-                    self.eq(f'ssl://0.0.0.0:0?hostname=00.axon.loop.vertex.link&ca=loop.vertex.link', yamlconf.get('dmon:listen'))
+                    self.eq((f'ssl://aha.loop.vertex.link:{ahaport}?certname=root@synapse',), yamlconf.get('aha:registry'))
+                    self.eq(f'ssl://0.0.0.0:0?hostname=00.axon.synapse&ca=synapse', yamlconf.get('dmon:listen'))
 
                     unfo = await axon.addUser('visi')
 
@@ -678,9 +663,9 @@ async def test_lib_aha_provision(self):
                     provurl = str(outp).split(':', 1)[1].strip()
                     with self.getTestSynDir() as syndir:
 
-                        capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt')
-                        crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt')
-                        keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key')
+                        capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt')
+                        crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
+                        keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.key')
 
                         for path in (capath, crtpath, keypath):
                             s_common.genfile(path)
@@ -693,7 +678,7 @@ async def test_lib_aha_provision(self):
 
                         teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
                         self.eq(teleyaml.get('version'), 1)
-                        self.eq(teleyaml.get('aha:servers'), (f'ssl://visi@aha.loop.vertex.link:{ahaport}',))
+                        self.eq(teleyaml.get('aha:servers'), (f'ssl://aha.loop.vertex.link:{ahaport}?certname=visi@synapse',))
 
                         certdir = s_telepath.s_certdir.CertDir(os.path.join(syndir, 'certs'))
                         async with await s_telepath.openurl('aha://visi@axon...', certdir=certdir) as prox:
@@ -776,7 +761,7 @@ async def test_lib_aha_provision(self):
                             await axon2.sync()
                             self.true(axon.isactive)
                             self.false(axon2.isactive)
-                            self.eq('aha://root@axon.loop.vertex.link', axon2.conf.get('mirror'))
+                            self.eq('aha://root@axon...', axon2.conf.get('mirror'))
 
                             with s_common.genfile(axn2path, 'prov.done') as fd:
                                 axon2providen = fd.read().decode().strip()
@@ -787,7 +772,7 @@ async def test_lib_aha_provision(self):
                             await axon2.sync()
                             self.true(axon.isactive)
                             self.false(axon2.isactive)
-                            self.eq('aha://root@axon.loop.vertex.link', axon2.conf.get('mirror'))
+                            self.eq('aha://root@axon...', axon2.conf.get('mirror'))
 
                 # Provision a mirror using aha:provision in the mirror cell.yaml as well.
                 # This is similar to the previous test block.
@@ -815,7 +800,7 @@ async def test_lib_aha_provision(self):
                         await axon03.sync()
                         self.true(axon.isactive)
                         self.false(axon03.isactive)
-                        self.eq('aha://root@axon.loop.vertex.link', axon03.conf.get('mirror'))
+                        self.eq('aha://root@axon...', axon03.conf.get('mirror'))
 
                         with s_common.genfile(axn3path, 'prov.done') as fd:
                             axon3providen = fd.read().decode().strip()
@@ -831,15 +816,15 @@ async def test_lib_aha_provision(self):
                         await axon3.sync()
                         self.true(axon.isactive)
                         self.false(axon3.isactive)
-                        self.eq('aha://root@axon.loop.vertex.link', axon03.conf.get('mirror'))
+                        self.eq('aha://root@axon...', axon03.conf.get('mirror'))
 
                         retn, outp = await self.execToolMain(s_a_list._main, [aha.getLocalUrl()])
                         self.eq(retn, 0)
                         outp.expect('Service              network                        leader')
-                        outp.expect('00.axon              loop.vertex.link               True')
-                        outp.expect('01.axon              loop.vertex.link               False')
-                        outp.expect('02.axon              loop.vertex.link               False')
-                        outp.expect('axon                 loop.vertex.link               True')
+                        outp.expect('00.axon              synapse                        True')
+                        outp.expect('01.axon              synapse                        False')
+                        outp.expect('02.axon              synapse                        False')
+                        outp.expect('axon                 synapse                        True')
 
                 # Ensure we can provision a service on a given listening ports
                 outp.clear()
@@ -854,14 +839,6 @@ async def test_lib_aha_provision(self):
                 outp.expect('ERROR: Invalid HTTPS port: 123456')
                 self.eq(1, ret)
 
-                outp.clear()
-                bad_conf_path = s_common.genpath(dirn, 'badconf.yaml')
-                s_common.yamlsave({'aha:network': 'aha.newp.net'}, bad_conf_path)
-                args = ('--url', aha.getLocalUrl(), 'bazfaz', '--cellyaml', bad_conf_path)
-                ret = await s_tools_provision_service.main(args, outp=outp)
-                outp.expect('ERROR: Provisioning aha:network must be equal to the Aha servers network')
-                self.eq(1, ret)
-
                 outp = self.getTestOutp()
                 argv = ('--url', aha.getLocalUrl(), 'bazfaz', '--dmon-port', '1234', '--https-port', '443')
                 await s_tools_provision_service.main(argv, outp=outp)
@@ -876,16 +853,6 @@ async def test_lib_aha_provision(self):
                     https_port = conf.get('https:port')
                     self.eq(https_port, 443)
 
-                # provisioning against a network that differs from the aha network fails.
-                bad_netw = 'stuff.goes.beep'
-                provinfo = {'conf': {'aha:network': bad_netw}}
-                with self.raises(s_exc.BadConfValu) as cm:
-                    async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller,
-                                                provinfo=provinfo) as conn:
-                        pass
-                self.isin('Provisioning aha:network must be equal to the Aha servers network',
-                          cm.exception.get('mesg'))
-
                 # We can generate urls and then drop them en-mass. They will not usable.
                 provurls = []
                 enrlursl = []
@@ -997,13 +964,6 @@ async def test_aha_httpapi(self):
                     self.eq(info.get('status'), 'err')
                     self.eq(info.get('code'), 'SchemaViolation')
 
-                # Break the Aha cell - not will provision after this.
-                _network = aha.conf.pop('aha:network')
-                async with sess.post(url, json={'name': '00.newp'}) as resp:
-                    info = await resp.json()
-                    self.eq(info.get('status'), 'err')
-                    self.eq(info.get('code'), 'NeedConfValu')
-
             # Not an admin
             await aha.addUser('lowuser', passwd='lowuser')
             async with self.getHttpSess(auth=('lowuser', 'lowuser'), port=httpsport) as sess:
diff --git a/synapse/tools/aha/enroll.py b/synapse/tools/aha/enroll.py
index 6b5b49095e8..061ee052cd3 100644
--- a/synapse/tools/aha/enroll.py
+++ b/synapse/tools/aha/enroll.py
@@ -76,7 +76,8 @@ async def main(argv, outp=s_output.stdout):
                 if isinstance(ahaurls, str):
                     ahaurls = (ahaurls,)
 
-                ahaurls = set(s_telepath.modurl(ahaurls, user=ahauser))
+                certname = f'{ahauser}@{ahanetw}'
+                ahaurls = set(s_telepath.modurl(ahaurls, certname=certname))
                 servers = teleyaml.get('aha:servers')
 
                 # repack the servers so lists are tuplized like values

From cba0bfed7e84d9660e495e2ac1851947eda55e1e Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Thu, 27 Jun 2024 17:50:51 -0400
Subject: [PATCH 02/68] wip

---
 docs/synapse/deploymentguide.rst | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 0326c835642..2b84671c5d1 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -108,9 +108,7 @@ Create the ``/srv/syn/aha/docker-compose.yaml`` file with contents::
         environment:
             - SYN_AHA_HTTPS_PORT=null
             - SYN_AHA_AHA_NAME=aha
-            - SYN_AHA_AHA_NETWORK=<yournetwork>
-            - SYN_AHA_DMON_LISTEN=ssl://aha.<yournetwork>?ca=<yournetwork>
-            - SYN_AHA_PROVISION_LISTEN=ssl://aha.<yournetwork>:27272
+            - SYN_AHA_DNS_NAME=aha.<yuournetwork>
 
 .. note::
 

From 8697e4ad67311c68bd6f9e8554d7ffebe5335d0c Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 28 Jun 2024 12:57:47 -0400
Subject: [PATCH 03/68] wip

---
 synapse/lib/aha.py                     | 290 +++++++++++++++++++++----
 synapse/lib/cell.py                    |  35 +--
 synapse/tests/test_axon.py             |   2 +-
 synapse/tests/test_cortex.py           |   9 +-
 synapse/tests/test_lib_aha.py          | 256 ++++++----------------
 synapse/tests/test_lib_cell.py         |   8 +-
 synapse/tests/test_lib_stormlib_aha.py |  70 +++---
 synapse/tests/utils.py                 | 150 +++++--------
 8 files changed, 444 insertions(+), 376 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index f3738eaf52e..b93d4c17833 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -12,6 +12,7 @@
 import synapse.daemon as s_daemon
 import synapse.telepath as s_telepath
 
+import synapse.lib.base as s_base
 import synapse.lib.cell as s_cell
 import synapse.lib.coro as s_coro
 import synapse.lib.nexus as s_nexus
@@ -125,11 +126,19 @@ async def get(self):
 
 class AhaApi(s_cell.CellApi):
 
-    async def getAhaUrls(self):
-        ahaurls = self.cell._getAhaUrls()
-        if ahaurls is not None:
-            return ahaurls
-        return()
+    @s_cell.adminapi()
+    async def getAhaClone(self, iden):
+        return await self.cell.getAhaClone(iden)
+
+    @s_cell.adminapi()
+    async def addAhaClone(self, host, port=27492, conf=None):
+        return await self.cell.addAhaClone(host, port=port, conf=conf)
+
+    async def getAhaUrls(self, user='root'):
+        ahaurls = await self.cell.getAhaUrls(user=user)
+        if ahaurls is None:
+            return ()
+        return ahaurls
 
     async def getAhaSvc(self, name, filters=None):
         '''
@@ -280,6 +289,10 @@ async def addAhaPoolSvc(self, poolname, svcname, info):
     async def delAhaPoolSvc(self, poolname, svcname):
         return await self.cell.delAhaPoolSvc(poolname, svcname)
 
+    async def iterAhaTopo(self):
+        async for item in self.cell.iterAhaTopo():
+            yield item
+
     async def iterPoolTopo(self, name):
 
         username = self.user.name.split('@')[0]
@@ -299,6 +312,20 @@ async def getAhaPools(self):
         async for item in self.cell.getAhaPools():
             yield item
 
+    async def getAhaServers(self):
+        return await self.cell.getAhaServers()
+
+    async def getAhaServer(self, host, port):
+        return await self.cell.getAhaServer(host, port)
+
+    @s_cell.adminapi()
+    async def addAhaServer(self, server):
+        return await self.cell.addAhaServer(server)
+
+    @s_cell.adminapi()
+    async def delAhaServer(self, host, port):
+        return await self.cell.delAhaServer(host, port)
+
     @s_cell.adminapi()
     async def addAhaSvcProv(self, name, provinfo=None):
         '''
@@ -366,10 +393,34 @@ async def _getSharedItem(self, name):
             await self.aha.delAhaUserEnroll(name)
             return EnrollApi(self.aha, userinfo)
 
+        clone = await self.aha.getAhaClone(name)
+        if clone is not None:
+            mesg = f'Retrieved AHA clone info for {name}'
+            host = clone.get('host')
+            logger.info(mesg, extra=await self.aha.getLogExtra(iden=name, host=host))
+            return CloneApi(self.aha, clone)
+
         mesg = f'Invalid provisioning identifier name={name}. This could be' \
                f' caused by the re-use of a provisioning URL.'
         raise s_exc.NoSuchName(mesg=mesg, name=name)
 
+class CloneApi:
+
+    def __init__(self, aha, clone):
+        self.aha = aha
+        self.clone = clone
+
+    async def getCloneDef(self):
+        return self.clone
+
+    async def readyToMirror(self):
+        return await self.aha.readyToMirror()
+
+    async def iterNewBackupArchive(self, name=None, remove=False):
+        async with self.aha.getLocalProxy() as proxy:
+            async for byts in proxy.iterNewBackupArchive(name=name, remove=remove):
+                yield byts
+
 class EnrollApi:
 
     def __init__(self, aha, userinfo):
@@ -377,9 +428,10 @@ def __init__(self, aha, userinfo):
         self.userinfo = userinfo
 
     async def getUserInfo(self):
+        user = self.userinfo.get('name')
         return {
-            'aha:urls': self.aha._getAhaUrls(),
-            'aha:user': self.userinfo.get('name'),
+            'aha:urls': await self.aha.getAhaUrls(user=user),
+            'aha:user': user,
             'aha:network': self.aha._getAhaNetwork(),
         }
 
@@ -464,12 +516,17 @@ class AhaCell(s_cell.Cell):
     confbase['mirror']['hidedocs'] = False  # type: ignore
     confbase['mirror']['hidecmdl'] = False  # type: ignore
     confdefs = {
+        'clone': {
+            'hidecmdl': True,
+            'description': 'Bootstrap a clone from the AHA clone URL.',
+            'type': ['string', 'null'],
+        },
         'dns:name': {
             'description': 'The registered DNS name used to reach the AHA service.',
             'type': ['string', 'null'],
         },
         'aha:urls': {
-            'description': 'A list of all available AHA server URLs.',
+            'description': 'Deprecated. AHA servers can now manage this automatically.',
             'type': ['string', 'array'],
             'items': {'type': 'string'},
         },
@@ -484,6 +541,34 @@ class AhaCell(s_cell.Cell):
     def getEnvPrefix(cls):
         return (f'SYN_AHA', f'SYN_{cls.__name__.upper()}', )
 
+    async def _initCellBoot(self):
+
+        path = s_common.genpath(self.dirn, 'cell.guid')
+        if os.path.isfile(path):
+            return
+
+        curl = self.conf.get('clone')
+        if curl is None:
+            return
+
+        logger.warning(f'Cloning AHA: {curl}')
+
+        async with await s_telepath.openurl(curl) as proxy:
+            clone = await proxy.getCloneDef()
+            await self._initCloneCell(proxy)
+
+        logger.warning('Cloning AHA: done!')
+
+        conf = s_common.yamlload(self.dirn, 'cell.yaml')
+        if conf is None:
+            conf = {}
+
+        conf.update(clone.get('conf', {}))
+
+        s_common.yamlsave(conf, self.dirn, 'cell.yaml')
+
+        self.conf.update(conf)
+
     async def initServiceStorage(self):
 
         # TODO plumb using a remote jsonstor?
@@ -492,6 +577,7 @@ async def initServiceStorage(self):
         slab = await s_lmdbslab.Slab.anit(dirn)
         slab.addResizeCallback(self.checkFreeSpace)
 
+        self.topobus = await s_base.Base.anit()
         self.jsonstor = await s_jsonstor.JsonStor.anit(slab, 'aha')  # type: s_jsonstor.JsonStor
 
         async def fini():
@@ -503,9 +589,103 @@ async def fini():
         self.slab.initdb('aha:provs')
         self.slab.initdb('aha:enrolls')
 
+        self.slab.initdb('aha:clones')
+        self.slab.initdb('aha:servers')
+
         self.slab.initdb('aha:pools')
+
+        self.topowindows = []
         self.poolwindows = collections.defaultdict(list)
 
+    async def getAhaServer(self, host, port):
+        lkey = s_msgpack.en((host, port))
+        byts = self.slab.get(lkey, db='aha:servers')
+        if byts is not None:
+            return s_msgpack.un(byts)
+
+    async def addAhaServer(self, server):
+
+        host = server.get('host')
+        port = server.setdefault('port', 27492)
+
+        # avoid a noop nexus change...
+        oldv = await self.getAhaServer(host, port)
+        if s_common.flatten(server) == s_common.flatten(oldv):
+            return False
+
+        return await self._push('aha:server:add', server)
+
+    async def _getTopoUpdate(self):
+        return ('aha:update', {
+            'servers': await self.getAhaServers(),
+        }),
+
+    async def _fireTopoUpdate(self):
+        update = await self._getTopoUpdate()
+        for wind in self.topowindows:
+            await wind.put(update)
+
+    @s_nexus.Pusher.onPush('aha:server:add')
+    async def _addAhaServer(self, server):
+        # TODO schema
+        host = server.get('host')
+        port = server.get('port')
+
+        oldv = None
+        lkey = s_msgpack.en((host, port))
+
+        byts = self.slab.get(lkey, db='aha:servers')
+        if byts is not None:
+            oldv = s_msgpack.un(byts)
+            if s_common.flatten(server) == s_common.flatten(oldv):
+                return False
+
+        self.slab.put(lkey, s_msgpack.en(server), db='aha:servers')
+
+        if self.isactive:
+            await self._fireTopoUpdate()
+
+        return True
+
+    @s_nexus.Pusher.onPushAuto('aha:server:del')
+    async def delAhaServer(self, host, port):
+
+        lkey = s_msgpack.en((host, port))
+
+        byts = self.slab.pop(lkey, db='aha:servers')
+        if byts is None:
+            return None
+
+        if self.isactive:
+            await self._fireTopoUpdate()
+
+        return s_msgpack.un(byts)
+
+    async def getAhaServers(self):
+        servers = []
+        for _, byts in self.slab.scanByFull(db='aha:servers'):
+            servers.append(s_msgpack.un(byts))
+        return servers
+
+    async def iterAhaTopo(self):
+
+        if not self.isactive:
+            async with await self.nexsroot.client.proxy() as proxy:
+                async for item in proxy.iterAhaTopo():
+                    yield item
+
+        async with await s_queue.Window.anit(maxsize=1000) as wind:
+
+            async def onfini():
+                self.topowindows.remove(wind)
+
+            wind.onfini(onfini)
+
+            async with self.nexsroot.applylock:
+                update = await self._getTopoUpdate()
+                await wind.put(update)
+                self.topowindows.append(wind)
+
     async def iterPoolTopo(self, name):
 
         name = self._getAhaName(name)
@@ -560,11 +740,12 @@ async def initServiceRuntime(self):
                 logger.info(f'Adding server certificate for {host}')
                 await self._genHostCert(host, signas=netw)
 
+            root = f'root@{netw}'
+            await self._genUserCert(root, signas=netw)
+
             user = self._getAhaAdmin()
             if user is not None:
-                if self.certdir.getUserCertPath(user) is None:
-                    logger.info(f'Adding user certificate for {user}@{netw}')
-                    await self._genUserCert(user, signas=netw)
+                await self._genUserCert(user, signas=netw)
 
     def _getDnsName(self):
         # emulate the old aha name.network behavior if the
@@ -593,8 +774,8 @@ def _getProvListen(self):
 
     def _getDmonListen(self):
 
-        lisn = self.conf.get('dmon:listen')
-        if lisn is not None:
+        lisn = self.conf.get('dmon:listen', s_common.novalu)
+        if lisn is not s_common.novalu:
             return lisn
 
         network = self._getAhaNetwork()
@@ -623,6 +804,11 @@ async def initServiceNetwork(self):
 
         await s_cell.Cell.initServiceNetwork(self)
 
+        # all AHA mirrors are registered
+        if hostname is not None and self.sockaddr is not None:
+            server = {'host': hostname, 'port': self.sockaddr[1]}
+            await self.addAhaServer(server)
+
         self.provdmon = None
 
         provurl = self._getProvListen()
@@ -712,6 +898,7 @@ async def addAhaSvc(self, name, info, network=None):
 
         # mostly for testing...
         await self.fire('aha:svcadd', svcinfo=svcinfo)
+        await self.fire(f'aha:svcadd:{svcfull}', svcinfo=svcinfo)
 
     def _getAhaName(self, name):
         # the modern version of names is absolute or ...
@@ -994,8 +1181,12 @@ async def _genHostCert(self, hostname, signas=None):
         await self.saveHostCert(hostname, pkey, cert)
 
     async def _genUserCert(self, username, signas=None):
+
         if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', '{username}.crt')):
             return
+
+        logger.info(f'Adding user certificate for {username}')
+
         pkey, cert = await s_coro.executor(self.certdir.genUserCert, username, signas=signas, save=False)
         pkey = self.certdir._pkeyToByts(pkey).decode()
         cert = self.certdir._certToByts(cert).decode()
@@ -1071,40 +1262,67 @@ async def signUserCsr(self, csrtext, signas=None):
 
         return self.certdir._certToByts(cert).decode()
 
-    def _getAhaUrls(self, user='root'):
+    async def getAhaUrls(self, user='root'):
 
+        # for backward compat...
         urls = self.conf.get('aha:urls')
         if urls is not None:
-            if isinstance(urls, str):
-                return (urls,)
             return urls
 
-        ahaname = self.conf.get('aha:name')
-        if ahaname is None:
-            return None
+        network = self._getAhaNetwork()
+        urls = []
+        for server in await self.getAhaServers():
+            host = server.get('host')
+            port = server.get('port')
+            urls.append(f'ssl://{host}:{port}?certname={user}@{network}')
+        return urls
+
+    def _getAhaUrl(self, user='root'):
+        port = self.sockaddr[1]
+        host = self._getDnsName()
+        network = self._getAhaNetwork()
+        return f'ssl://{host}:{port}?certname={user}@{network}'
 
-        ahanetw = self._getAhaNetwork()
+    async def getAhaClone(self, iden):
+        lkey = s_common.uhex(iden)
+        byts = self.slab.get(lkey, db='aha:clones')
+        if byts is not None:
+            return s_msgpack.un(byts)
 
-        if self.sockaddr is None or not isinstance(self.sockaddr, tuple):
-            return None
+    async def addAhaClone(self, host, port=27492, conf=None):
+
+        if conf is None:
+            conf = {}
+
+        network = self._getAhaNetwork()
+
+        conf['mirror'] = self._getAhaUrl()
+
+        conf['dns:name'] = host
+        conf['aha:network'] = network
+        conf['dmon:listen'] = f'ssl://0.0.0.0:{port}?hostname={host}&ca={network}'
 
-        # FIXME this is the problematic use case
-        # FIXME may need to generate an AHA CA signed server cert for DNS NAME
+        iden = s_common.guid()
+        clone = {
+            'iden': iden,
+            'host': host,
+            'port': port,
+            'conf': conf,
+        }
+        return await self._push('aha:clone:add', clone)
 
-        # TODO this could eventually enumerate others via itself
-        netloc = self._getDnsName()
-        return (f'ssl://{netloc}:{self.sockaddr[1]}?certname={user}@{ahanetw}',)
+    @s_nexus.Pusher.onPush('aha:clone:add')
+    async def _addAhaClone(self, clone):
+        iden = clone.get('iden')
+        lkey = s_common.uhex(iden)
+        self.slab.put(lkey, s_msgpack.en(clone), db='aha:clones')
+        return self._getProvClientUrl(iden)
 
     async def addAhaSvcProv(self, name, provinfo=None):
 
         if not name:
             raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.')
 
-        ahaurls = self._getAhaUrls()
-        if ahaurls is None:
-            mesg = 'AHA server has no configured aha:urls.'
-            raise s_exc.NeedConfValu(mesg=mesg)
-
         self._reqProvListen()
 
         if provinfo is None:
@@ -1122,7 +1340,8 @@ async def addAhaSvcProv(self, name, provinfo=None):
         if ahaadmin is not None: # pragma: no cover
             conf.setdefault('aha:admin', ahaadmin)
 
-        conf.setdefault('aha:user', 'root')
+        ahauser = conf.setdefault('aha:user', 'root')
+        ahaurls = await self.getAhaUrls(user=ahauser)
 
         conf['aha:network'] = netw
 
@@ -1148,11 +1367,6 @@ async def addAhaSvcProv(self, name, provinfo=None):
         if peer:
             conf.setdefault('aha:leader', leader)
 
-        # allow user to win over leader
-        ahauser = conf.get('aha:user')
-        certname = f'{ahauser}@{netw}'
-        ahaurls = s_telepath.modurl(ahaurls, certname=certname)
-
         conf.setdefault('aha:registry', ahaurls)
 
         mirname = provinfo.get('mirror')
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index db3a66669f5..85b1b65507f 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1174,8 +1174,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         self.backlastexc = None  # err, errmsg, errtrace of last backup
 
         if self.conf.get('mirror') and not self.conf.get('nexslog:en'):
-            mesg = 'Mirror mode requires nexslog:en=True'
-            raise s_exc.BadConfValu(mesg=mesg)
+            self.modCellConf({'nexslog:en': True})
 
         # construct our nexsroot instance ( but do not start it )
         await s_nexus.Pusher.__anit__(self, self.iden)
@@ -1521,8 +1520,8 @@ def _getAhaNetwork(self):
 
     def _getDmonListen(self):
 
-        lisn = self.conf.get('dmon:listen')
-        if lisn is not None:
+        lisn = self.conf.get('dmon:listen', s_common.novalu)
+        if lisn is not s_common.novalu:
             return lisn
 
         network = self._getAhaNetwork()
@@ -3407,6 +3406,7 @@ def getArgParser(cls, conf=None):
         return pars
 
     async def _initCellBoot(self):
+        # NOTE: best hook point for custom provisioning
 
         pnfo = await self._bootCellProv()
 
@@ -3700,23 +3700,14 @@ async def readyToMirror(self):
             await self.nexsroot.enNexsLog()
             await self.sync()
 
-    async def _bootCellMirror(self, pnfo):
-        # this function must assume almost nothing is initialized
-        # but that's ok since it will only run rarely.
-        # It assumes it has a tuple of (provisioning configuration, provisioning iden) available
-        murl = self.conf.reqConfValu('mirror')
-        provconf, providen = pnfo
-
-        logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)')
+    async def _initCloneCell(self, proxy):
 
         tarpath = s_common.genpath(self.dirn, 'tmp', 'bootstrap.tgz')
-
         try:
 
-            async with await s_telepath.openurl(murl) as cell:
-                await cell.readyToMirror()
+                await proxy.readyToMirror()
                 with s_common.genfile(tarpath) as fd:
-                    async for byts in cell.iterNewBackupArchive(remove=True):
+                    async for byts in proxy.iterNewBackupArchive(remove=True):
                         fd.write(byts)
 
                 with tarfile.open(tarpath) as tgz:
@@ -3731,6 +3722,18 @@ async def _bootCellMirror(self, pnfo):
             if os.path.isfile(tarpath):
                 os.unlink(tarpath)
 
+    async def _bootCellMirror(self, pnfo):
+        # this function must assume almost nothing is initialized
+        # but that's ok since it will only run rarely.
+        # It assumes it has a tuple of (provisioning configuration, provisioning iden) available
+        murl = self.conf.reqConfValu('mirror')
+        provconf, providen = pnfo
+
+        logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)')
+
+        async with await s_telepath.openurl(murl) as proxy:
+            await self._initCloneCell(proxy)
+
         # Remove aha:provision from cell.yaml if it exists and the iden differs.
         mnfo = s_common.yamlload(self.dirn, 'cell.yaml')
         if mnfo:
diff --git a/synapse/tests/test_axon.py b/synapse/tests/test_axon.py
index ceb0d5cbbd6..f618e7edf58 100644
--- a/synapse/tests/test_axon.py
+++ b/synapse/tests/test_axon.py
@@ -1133,7 +1133,7 @@ async def test_axon_blob_v00_v01(self):
 
     async def test_axon_mirror(self):
 
-        async with self.getTestAhaProv() as aha:
+        async with self.getTestAha() as aha:
 
             axon00dirn = s_common.gendir(aha.dirn, 'tmp', 'axon00')
             axon01dirn = s_common.gendir(aha.dirn, 'tmp', 'axon01')
diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py
index 5acb96bdffb..cdafc355a0e 100644
--- a/synapse/tests/test_cortex.py
+++ b/synapse/tests/test_cortex.py
@@ -7904,7 +7904,8 @@ async def test_cortex_ext_httpapi(self):
                 await core.delHttpExtApi('notAGuid')
 
     async def test_cortex_query_offload(self):
-        async with self.getTestAhaProv() as aha:
+
+        async with self.getTestAha() as aha:
 
             async with await s_base.Base.anit() as base:
 
@@ -7927,11 +7928,11 @@ async def test_cortex_query_offload(self):
 
                     msgs = await core00.stormlist('aha.pool.add pool00...')
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('Created AHA service pool: pool00.loop.vertex.link', msgs)
+                    self.stormIsInPrint('Created AHA service pool: pool00.synapse', msgs)
 
                     msgs = await core00.stormlist('aha.pool.svc.add pool00... 01.core...')
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.loop.vertex.link)', msgs)
+                    self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.synapse)', msgs)
 
                     msgs = await core00.stormlist('cortex.storm.pool.set newp')
                     self.stormIsInErr(':// not found in [newp]', msgs)
@@ -8079,7 +8080,7 @@ async def test_cortex_query_offload(self):
                     waiter = core01.stormpool.waiter(1, 'svc:del')
                     msgs = await core01.stormlist('aha.pool.svc.del pool00... 01.core...', opts={'mirror': False})
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('AHA service (01.core.loop.vertex.link) removed from service pool (pool00.loop.vertex.link)', msgs)
+                    self.stormIsInPrint('AHA service (01.core.synapse) removed from service pool (pool00.synapse)', msgs)
 
                     # TODO: this wait should not return None
                     await waiter.wait(timeout=3)
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index 9d2f4a9f06e..0e10fb7bf65 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -44,30 +44,37 @@ async def exectelecall(self, url, meth, *args, **kwargs):
 
 class AhaTest(s_test.SynTest):
 
-    async def test_lib_aha_mirrors(self):
+    async def test_lib_aha_clone(self):
 
         with self.getTestDir() as dirn:
+
             dir0 = s_common.gendir(dirn, 'aha0')
             dir1 = s_common.gendir(dirn, 'aha1')
 
-            conf = {'nexslog:en': True}
+            async with self.getTestAha(dirn=dir0) as aha0:
 
-            async with self.getTestAha(conf={'nexslog:en': True}, dirn=dir0) as aha0:
-                user = await aha0.auth.addUser('reguser', passwd='secret')
-                await user.setAdmin(True)
+                async with aha0.getLocalProxy() as proxy0:
+                    self.len(1, await proxy0.getAhaUrls())
+                    self.len(1, await proxy0.getAhaServers())
 
-            s_tools_backup.backup(dir0, dir1)
+                    purl = await proxy0.addAhaClone('01.aha.loop.vertex.link')
 
-            async with self.getTestAha(conf=conf, dirn=dir0) as aha0:
-                upstream_url = aha0.getLocalUrl()
+                conf1 = {'clone': purl}
+                async with self.getTestAha(conf=conf1, dirn=dir1) as aha1:
 
-                mirrorconf = {
-                    'nexslog:en': True,
-                    'mirror': upstream_url,
-                }
+                    await aha1.sync()
+
+                    self.eq(aha0.iden, aha1.iden)
+                    self.nn(aha1.conf.get('mirror'))
+
+                    serv0 = await aha0.getAhaServers()
+                    serv1 = await aha1.getAhaServers()
+
+                    self.len(2, serv0)
+                    self.eq(serv0, serv1)
+
+                    # ensure some basic functionality is being properly mirrored
 
-                async with self.getTestAha(conf=mirrorconf, dirn=dir1) as aha1:
-                    # CA is nexus-fied
                     cabyts = await aha0.genCaCert('mirrorca')
                     await aha1.sync()
                     mirbyts = await aha1.genCaCert('mirrorca')
@@ -98,27 +105,22 @@ async def test_lib_aha_mirrors(self):
     async def test_lib_aha_offon(self):
         with self.getTestDir() as dirn:
             cryo0_dirn = s_common.gendir(dirn, 'cryo0')
-            conf = {'auth:passwd': 'secret'}
-            async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha:
-                host, port = await aha.dmon.listen('tcp://127.0.0.1:0')
+            async with self.getTestAha(dirn=dirn) as aha:
+                purl = await aha.addAhaSvcProv('0.cryo')
 
                 wait00 = aha.waiter(1, 'aha:svcadd')
-                cryo_conf = {
-                    'aha:name': '0.cryo',
-                    'aha:registry': f'tcp://root:secret@127.0.0.1:{port}',
-                    'dmon:listen': 'tcp://0.0.0.0:0/',
-                }
-                async with self.getTestCryo(dirn=cryo0_dirn, conf=cryo_conf) as cryo:
+
+                conf = {'aha:provision': purl}
+                async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
                     self.isin(len(await wait00.wait(timeout=6)), (1, 2))
 
                     svc = await aha.getAhaSvc('0.cryo...')
                     linkiden = svc.get('svcinfo', {}).get('online')
-                    self.nn(linkiden)
 
                     # Tear down the Aha cell.
                     await aha.__aexit__(None, None, None)
 
-            async with self.getTestAha(conf=conf.copy(), dirn=dirn) as aha:
+            async with self.getTestAha(dirn=dirn) as aha:
                 wait01 = aha.waiter(1, 'aha:svcdown')
                 await wait01.wait(timeout=6)
                 svc = await aha.getAhaSvc('0.cryo...')
@@ -377,15 +379,16 @@ async def test_lib_aha(self):
                 uinfo = ahainfo.get('urlinfo', {})
                 self.eq(uinfo.get('scheme'), 'unix')
                 self.none(uinfo.get('port'))
-                self.none(aha._getAhaUrls())
+                self.len(1, await aha.getAhaUrls())
 
             conf['dmon:listen'] = 'tcp://0.0.0.0:0/'
             async with self.getTestAha(conf=conf) as aha:
                 ahainfo = await aha.getAhaInfo()
+                ahaurls = await aha.getAhaUrls()
                 uinfo = ahainfo.get('urlinfo', {})
                 self.eq(uinfo.get('scheme'), 'tcp')
                 self.gt(uinfo.get('port'), 0)
-                self.eq(aha._getAhaUrls()[0], f'ssl://0.test.foo:{aha.sockaddr[1]}?certname=root@foo')
+                self.eq(ahaurls[0], f'ssl://00.aha.loop.vertex.link:{aha.sockaddr[1]}?certname=root@foo')
 
     async def test_lib_aha_loadenv(self):
 
@@ -430,53 +433,23 @@ async def test_lib_aha_finid_cell(self):
 
         async with self.getTestAha() as aha:
 
-            cryo0_dirn = s_common.gendir(aha.dirn, 'cryo0')
-
-            host, port = await aha.dmon.listen('tcp://127.0.0.1:0')
-            await aha.auth.rootuser.setPasswd('hehehaha')
-
-            aharegistry = [f'tcp://root:hehehaha@127.0.0.1:{port}',
-                          f'tcp://root:hehehaha@127.0.0.1:{port}']
-            atup = tuple(aharegistry)
-
             wait00 = aha.waiter(1, 'aha:svcadd')
-            conf = {
-                'aha:name': '0.cryo',
-                'aha:admin': 'root@synapse',
-                'aha:registry': aharegistry,
-                'dmon:listen': 'tcp://0.0.0.0:0/',
-            }
-            async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
+            conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')}
 
-                await cryo.auth.rootuser.setPasswd('secret')
-
-                ahaadmin = await cryo.auth.getUserByName('root@synapse')
-                self.nn(ahaadmin)
-                self.true(ahaadmin.isAdmin())
-
-                await wait00.wait(timeout=2)
+            async with self.getTestCryo(conf=conf) as cryo:
 
-                self.isin(atup, s_telepath.aha_clients)
+                self.true(await wait00.wait(timeout=2))
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
+                async with await s_telepath.openurl('aha://0.cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
-                _ahaclient = s_telepath.aha_clients.get(atup).get('client')
-                _aprx = await _ahaclient.proxy()
+                proxy = await cryo.ahaclient.proxy()
 
                 await aha.fini()
 
-                self.true(await _aprx.waitfini(timeout=10))
-
-                orig = s_telepath.Client.proxy
-                async def quickproxy(self, timeout):
-                    return await orig(self, timeout=0.1)
-
-                with mock.patch('synapse.telepath.Client.proxy', quickproxy):
-                    with self.raises(asyncio.TimeoutError):
-
-                        async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
-                            self.fail('Should never reach a connection.')
+                self.true(await proxy.waitfini(timeout=10))
+                with self.raises(asyncio.TimeoutError):
+                    await cryo.ahaclient.proxy(timeout=0.1)
 
     async def test_lib_aha_onlink_fail(self):
 
@@ -484,22 +457,10 @@ async def test_lib_aha_onlink_fail(self):
 
             async with self.getTestAha() as aha:
 
-                cryo0_dirn = s_common.gendir(aha.dirn, 'cryo0')
-
-                host, port = await aha.dmon.listen('tcp://127.0.0.1:0')
-                await aha.auth.rootuser.setPasswd('secret')
-
                 aha.testerr = True
-
                 wait00 = aha.waiter(1, 'aha:svcadd')
-                conf = {
-                    'aha:name': '0.cryo',
-                    'aha:registry': f'tcp://root:secret@127.0.0.1:{port}',
-                    'dmon:listen': 'tcp://0.0.0.0:0/',
-                }
-                async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
-
-                    await cryo.auth.rootuser.setPasswd('secret')
+                conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')}
+                async with self.getTestCryo(conf=conf) as cryo:
 
                     self.none(await wait00.wait(timeout=2))
 
@@ -515,7 +476,7 @@ async def test_lib_aha_onlink_fail(self):
                     self.nn(svc)
                     self.nn(svc.get('svcinfo', {}).get('online'))
 
-                    async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
+                    async with await s_telepath.openurl('aha://0.cryo...') as proxy:
                         self.nn(await proxy.getCellIden())
 
     async def test_lib_aha_bootstrap(self):
@@ -545,7 +506,8 @@ async def test_lib_aha_bootstrap(self):
 
     async def test_lib_aha_noconf(self):
 
-        async with self.getTestAha() as aha:
+        conf = {'dns:name': None}
+        async with self.getTestAha(conf=conf) as aha:
 
             with self.raises(s_exc.NeedConfValu):
                 await aha.addAhaSvcProv('hehe')
@@ -572,7 +534,6 @@ async def test_lib_aha_provision(self):
             async with self.getTestAha(dirn=dirn, conf=conf) as aha:
 
                 ahaport = aha.sockaddr[1]
-                aha.conf['aha:urls'] = f'ssl://aha.loop.vertex.link:{ahaport}'
 
                 url = aha.getLocalUrl()
 
@@ -876,21 +837,9 @@ async def test_lib_aha_provision(self):
 
     async def test_aha_httpapi(self):
 
-        conf = {
-            'aha:name': 'aha',
-            'aha:network': 'loop.vertex.link',
-            'provision:listen': 'ssl://aha.loop.vertex.link:0'
-        }
-        async with self.getTestAha(conf=conf) as aha:
-            await aha.auth.rootuser.setPasswd('secret')
-
-            addr, port = aha.provdmon.addr
-            # update the config to reflect the dynamically bound port
-            aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}'
+        async with self.getTestAha() as aha:
 
-            # do this config ex-post-facto due to port binding...
-            host, ahaport = await aha.dmon.listen('ssl://0.0.0.0:0?hostname=aha.loop.vertex.link&ca=loop.vertex.link')
-            aha.conf['aha:urls'] = f'ssl://aha.loop.vertex.link:{ahaport}'
+            await aha.auth.rootuser.setPasswd('secret')
 
             host, httpsport = await aha.addHttpsPort(0)
             url = f'https://localhost:{httpsport}/api/v1/aha/provision/service'
@@ -973,15 +922,11 @@ async def test_aha_httpapi(self):
                     self.eq(info.get('code'), 'AuthDeny')
 
     async def test_aha_connect_back(self):
-        async with self.getTestAhaProv() as aha:  # type: s_aha.AhaCell
 
-            async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller) as conn:
+        async with self.getTestAha() as aha:  # type: s_aha.AhaCell
 
-                ahaurl = aha.conf.get('aha:urls')[0]
-                ahaurl = s_telepath.modurl(ahaurl, user='root')
-
-                # This adminapi fails if the ssl://root@aha.loop.vertex.link
-                # session is not an admin user.
+            async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller) as conn:
+                ahaurl = aha._getAhaUrl()
                 await conn.exectelecall(ahaurl, 'getNexsIndx')
 
             self.true(conn.ahaclient.isfini)
@@ -989,26 +934,7 @@ async def test_aha_connect_back(self):
     async def test_aha_util_helpers(self):
 
         # Mainly for test helper coverage.
-
-        async with self.getTestAhaProv(conf={'auth:passwd': 'secret'}) as aha:  # type: s_aha.AhaCell
-            root = await aha.auth.getUserByName('root')
-            self.true(await root.tryPasswd('secret'))
-
-            import synapse.cortex as s_cortex
-
-            with self.getTestDir() as dirn:
-                cdr0 = s_common.genpath(dirn, 'core00')
-                cdr1 = s_common.genpath(dirn, 'core01')
-
-                async with self.addSvcToAha(aha, '00.core', s_cortex.Cortex, dirn=cdr0) as core00:
-                    async with self.addSvcToAha(aha, '01.core', s_cortex.Cortex, dirn=cdr1,
-                                                provinfo={'mirror': 'core'}) as core01:
-                        self.len(1, await core00.nodes('[inet:asn=0]'))
-                        await core01.sync()
-                        self.len(1, await core01.nodes('inet:asn=0'))
-
-        # Simple test setups should work without issue
-        async with self.getTestAhaProv() as aha:
+        async with self.getTestAha() as aha:
             async with self.addSvcToAha(aha, '00.cell', s_cell.Cell) as cell00:  # type: s_cell.Cell
                 async with self.addSvcToAha(aha, '01.cell', s_cell.Cell,
                                             provinfo={'mirror': 'cell'}) as cell01:  # type: s_cell.Cell
@@ -1023,26 +949,12 @@ async def test_aha_restart(self):
                 svc0dirn = s_common.gendir(dirn, 'svc00')
                 svc1dirn = s_common.gendir(dirn, 'svc01')
                 async with await s_base.Base.anit() as cm:
-                    aconf = {
-                        'aha:name': 'aha',
-                        'aha:network': 'loop.vertex.link',
-                        'provision:listen': 'ssl://aha.loop.vertex.link:0'
-                    }
-                    name = aconf.get('aha:name')
-                    netw = aconf.get('aha:network')
-                    dnsname = f'{name}.{netw}'
+
+                    aconf = {'dns:name': 'aha.loop.vertex.link'}
 
                     aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf)
                     await cm.enter_context(aha)
 
-                    addr, port = aha.provdmon.addr
-                    # update the config to reflect the dynamically bound port
-                    aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}'
-
-                    # do this config ex-post-facto due to port binding...
-                    host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}')
-                    aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',)
-
                     onetime = await aha.addAhaSvcProv('00.svc', provinfo=None)
                     sconf = {'aha:provision': onetime}
                     s_common.yamlsave(sconf, svc0dirn, 'cell.yaml')
@@ -1060,7 +972,7 @@ async def test_aha_restart(self):
                     await svc1.sync()
 
                     # Get Aha services
-                    snfo = await aha.getAhaSvc('01.svc.loop.vertex.link')
+                    snfo = await aha.getAhaSvc('01.svc...')
                     svcinfo = snfo.get('svcinfo')
                     ready = svcinfo.get('ready')
                     self.true(ready)
@@ -1068,20 +980,12 @@ async def test_aha_restart(self):
                     # Fini the Aha service.
                     await aha.fini()
 
-                    # Reuse our listening port we just deployed services with
-                    aconf = {
-                        'aha:name': 'aha',
-                        'aha:network': 'loop.vertex.link',
-                        'provision:listen': 'ssl://aha.loop.vertex.link:0',  # we do not care about provisioning
-                        'dmon:listen': f'ssl://{dnsname}:{ahaport}?hostname={dnsname}&ca={netw}'
-                    }
-
                     # Restart aha
                     aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf)
                     await cm.enter_context(aha)
 
                     # services are cleared
-                    snfo = await aha.getAhaSvc('01.svc.loop.vertex.link')
+                    snfo = await aha.getAhaSvc('01.svc...')
                     svcinfo = snfo.get('svcinfo')
                     ready = svcinfo.get('ready')
                     online = svcinfo.get('online')
@@ -1096,7 +1000,7 @@ async def test_aha_restart(self):
                     self.ge(len(await waiter.wait(timeout=12)), n)
 
                     # svc01 has reconnected and the ready state has been re-registered
-                    snfo = await aha.getAhaSvc('01.svc.loop.vertex.link')
+                    snfo = await aha.getAhaSvc('01.svc...')
                     svcinfo = snfo.get('svcinfo')
                     ready = svcinfo.get('ready')
                     online = svcinfo.get('online')
@@ -1105,7 +1009,7 @@ async def test_aha_restart(self):
 
     async def test_aha_service_pools(self):
 
-        async with self.getTestAhaProv() as aha:
+        async with self.getTestAha() as aha:
 
             async with await s_base.Base.anit() as base:
 
@@ -1126,7 +1030,7 @@ async def test_aha_service_pools(self):
 
                     msgs = await core00.stormlist('aha.pool.add pool00...')
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('Created AHA service pool: pool00.loop.vertex.link', msgs)
+                    self.stormIsInPrint('Created AHA service pool: pool00.synapse', msgs)
 
                     # Pool has no members....
                     pool = await s_telepath.open('aha://pool00...')
@@ -1135,7 +1039,7 @@ async def test_aha_service_pools(self):
 
                     msgs = await core00.stormlist('aha.pool.svc.add pool00... 00...')
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('AHA service (00...) added to service pool (pool00.loop.vertex.link)', msgs)
+                    self.stormIsInPrint('AHA service (00...) added to service pool (pool00.synapse)', msgs)
 
                     self.len(1, await waiter.wait(timeout=12))
                     prox = await pool.proxy(timeout=12)
@@ -1150,12 +1054,12 @@ async def test_aha_service_pools(self):
                     self.len(1, poolinfo['services'])
 
                     msgs = await core00.stormlist('aha.pool.list')
-                    self.stormIsInPrint('Pool: pool00.loop.vertex.link', msgs)
-                    self.stormIsInPrint('    00.loop.vertex.link', msgs)
+                    self.stormIsInPrint('Pool: pool00.synapse', msgs)
+                    self.stormIsInPrint('    00.synapse', msgs)
                     self.stormIsInPrint('1 pools', msgs)
 
-                    msgs = await core00.stormlist('$lib.print($lib.aha.pool.get(pool00.loop.vertex.link))')
-                    self.stormIsInPrint('aha:pool: pool00.loop.vertex.link', msgs)
+                    msgs = await core00.stormlist('$lib.print($lib.aha.pool.get(pool00.synapse))')
+                    self.stormIsInPrint('aha:pool: pool00.synapse', msgs)
 
                     async with await s_telepath.open('aha://pool00...') as pool:
 
@@ -1166,11 +1070,11 @@ async def test_aha_service_pools(self):
 
                         msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...')
                         self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('AHA service (01...) added to service pool (pool00.loop.vertex.link)', msgs)
+                        self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs)
 
                         msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...')
                         self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('AHA service (01...) added to service pool (pool00.loop.vertex.link)', msgs)
+                        self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs)
 
                         await waiter.wait(timeout=3)
 
@@ -1178,12 +1082,12 @@ async def test_aha_service_pools(self):
                         self.len(2, poolinfo['services'])
 
                         self.nn(poolinfo['created'])
-                        self.nn(poolinfo['services']['00.loop.vertex.link']['created'])
-                        self.nn(poolinfo['services']['01.loop.vertex.link']['created'])
+                        self.nn(poolinfo['services']['00.synapse']['created'])
+                        self.nn(poolinfo['services']['01.synapse']['created'])
 
                         self.eq(core00.auth.rootuser.iden, poolinfo['creator'])
-                        self.eq(core00.auth.rootuser.iden, poolinfo['services']['00.loop.vertex.link']['creator'])
-                        self.eq(core00.auth.rootuser.iden, poolinfo['services']['01.loop.vertex.link']['creator'])
+                        self.eq(core00.auth.rootuser.iden, poolinfo['services']['00.synapse']['creator'])
+                        self.eq(core00.auth.rootuser.iden, poolinfo['services']['01.synapse']['creator'])
 
                         for client in pool.clients.values():
                             await client.proxy(timeout=3)
@@ -1205,7 +1109,7 @@ async def test_aha_service_pools(self):
 
                         msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
                         self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('AHA service (00.loop.vertex.link) removed from service pool (pool00.loop.vertex.link)',
+                        self.stormIsInPrint('AHA service (00.synapse) removed from service pool (pool00.synapse)',
                                             msgs)
 
                         msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
@@ -1221,7 +1125,7 @@ async def test_aha_service_pools(self):
 
                     msgs = await core00.stormlist('aha.pool.del pool00...')
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('Removed AHA service pool: pool00.loop.vertex.link', msgs)
+                    self.stormIsInPrint('Removed AHA service pool: pool00.synapse', msgs)
 
     async def test_aha_reprovision(self):
         with self.withNexusReplay() as stack:
@@ -1234,23 +1138,14 @@ async def test_aha_reprovision(self):
                     aconf = {
                         'aha:name': 'aha',
                         'aha:network': 'loop.vertex.link',
-                        'provision:listen': 'ssl://aha.loop.vertex.link:0'
+                        'dns:name': 'aha.loop.vertex.link',
                     }
                     name = aconf.get('aha:name')
                     netw = aconf.get('aha:network')
-                    dnsname = f'{name}.{netw}'
 
                     aha = await s_aha.AhaCell.anit(aha00dirn, conf=aconf)
                     await cm.enter_context(aha)
 
-                    addr, port = aha.provdmon.addr
-                    # update the config to reflect the dynamically bound port
-                    aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}'
-
-                    # do this config ex-post-facto due to port binding...
-                    host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}')
-                    aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',)
-
                     onetime = await aha.addAhaSvcProv('00.svc', provinfo=None)
                     sconf = {'aha:provision': onetime}
                     s_common.yamlsave(sconf, svc0dirn, 'cell.yaml')
@@ -1281,23 +1176,14 @@ async def test_aha_reprovision(self):
                     aconf = {
                         'aha:name': 'aha',
                         'aha:network': 'loop.vertex.link',
-                        'provision:listen': 'ssl://aha.loop.vertex.link:0'
+                        'dns:name': 'aha.loop.vertex.link',
                     }
                     name = aconf.get('aha:name')
                     netw = aconf.get('aha:network')
-                    dnsname = f'{name}.{netw}'
 
                     aha = await s_aha.AhaCell.anit(aha01dirn, conf=aconf)
                     await cm.enter_context(aha)
 
-                    addr, port = aha.provdmon.addr
-                    # update the config to reflect the dynamically bound port
-                    aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}'
-
-                    # do this config ex-post-facto due to port binding...
-                    host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}')
-                    aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',)
-
                     onetime = await aha.addAhaSvcProv('00.svc', provinfo=None)
                     sconf = {'aha:provision': onetime}
                     s_common.yamlsave(sconf, svc0dirn, 'cell.yaml')
diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py
index 306b9df07d6..03659508394 100644
--- a/synapse/tests/test_lib_cell.py
+++ b/synapse/tests/test_lib_cell.py
@@ -1811,9 +1811,7 @@ async def test_backup_restore_aha(self):
         # ensure the mirror has a
         # backup the mirror
         # restore the backup
-        async with self.getTestAhaProv(conf={'auth:passwd': 'secret'}) as aha:  # type: s_aha.AhaCell
-            root = await aha.auth.getUserByName('root')
-            self.true(await root.tryPasswd('secret'))
+        async with self.getTestAha() as aha:  # type: s_aha.AhaCell
 
             with self.getTestDir() as dirn:
                 cdr0 = s_common.genpath(dirn, 'core00')
@@ -1896,9 +1894,7 @@ async def test_backup_restore_double_promote_aha(self):
         # ensure the mirror has a
         # backup the mirror
         # restore the backup
-        async with self.getTestAhaProv(conf={'auth:passwd': 'secret'}) as aha:  # type: s_aha.AhaCell
-            root = await aha.auth.getUserByName('root')
-            self.true(await root.tryPasswd('secret'))
+        async with self.getTestAha() as aha:  # type: s_aha.AhaCell
 
             with self.getTestDir() as dirn:
                 cdr0 = s_common.genpath(dirn, 'core00')
diff --git a/synapse/tests/test_lib_stormlib_aha.py b/synapse/tests/test_lib_stormlib_aha.py
index 9c17a317e02..c423088df96 100644
--- a/synapse/tests/test_lib_stormlib_aha.py
+++ b/synapse/tests/test_lib_stormlib_aha.py
@@ -10,7 +10,7 @@ class AhaLibTest(s_test.SynTest):
 
     async def test_stormlib_aha_basics(self):
 
-        async with self.getTestAhaProv() as aha:
+        async with self.getTestAha() as aha:
 
             with self.getTestDir() as dirn:
 
@@ -36,24 +36,24 @@ async def test_stormlib_aha_basics(self):
                 self.len(5, svcs)
 
                 svc = await core00.callStorm('return( $lib.aha.get(core...) )')
-                self.eq('core.loop.vertex.link', svc.get('name'))
-                svc = await core00.callStorm('return( $lib.aha.get(core.loop.vertex.link))')
-                self.eq('core.loop.vertex.link', svc.get('name'))
+                self.eq('core.synapse', svc.get('name'))
+                svc = await core00.callStorm('return( $lib.aha.get(core.synapse))')
+                self.eq('core.synapse', svc.get('name'))
                 svc = await core00.callStorm('return( $lib.aha.get(00.cell...))')
-                self.eq('00.cell.loop.vertex.link', svc.get('name'))
+                self.eq('00.cell.synapse', svc.get('name'))
                 svc = await core00.callStorm('return( $lib.aha.get(cell...))')
-                self.eq('cell.loop.vertex.link', svc.get('name'))
+                self.eq('cell.synapse', svc.get('name'))
                 svc = await core00.callStorm('$f=({"mirror": (true)}) return( $lib.aha.get(cell..., filters=$f))')
-                self.eq('01.cell.loop.vertex.link', svc.get('name'))
+                self.eq('01.cell.synapse', svc.get('name'))
 
                 # List the aha services available
                 msgs = await core00.stormlist('aha.svc.list --nexus')
                 self.stormIsInPrint('Nexus', msgs)
-                self.stormIsInPrint('00.cell.loop.vertex.link                      true   true   true', msgs)
-                self.stormIsInPrint('01.cell.loop.vertex.link                      false  true   true', msgs)
-                self.stormIsInPrint('cell.loop.vertex.link                         true   true   true', msgs)
-                self.stormIsInPrint('core.loop.vertex.link                         null   true   true', msgs)
-                self.stormIsInPrint('mysvc.loop.vertex.link                        null   true   true', msgs)
+                self.stormIsInPrint('00.cell.synapse                      true   true   true', msgs, whitespace=False)
+                self.stormIsInPrint('01.cell.synapse                      false  true   true', msgs, whitespace=False)
+                self.stormIsInPrint('cell.synapse                         true   true   true', msgs, whitespace=False)
+                self.stormIsInPrint('core.synapse                         null   true   true', msgs, whitespace=False)
+                self.stormIsInPrint('mysvc.synapse                        null   true   true', msgs, whitespace=False)
 
                 msgs = await core00.stormlist('aha.svc.list')
                 self.stormNotInPrint('Nexus', msgs)
@@ -63,24 +63,24 @@ async def test_stormlib_aha_basics(self):
                 # Omit checking part of that.
                 emsg = '''Resolved cell... to an AHA Service.
 
-Name:       cell.loop.vertex.link
+Name:       cell.synapse
 Online:     true
 Ready:      true
 Run iden:   ********************************
 Cell iden:  ********************************
 Leader:     cell
 Connection information:
-    ca:         loop.vertex.link
+    ca:         synapse
 '''
                 self.stormIsInPrint(emsg, msgs, deguid=True)
-                self.stormIsInPrint('    hostname:   00.cell.loop.vertex.link', msgs)
+                self.stormIsInPrint('    hostname:   00.cell.synapse', msgs)
                 self.stormIsInPrint('    scheme:     ssl', msgs)
                 self.stormIsInPrint('    user:       root', msgs)
 
                 msgs = await core00.stormlist('aha.svc.stat --nexus cell...')
                 emsg = '''Resolved cell... to an AHA Service.
 
-Name:       cell.loop.vertex.link
+Name:       cell.synapse
 Online:     true
 Ready:      true
 Run iden:   ********************************
@@ -88,14 +88,14 @@ async def test_stormlib_aha_basics(self):
 Leader:     cell
 Nexus:      1
 Connection information:
-    ca:         loop.vertex.link
+    ca:         synapse
 '''
                 self.stormIsInPrint(emsg, msgs, deguid=True)
 
                 msgs = await core00.stormlist('aha.svc.stat --nexus 01.cell...')
                 emsg = '''Resolved 01.cell... to an AHA Service.
 
-Name:       01.cell.loop.vertex.link
+Name:       01.cell.synapse
 Online:     true
 Ready:      true
 Run iden:   ********************************
@@ -103,15 +103,15 @@ async def test_stormlib_aha_basics(self):
 Leader:     cell
 Nexus:      1
 Connection information:
-    ca:         loop.vertex.link
+    ca:         synapse
 '''
                 self.stormIsInPrint(emsg, msgs, deguid=True)
 
                 # Full name works
-                msgs = await core00.stormlist('aha.svc.stat 01.cell.loop.vertex.link')
-                emsg = '''Resolved 01.cell.loop.vertex.link to an AHA Service.
+                msgs = await core00.stormlist('aha.svc.stat 01.cell.synapse')
+                emsg = '''Resolved 01.cell.synapse to an AHA Service.
 
-Name:       01.cell.loop.vertex.link
+Name:       01.cell.synapse
 '''
                 self.stormIsInPrint(emsg, msgs, deguid=True)
 
@@ -128,8 +128,8 @@ async def test_stormlib_aha_basics(self):
                 emsg = '''Resolved pool00... to an AHA Pool.
 
 The pool currently has 1 members.
-AHA Pool:   pool00.loop.vertex.link
-Member:     00.cell.loop.vertex.link'''
+AHA Pool:   pool00.synapse
+Member:     00.cell.synapse'''
                 self.stormIsInPrint(emsg, msgs)
 
                 # Shut down a service
@@ -139,16 +139,16 @@ async def test_stormlib_aha_basics(self):
                 self.len(nevents, await waiter.wait(timeout=12))
 
                 msgs = await core00.stormlist('aha.svc.list')
-                self.stormIsInPrint('01.cell.loop.vertex.link                      false  false  false', msgs)
+                self.stormIsInPrint('01.cell.synapse false  false  false', msgs, whitespace=False)
 
                 # Fake a record
                 await aha.addAhaSvc('00.newp', info={'urlinfo': {'scheme': 'tcp', 'host': '0.0.0.0', 'port': '3030'}},
-                                    network='loop.vertex.link')
+                                    network='synapse')
 
                 msgs = await core00.stormlist('aha.svc.list --nexus')
-                emsg = '00.newp.loop.vertex.link                      null   false  null  0.0.0.0         3030  ' \
+                emsg = '00.newp.synapse null   false  null  0.0.0.0         3030  ' \
                        'Service is not online. Will not attempt to retrieve its nexus offset.'
-                self.stormIsInPrint(emsg, msgs)
+                self.stormIsInPrint(emsg, msgs, whitespace=False)
 
                 self.none(await core00.callStorm('return($lib.aha.del(00.newp...))'))
                 msgs = await core00.stormlist('aha.svc.list')
@@ -161,22 +161,22 @@ async def test_stormlib_aha_basics(self):
                                                                  'port': '3030'},
                                                      'online': guid,
                                                      },
-                                    network='loop.vertex.link')
+                                    network='synapse')
                 msgs = await core00.stormlist('aha.svc.list --nexus')
-                emsg = '00.newp.loop.vertex.link                      null   true   null  0.0.0.0         3030  ' \
-                       'Failed to connect to Telepath service: "aha://00.newp.loop.vertex.link/" error:'
-                self.stormIsInPrint(emsg, msgs)
+                emsg = '00.newp.synapse null   true   null  0.0.0.0         3030  ' \
+                       'Failed to connect to Telepath service: "aha://00.newp.synapse/" error:'
+                self.stormIsInPrint(emsg, msgs, whitespace=False)
 
                 msgs = await core00.stormlist('aha.svc.stat --nexus 00.newp...')
                 emsg = '''Resolved 00.newp... to an AHA Service.
 
-Name:       00.newp.loop.vertex.link
+Name:       00.newp.synapse
 Online:     true
 Ready:      null
 Run iden:   $lib.null
 Cell iden:  $lib.null
 Leader:     Service did not register itself with a leader name.
-Nexus:      Failed to connect to Telepath service: "aha://00.newp.loop.vertex.link/" error: [Errno 111] Connect call failed ('0.0.0.0', 3030)
+Nexus:      Failed to connect to Telepath service: "aha://00.newp.synapse/" error: [Errno 111] Connect call failed ('0.0.0.0', 3030)
 Connection information:
     host:       0.0.0.0
     port:       3030
@@ -185,7 +185,7 @@ async def test_stormlib_aha_basics(self):
                 self.stormIsInPrint(emsg, msgs)
 
                 # Delete the fake service with its full service name
-                self.none(await core00.callStorm('return($lib.aha.del(00.newp.loop.vertex.link))'))
+                self.none(await core00.callStorm('return($lib.aha.del(00.newp.synapse))'))
                 self.none(await core00.callStorm('return($lib.aha.get(00.newp...))'))
 
                 # Coverage for sad paths
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index ab8ddba3f24..4df3109121f 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -677,7 +677,7 @@ async def fini(self):
 
 class TstOutPut(s_output.OutPutStr):
 
-    def expect(self, substr, throw=True):
+    def expect(self, substr, throw=True, whitespace=True):
         '''
         Check if a string is present in the messages captured by the OutPutStr object.
 
@@ -689,6 +689,11 @@ def expect(self, substr, throw=True):
             bool: True if the string is present; False if the string is not present and throw is False.
         '''
         outs = str(self)
+
+        if not whitespace:
+            outs = ' '.join(outs.split())
+            substr = ' '.join(outs.split())
+
         if outs.find(substr) == -1:
             if throw:
                 mesg = 'TestOutPut.expect(%s) not in %s' % (substr, outs)
@@ -1297,12 +1302,7 @@ async def getTestCore(self, conf=None, dirn=None):
         Returns:
             s_cortex.Cortex: A Cortex object.
         '''
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-
-        conf = copy.deepcopy(conf)
+        conf = self.getCellConf(conf=conf)
 
         mods = conf.get('modules')
 
@@ -1342,11 +1342,7 @@ async def getTestCoreAndProxy(self, conf=None, dirn=None):
     @contextlib.asynccontextmanager
     async def getTestJsonStor(self, dirn=None, conf=None):
 
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-        conf = copy.deepcopy(conf)
+        conf = self.getCellConf(conf=conf)
 
         with self.withNexusReplay():
             if dirn is not None:
@@ -1367,20 +1363,10 @@ async def getTestCryo(self, dirn=None, conf=None):
         Returns:
             s_cryotank.CryoCell: Test cryocell.
         '''
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-        conf = copy.deepcopy(conf)
+        conf = self.getCellConf(conf=conf)
 
         with self.withNexusReplay():
-            if dirn is not None:
-                async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
-                    yield cryo
-
-                return
-
-            with self.getTestDir() as dirn:
+            with self.mayTestDir(dirn) as dirn:
                 async with await s_cryotank.CryoCell.anit(dirn, conf=conf) as cryo:
                     yield cryo
 
@@ -1451,62 +1437,59 @@ async def getTestCoreProxSvc(self, ssvc, ssvc_conf=None, core_conf=None):
 
                 yield core, prox, testsvc
 
+    @contextlib.contextmanager
+    def mayTestDir(self, dirn):
+
+        if dirn is not None:
+            yield dirn
+            return
+
+        with self.getTestDir() as dirn:
+            yield dirn
+
     @contextlib.asynccontextmanager
     async def getTestAha(self, conf=None, dirn=None):
 
         if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-        conf = copy.deepcopy(conf)
+            conf = {}
 
-        with self.withNexusReplay():
-            if dirn:
-                async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
-                    yield aha
-            else:
-                with self.getTestDir() as dirn:
-                    async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
-                        yield aha
+        conf = s_msgpack.deepcopy(conf)
 
-    @contextlib.asynccontextmanager
-    async def getTestAhaProv(self, conf=None, dirn=None):
-        '''
-        Get an Aha cell that is configured for provisioning on aha.loop.vertex.link.
+        hasdmonlisn = 'dmon:listen' in conf
+        hasprovlisn = 'provision:listen' in conf
 
-        Args:
-            conf: Optional configuration information for the Aha cell.
-            dirn: Optional path to create the Aha cell in.
+        network = conf.setdefault('aha:network', 'synapse')
+        hostname = conf.setdefault('dns:name', '00.aha.loop.vertex.link')
 
-        Returns:
-            s_aha.AhaCell: The provisioned Aha cell.
-        '''
-        bconf = {
-            'aha:name': 'aha',
-            'aha:network': 'loop.vertex.link',
-            'provision:listen': 'ssl://aha.loop.vertex.link:0'
-        }
+        if hostname:
+            conf.setdefault('provision:listen', f'ssl://0.0.0.0:0?hostname={hostname}')
+            conf.setdefault('dmon:listen', f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}')
 
-        if conf is None:
-            conf = bconf
-        else:
-            for k, v in bconf.items():
-                conf.setdefault(k, v)
+        conf.setdefault('health:sysctl:checks', False)
 
-        name = conf.get('aha:name')
-        netw = conf.get('aha:network')
-        dnsname = f'{name}.{netw}'
+        with self.withNexusReplay():
+            with self.mayTestDir(dirn) as dirn:
 
-        async with self.getTestAha(conf=conf, dirn=dirn) as aha:
-            addr, port = aha.provdmon.addr
-            # update the config to reflect the dynamically bound port
-            aha.conf['provision:listen'] = f'ssl://{dnsname}:{port}'
+                async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
 
-            # do this config ex-post-facto due to port binding...
-            host, ahaport = await aha.dmon.listen(f'ssl://0.0.0.0:0?hostname={dnsname}&ca={netw}')
-            aha.conf['aha:urls'] = (f'ssl://{dnsname}:{ahaport}',)
+                    mods = {}
+                    if not hasdmonlisn and hostname:
+                        mods['dmon:listen'] = f'ssl://0.0.0.0:{aha.sockaddr[1]}?hostname={hostname}&ca={network}'
 
-            yield aha
+                    if not hasprovlisn and hostname:
+                        mods['provision:listen'] = f'ssl://0.0.0.0:{aha.provaddr[1]}?hostname={hostname}'
+
+                    if mods:
+                        aha.modCellConf(mods)
+
+                    yield aha
+
+    def getCellConf(self, conf=None):
+        if conf is None:
+            conf = {}
+        conf = s_msgpack.deepcopy(conf)
+        conf.setdefault('health:sysctl:checks', False)
+        return conf
 
     @contextlib.asynccontextmanager
     async def addSvcToAha(self, aha, svcname, ctor,
@@ -1528,36 +1511,18 @@ async def addSvcToAha(self, aha, svcname, ctor,
             The config data for the cell is pushed into dirn/cell.yaml.
             The cells are created with the ``ctor.anit()`` function.
         '''
+        svcfull = f'{svcname}.{aha._getAhaNetwork()}'
         onetime = await aha.addAhaSvcProv(svcname, provinfo=provinfo)
 
-        if conf is None:
-            conf = {
-                'health:sysctl:checks': False,
-            }
-
+        conf = self.getCellConf(conf=conf)
         conf['aha:provision'] = onetime
 
-        logger.info(f'Adding {svcname}')
-
-        n = 1  # a simple svcname like "foo"
-        if len(svcname.split('.')) > 1:
-            n = 2  # A replica name like 00.foo
-        if provinfo and 'mirror' in provinfo:
-            n = 1  # A mirror like 01.foo with mirror: foo
-
-        waiter = aha.waiter(n, 'aha:svcadd')
-
-        if dirn:
+        waiter = aha.waiter(1, f'aha:svcadd:{svcfull}')
+        with self.mayTestDir(dirn) as dirn:
             s_common.yamlsave(conf, dirn, 'cell.yaml')
             async with await ctor.anit(dirn) as svc:
-                self.ge(len(await waiter.wait(timeout=12)), n)
+                self.true(await waiter.wait(timeout=12))
                 yield svc
-        else:
-            with self.getTestDir() as dirn:
-                s_common.yamlsave(conf, dirn, 'cell.yaml')
-                async with await ctor.anit(dirn) as svc:
-                    self.ge(len(await waiter.wait(timeout=12)), n)
-                    yield svc
 
     async def addSvcToCore(self, svc, core, svcname='svc'):
         '''
@@ -2154,7 +2119,7 @@ async def agenlen(self, x, obj, msg=None):
             count += 1
         self.eq(x, count, msg=msg)
 
-    def stormIsInPrint(self, mesg, mesgs, deguid=False):
+    def stormIsInPrint(self, mesg, mesgs, deguid=False, whitespace=True):
         '''
         Check if a string is present in all of the print messages from a stream of storm messages.
 
@@ -2166,6 +2131,9 @@ def stormIsInPrint(self, mesg, mesgs, deguid=False):
             mesg = deguidify(mesg)
 
         print_str = '\n'.join([m[1].get('mesg') for m in mesgs if m[0] == 'print'])
+        if not whitespace:
+            mesg = ' '.join(mesg.split())
+            print_str = ' '.join(print_str.split())
 
         if deguid:
             print_str = deguidify(print_str)

From c9a9a9d0d9ee3621b1d57f4dbb2b140e6949b15d Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 28 Jun 2024 15:42:24 -0400
Subject: [PATCH 04/68] wip

---
 synapse/tests/test_cortex.py | 53 ++++++++++--------------------------
 synapse/tests/utils.py       |  7 ++---
 2 files changed, 16 insertions(+), 44 deletions(-)

diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py
index cdafc355a0e..bf2b9ca9413 100644
--- a/synapse/tests/test_cortex.py
+++ b/synapse/tests/test_cortex.py
@@ -54,40 +54,21 @@ async def test_cortex_cellguid(self):
     async def test_cortex_handoff(self):
 
         with self.getTestDir() as dirn:
-            ahadir = s_common.genpath(dirn, 'aha00')
-            coredir0 = s_common.genpath(dirn, 'core00')
-            coredir1 = s_common.genpath(dirn, 'core01')
-            coredir2 = s_common.genpath(dirn, 'core02',)
-
-            conf = {
-                'aha:name': 'aha',
-                'aha:network': 'newp',
-                'provision:listen': 'tcp://127.0.0.1:0',
-            }
-            async with self.getTestAha(dirn=ahadir, conf=conf) as aha:
-
-                provaddr, provport = aha.provdmon.addr
-                aha.conf['provision:listen'] = f'tcp://127.0.0.1:{provport}'
+            async with self.getTestAha() as aha:
 
-                ahahost, ahaport = await aha.dmon.listen('ssl://127.0.0.1:0?hostname=aha.newp&ca=newp')
-                aha.conf['aha:urls'] = (f'ssl://127.0.0.1:{ahaport}?hostname=aha.newp',)
+                conf = {'aha:provision': await aha.addAhaSvcProv('00.cortex')}
 
-                provurl = await aha.addAhaSvcProv('00.cortex')
-                coreconf = {'aha:provision': provurl, 'nexslog:en': False}
-
-                async with self.getTestCore(dirn=coredir0, conf=coreconf) as core00:
+                async with self.getTestCore(conf=conf) as core00:
 
                     with self.raises(s_exc.BadArg):
                         await core00.handoff(core00.getLocalUrl())
 
                     self.false((await core00.getCellInfo())['cell']['uplink'])
 
-                    provinfo = {'mirror': '00.cortex'}
-                    provurl = await aha.addAhaSvcProv('01.cortex', provinfo=provinfo)
-
                     # provision with the new hostname and mirror config
-                    coreconf = {'aha:provision': provurl}
-                    async with self.getTestCore(dirn=coredir1, conf=coreconf) as core01:
+                    provinfo = {'mirror': '00.cortex'}
+                    conf = {'aha:provision': await aha.addAhaSvcProv('01.cortex', provinfo=provinfo)}
+                    async with self.getTestCore(conf=conf) as core01:
 
                         # test out connecting to the leader but having aha chose a mirror
                         async with s_telepath.loadTeleCell(core01.dirn):
@@ -117,9 +98,9 @@ async def test_cortex_handoff(self):
                         self.true((await core00.getCellInfo())['cell']['uplink'])
                         self.false((await core01.getCellInfo())['cell']['uplink'])
 
-                        mods00 = s_common.yamlload(coredir0, 'cell.mods.yaml')
-                        mods01 = s_common.yamlload(coredir1, 'cell.mods.yaml')
-                        self.eq(mods00, {'mirror': 'aha://01.cortex.newp'})
+                        mods00 = s_common.yamlload(core00.dirn, 'cell.mods.yaml')
+                        mods01 = s_common.yamlload(core01.dirn, 'cell.mods.yaml')
+                        self.eq(mods00, {'mirror': 'aha://01.cortex.synapse'})
                         self.eq(mods01, {'mirror': None})
 
                         await core00.nodes('[inet:ipv4=5.5.5.5]')
@@ -129,12 +110,11 @@ async def test_cortex_handoff(self):
                         # This pops the mirror config out of the mods file we copied
                         # from the backup.
                         provinfo = {'mirror': '01.cortex'}
-                        provurl = await aha.addAhaSvcProv('02.cortex', provinfo=provinfo)
-                        coreconf = {'aha:provision': provurl}
-                        async with self.getTestCore(dirn=coredir2, conf=coreconf) as core02:
+                        conf = {'aha:provision': await aha.addAhaSvcProv('02.cortex', provinfo=provinfo)}
+                        async with self.getTestCore(conf=conf) as core02:
                             self.false(core02.isactive)
-                            self.eq(core02.conf.get('mirror'), 'aha://root@01.cortex.newp')
-                            mods02 = s_common.yamlload(coredir2, 'cell.mods.yaml')
+                            self.eq(core02.conf.get('mirror'), 'aha://root@01.cortex...')
+                            mods02 = s_common.yamlload(core02.dirn, 'cell.mods.yaml')
                             self.eq(mods02, {})
                             # The mirror writeback and change distribution works
                             self.len(0, await core01.nodes('inet:ipv4=6.6.6.6'))
@@ -144,7 +124,7 @@ async def test_cortex_handoff(self):
                             self.len(1, await core01.nodes('inet:ipv4=6.6.6.6'))
                             self.len(1, await core00.nodes('inet:ipv4=6.6.6.6'))
                             # list mirrors
-                            exp = ['aha://00.cortex.newp', 'aha://02.cortex.newp']
+                            exp = ['aha://00.cortex.synapse', 'aha://02.cortex.synapse']
                             self.sorteq(exp, await core00.getMirrorUrls())
                             self.sorteq(exp, await core01.getMirrorUrls())
                             self.sorteq(exp, await core02.getMirrorUrls())
@@ -5373,11 +5353,6 @@ async def test_cortex_mirror(self):
 
                 url = core00.getLocalUrl()
 
-                core01conf = {'nexslog:en': False, 'mirror': url}
-                with self.raises(s_exc.BadConfValu):
-                    async with self.getTestCore(dirn=path01, conf=core01conf) as core01:
-                        self.fail('Should never get here.')
-
                 core01conf = {'mirror': url}
 
                 async with self.getTestCore(dirn=path01, conf=core01conf) as core01:
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index 4df3109121f..86fdce2a9c8 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -1304,11 +1304,8 @@ async def getTestCore(self, conf=None, dirn=None):
         '''
         conf = self.getCellConf(conf=conf)
 
-        mods = conf.get('modules')
-
-        if mods is None:
-            mods = []
-            conf['modules'] = mods
+        mods = list(conf.get('modules', ()))
+        conf['modules'] = mods
 
         mods.insert(0, ('synapse.tests.utils.TestModule', {'key': 'valu'}))
 

From c15ba7c12a4914cef0f7302bf3693d8bf57b8891 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 28 Jun 2024 17:07:00 -0400
Subject: [PATCH 05/68] wip

---
 synapse/tests/test_lib_stormlib_cell.py |  19 +--
 synapse/tests/test_tools_aha.py         | 169 +++++++++---------------
 2 files changed, 69 insertions(+), 119 deletions(-)

diff --git a/synapse/tests/test_lib_stormlib_cell.py b/synapse/tests/test_lib_stormlib_cell.py
index 0ce45cb1d3e..95732aa4934 100644
--- a/synapse/tests/test_lib_stormlib_cell.py
+++ b/synapse/tests/test_lib_stormlib_cell.py
@@ -76,18 +76,7 @@ async def test_stormlib_cell_uptime(self):
 
     async def test_stormlib_cell_getmirrors(self):
 
-        conf = {
-            'aha:name': 'aha',
-            'aha:network': 'mynet',
-            'provision:listen': 'tcp://127.0.0.1:0',
-        }
-        async with self.getTestCell(s_aha.AhaCell, conf=conf) as aha:
-
-            provaddr, provport = aha.provdmon.addr
-            aha.conf['provision:listen'] = f'tcp://127.0.0.1:{provport}'
-
-            ahahost, ahaport = await aha.dmon.listen('ssl://127.0.0.1:0?hostname=aha.mynet&ca=mynet')
-            aha.conf['aha:urls'] = (f'ssl://127.0.0.1:{ahaport}?hostname=aha.mynet',)
+        async with self.getTestAha() as aha:
 
             provurl = await aha.addAhaSvcProv('00.cortex')
             coreconf = {'aha:provision': provurl}
@@ -116,7 +105,7 @@ async def test_stormlib_cell_getmirrors(self):
                     await core01.nodes('[ inet:ipv4=1.2.3.4 ]')
                     self.len(1, await core00.nodes('inet:ipv4=1.2.3.4'))
 
-                    expurls = ['aha://01.cortex.mynet']
+                    expurls = ['aha://01.cortex.synapse']
 
                     self.eq(expurls, await core00.callStorm('return($lib.cell.getMirrorUrls())'))
                     self.eq(expurls, await core01.callStorm('return($lib.cell.getMirrorUrls())'))
@@ -148,11 +137,11 @@ async def test_stormlib_cell_getmirrors(self):
 
                         await svc01.sync()
 
-                        expurls = ('aha://01.testsvc.mynet',)
+                        expurls = ('aha://01.testsvc.synapse',)
 
                         self.eq(expurls, await core00.callStorm('return($lib.cell.getMirrorUrls(name=testsvc))'))
 
-                    await aha.delAhaSvc('00.testsvc.mynet')
+                    await aha.delAhaSvc('00.testsvc.synapse')
 
                     with self.raises(s_exc.NoSuchName):
                         await core00.callStorm('return($lib.cell.getMirrorUrls(name=testsvc))')
diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py
index 647a6919546..a4fd7012451 100644
--- a/synapse/tests/test_tools_aha.py
+++ b/synapse/tests/test_tools_aha.py
@@ -17,40 +17,37 @@ class AhaToolsTest(s_t_utils.SynTest):
 
     async def test_aha_list(self):
 
-        ephemeral_address = 'tcp://0.0.0.0:0/'
-        async with self.getTestAha(conf={'auth:passwd': 'root',
-                                         'dmon:listen': ephemeral_address}) as aha:
-            _, port = aha.sockaddr
-            ahaurl = f'tcp://root:root@127.0.0.1:{port}'
-            conf0 = {
-                'aha:registry': ahaurl,
-                'aha:name': 'cell0',
-                'aha:network': 'demo.net',
-                'dmon:listen': ephemeral_address,
-            }
-            conf1 = {
-                'aha:registry': ahaurl,
-                'aha:name': 'cell1',
-                'aha:network': 'example.net',
-                'dmon:listen': ephemeral_address,
-            }
+        async with self.getTestAha() as aha:
+
             waiter = aha.waiter(2, 'aha:svcadd')
+            conf0 = {'aha:provision': await aha.addAhaSvcProv('cell0')}
+
+            provinfo = {'aha:network': 'example.net'}
+            conf1 = {'aha:provision': await aha.addAhaSvcProv('cell1', provinfo=provinfo)}
+
+            ahaurl = aha.getLocalUrl()
+
             async with self.getTestCell(s_cell.Cell, conf=conf0) as cell0:
+
                 async with self.getTestCell(s_cell.Cell, conf=conf1) as cell1:
+
                     self.true(await waiter.wait(timeout=6))
+
                     argv = [ahaurl]
                     retn, outp = await self.execToolMain(s_a_list._main, argv)
                     self.eq(retn, 0)
-                    outp.expect('Service              network                        leader')
-                    outp.expect('cell0                demo.net                       None')
-                    outp.expect('cell1                example.net                    None')
+
+                    outp.expect('''
+                        Service              network                        leader
+                        cell0                synapse                        None
+                        cell1                example.net                    None
+                    ''', whitespace=False)
 
                     argv = [ahaurl, 'demo.net']
                     retn, outp = await self.execToolMain(s_a_list._main, argv)
                     self.eq(retn, 0)
-                    outp.expect('Service              network')
-                    outp.expect('cell0                demo.net')
-                    self.false(outp.expect('cell1                example.net', throw=False))
+                    outp.expect('Service              network', whitespace=False)
+                    outp.expect('cell0                demo.net', whitespace=False)
 
         async with self.getTestCore() as core:
             curl = core.getLocalUrl()
@@ -94,102 +91,66 @@ async def test_aha_easycert(self):
 
     async def test_aha_enroll(self):
 
-        with self.getTestDir() as dirn:
-
-            conf = {
-                'aha:name': 'aha',
-                'aha:network': 'loop.vertex.link',
-                'provision:listen': 'ssl://aha.loop.vertex.link:0',
-                'dmon:listen': 'tcp://0.0.0.0:0'
-            }
-            async with self.getTestAha(dirn=dirn, conf=conf) as aha:
+        async with self.getTestAha() as aha:
 
-                addr, port = aha.provdmon.addr
-                aha.conf['provision:listen'] = f'ssl://aha.loop.vertex.link:{port}'
+            argv = ['--url', aha.getLocalUrl(), 'visi']
+            retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
+            self.isin('one-time use URL:', str(outp))
 
-                host_, ahaport = aha.sockaddr
-                self.eq(aha._getAhaUrls(), [f'ssl://aha.loop.vertex.link:{ahaport}'])
+            provurl = str(outp).split(':', 1)[1].strip()
+            with self.getTestSynDir() as syndir:
 
-                argv = ['--url', aha.getLocalUrl(), 'visi']
-                retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
-                self.isin('one-time use URL:', str(outp))
-
-                provurl = str(outp).split(':', 1)[1].strip()
-                with self.getTestSynDir() as syndir:
+                capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt')
+                crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
+                keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
 
-                    capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt')
-                    crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt')
-                    keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key')
+                retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
+                self.eq(0, retn)
 
-                    for path in (capath, crtpath, keypath):
-                        s_common.genfile(path)
+                for path in (capath, crtpath, keypath):
+                    self.true(os.path.isfile(path))
+                    self.gt(os.path.getsize(path), 0)
 
-                    yamlpath = s_common.genpath(syndir, 'telepath.yaml')
-                    s_common.yamlsave({'aha:servers': 'cell://aha'}, yamlpath)
+                teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
+                self.eq(teleyaml.get('version'), 1)
+                self.len(1, teleyaml.get('aha:servers'))
 
-                    retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
-                    self.eq(0, retn)
+                shutil.rmtree(s_common.genpath(syndir, 'certs'))
 
-                    for path in (capath, crtpath, keypath):
-                        self.gt(os.path.getsize(path), 0)
-
-                    teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
-                    self.eq(teleyaml.get('version'), 1)
-                    self.eq(teleyaml.get('aha:servers'), ('cell://aha', f'ssl://visi@aha.loop.vertex.link:{ahaport}'))
-
-                    shutil.rmtree(s_common.genpath(syndir, 'certs'))
-
-                    def _strUrl(self):
-                        return 'ssl://aha.loop.vertex.link'
-
-                    with mock.patch('synapse.lib.aha.AhaCell._getAhaUrls', _strUrl):
-
-                        argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
-                        retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
-                        self.eq(retn, 0)
-                        self.isin('one-time use URL:', str(outp))
-
-                        provurl = str(outp).split(':', 1)[1].strip()
-
-                        retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
-
-                        servers = ['cell://aha',
-                                   'ssl://visi@aha.loop.vertex.link',
-                                   f'ssl://visi@aha.loop.vertex.link:{ahaport}']
+                argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
+                retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
+                self.eq(retn, 0)
+                self.isin('one-time use URL:', str(outp))
 
-                        teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
-                        self.sorteq(teleyaml.get('aha:servers'), servers)
+                provurl = str(outp).split(':', 1)[1].strip()
 
-                        # Just return the URL
-                        retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url'])
-                        self.eq(retn, 0)
-                        self.notin('one-time use URL:', str(outp))
-                        self.isin('ssl://', str(outp))
+                retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
 
-                with self.getTestSynDir() as syndir:
+                # Just return the URL
+                retn, outp = await self.execToolMain(s_a_provision_user.main, argv + ['--only-url'])
+                self.eq(retn, 0)
+                self.notin('one-time use URL:', str(outp))
+                self.isin('ssl://', str(outp))
 
-                    argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
-                    retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
-                    self.eq(retn, 0)
-                    provurl = str(outp).split(':', 1)[1].strip()
+            with self.getTestSynDir() as syndir:
 
-                    capath = s_common.genpath(syndir, 'certs', 'cas', 'loop.vertex.link.crt')
-                    crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.crt')
-                    keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@loop.vertex.link.key')
+                argv = ['--again', '--url', aha.getLocalUrl(), 'visi']
+                retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
+                self.eq(retn, 0)
+                provurl = str(outp).split(':', 1)[1].strip()
 
-                    for path in (capath, crtpath, keypath):
-                        s_common.genfile(path)
+                capath = s_common.genpath(syndir, 'certs', 'cas', 'synapse.crt')
+                crtpath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.crt')
+                keypath = s_common.genpath(syndir, 'certs', 'users', 'visi@synapse.key')
 
-                    yamlpath = s_common.genpath(syndir, 'telepath.yaml')
-                    s_common.yamlsave({'aha:servers': ['cell://aha', 'cell://foo', ['cell://bar']]}, yamlpath)
+                for path in (capath, crtpath, keypath):
+                    s_common.genfile(path)
 
-                    retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
-                    self.eq(0, retn)
+                retn, outp = await self.execToolMain(s_a_enroll.main, (provurl,))
+                self.eq(0, retn)
 
-                    for path in (capath, crtpath, keypath):
-                        self.gt(os.path.getsize(path), 0)
+                for path in (capath, crtpath, keypath):
+                    self.gt(os.path.getsize(path), 0)
 
-                    teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
-                    self.eq(teleyaml.get('version'), 1)
-                    self.eq(teleyaml.get('aha:servers'), ('cell://aha', 'cell://foo', ('cell://bar',),
-                                                          f'ssl://visi@aha.loop.vertex.link:{ahaport}'))
+                teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
+                self.eq(teleyaml.get('version'), 1)

From 9ac5fbf6841c413efdcc47e0ada10766a8673efa Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Sat, 29 Jun 2024 17:26:05 -0400
Subject: [PATCH 06/68] wip

---
 synapse/lib/cell.py            | 20 +++++++++++++-------
 synapse/tests/test_lib_cell.py |  2 +-
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 85b1b65507f..576b018c804 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1491,22 +1491,23 @@ def _getAhaAdmin(self):
 
     async def _initAhaRegistry(self):
 
+        # NOTE: This API needs to be safe to call again
         ahaurl = self.conf.get('aha:registry')
         if ahaurl is not None:
 
             info = await s_telepath.addAhaUrl(ahaurl)
-            if self.ahaclient is None:
-                self.ahaclient = await s_telepath.Client.anit(info.get('url'))
-                self.ahaclient._fini_atexit = True
-                self.onfini(self.ahaclient)
+            if self.ahaclient is not None:
+                await self.ahaclient.fini()
 
-            async def finiaha():
+            self.ahaclient = await s_telepath.Client.anit(ahaurl)
+            self.onfini(self.ahaclient)
+
+            async def fini():
                 await s_telepath.delAhaUrl(ahaurl)
 
-            self.onfini(finiaha)
+            self.ahaclient.onfini(fini)
 
         ahauser = self.conf.get('aha:user')
-        ahanetw = self._getAhaNetwork()
 
         ahaadmin = self._getAhaAdmin()
         if ahaadmin is not None:
@@ -1652,15 +1653,19 @@ async def _initAhaService(self):
         async def _runAhaRegLoop():
 
             while not self.isfini:
+
                 try:
                     proxy = await self.ahaclient.proxy()
+
                     info = await self.getAhaInfo()
                     await proxy.addAhaSvc(ahaname, info, network=ahanetw)
                     if self.isactive and ahalead is not None:
                         await proxy.addAhaSvc(ahalead, info, network=ahanetw)
+
                 except Exception as e:
                     logger.exception(f'Error registering service {self.ahasvcname} with AHA: {e}')
                     await self.waitfini(1)
+
                 else:
                     await proxy.waitfini()
 
@@ -4073,6 +4078,7 @@ async def getCellInfo(self):
                 'type': self.getCellType(),
                 'iden': self.getCellIden(),
                 'active': self.isactive,
+                'started': self.startms,
                 'ready': self.nexsroot.ready.is_set(),
                 'commit': self.COMMIT,
                 'version': self.VERSION,
diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py
index 03659508394..c7a942dd57e 100644
--- a/synapse/tests/test_lib_cell.py
+++ b/synapse/tests/test_lib_cell.py
@@ -520,7 +520,7 @@ async def test_cell_getinfo(self):
                 self.isin('cortex:defaults', cnfo.get('cellvers', {}))
 
                 # Defaults aha data is
-                self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': None})
+                self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': 'synapse'})
 
                 # Synapse information
                 self.eq(snfo.get('version'), s_version.version)

From 4b699093870c8157197b1675eda115e3b743c3d9 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 1 Jul 2024 16:10:38 -0400
Subject: [PATCH 07/68] wip

---
 synapse/lib/aha.py            | 57 ++++-------------------------
 synapse/lib/cell.py           | 69 ++++++++++++++++++-----------------
 synapse/lib/nexus.py          | 23 +++++++++---
 synapse/tests/test_lib_aha.py |  1 +
 synapse/tests/utils.py        |  8 +++-
 5 files changed, 67 insertions(+), 91 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index b93d4c17833..d68160acdb6 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -289,10 +289,6 @@ async def addAhaPoolSvc(self, poolname, svcname, info):
     async def delAhaPoolSvc(self, poolname, svcname):
         return await self.cell.delAhaPoolSvc(poolname, svcname)
 
-    async def iterAhaTopo(self):
-        async for item in self.cell.iterAhaTopo():
-            yield item
-
     async def iterPoolTopo(self, name):
 
         username = self.user.name.split('@')[0]
@@ -577,7 +573,6 @@ async def initServiceStorage(self):
         slab = await s_lmdbslab.Slab.anit(dirn)
         slab.addResizeCallback(self.checkFreeSpace)
 
-        self.topobus = await s_base.Base.anit()
         self.jsonstor = await s_jsonstor.JsonStor.anit(slab, 'aha')  # type: s_jsonstor.JsonStor
 
         async def fini():
@@ -594,7 +589,6 @@ async def fini():
 
         self.slab.initdb('aha:pools')
 
-        self.topowindows = []
         self.poolwindows = collections.defaultdict(list)
 
     async def getAhaServer(self, host, port):
@@ -615,16 +609,6 @@ async def addAhaServer(self, server):
 
         return await self._push('aha:server:add', server)
 
-    async def _getTopoUpdate(self):
-        return ('aha:update', {
-            'servers': await self.getAhaServers(),
-        }),
-
-    async def _fireTopoUpdate(self):
-        update = await self._getTopoUpdate()
-        for wind in self.topowindows:
-            await wind.put(update)
-
     @s_nexus.Pusher.onPush('aha:server:add')
     async def _addAhaServer(self, server):
         # TODO schema
@@ -642,9 +626,6 @@ async def _addAhaServer(self, server):
 
         self.slab.put(lkey, s_msgpack.en(server), db='aha:servers')
 
-        if self.isactive:
-            await self._fireTopoUpdate()
-
         return True
 
     @s_nexus.Pusher.onPushAuto('aha:server:del')
@@ -656,9 +637,6 @@ async def delAhaServer(self, host, port):
         if byts is None:
             return None
 
-        if self.isactive:
-            await self._fireTopoUpdate()
-
         return s_msgpack.un(byts)
 
     async def getAhaServers(self):
@@ -667,25 +645,6 @@ async def getAhaServers(self):
             servers.append(s_msgpack.un(byts))
         return servers
 
-    async def iterAhaTopo(self):
-
-        if not self.isactive:
-            async with await self.nexsroot.client.proxy() as proxy:
-                async for item in proxy.iterAhaTopo():
-                    yield item
-
-        async with await s_queue.Window.anit(maxsize=1000) as wind:
-
-            async def onfini():
-                self.topowindows.remove(wind)
-
-            wind.onfini(onfini)
-
-            async with self.nexsroot.applylock:
-                update = await self._getTopoUpdate()
-                await wind.put(update)
-                self.topowindows.append(wind)
-
     async def iterPoolTopo(self, name):
 
         name = self._getAhaName(name)
@@ -734,16 +693,16 @@ async def initServiceRuntime(self):
                 await self.genCaCert(netw)
 
             name = self.conf.get('aha:name')
-
-            host = f'{name}.{netw}'
-            if self.certdir.getHostCertPath(host) is None:
-                logger.info(f'Adding server certificate for {host}')
-                await self._genHostCert(host, signas=netw)
+            if name is not None:
+                host = f'{name}.{netw}'
+                if self.certdir.getHostCertPath(host) is None:
+                    logger.info(f'Adding server certificate for {host}')
+                    await self._genHostCert(host, signas=netw)
 
             root = f'root@{netw}'
             await self._genUserCert(root, signas=netw)
 
-            user = self._getAhaAdmin()
+            user = self.conf.get('aha:admin')
             if user is not None:
                 await self._genUserCert(user, signas=netw)
 
@@ -1309,14 +1268,14 @@ async def addAhaClone(self, host, port=27492, conf=None):
             'port': port,
             'conf': conf,
         }
-        return await self._push('aha:clone:add', clone)
+        await self._push('aha:clone:add', clone)
+        return self._getProvClientUrl(iden)
 
     @s_nexus.Pusher.onPush('aha:clone:add')
     async def _addAhaClone(self, clone):
         iden = clone.get('iden')
         lkey = s_common.uhex(iden)
         self.slab.put(lkey, s_msgpack.en(clone), db='aha:clones')
-        return self._getProvClientUrl(iden)
 
     async def addAhaSvcProv(self, name, provinfo=None):
 
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 576b018c804..a1e6c25ea13 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -961,7 +961,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
             'type': 'string',
         },
         'aha:network': {
-            'description': 'The AHA service network. Defaults to "synapse".',
+            'description': 'The AHA service network.',
             'type': 'string',
         },
         'aha:registry': {
@@ -1104,6 +1104,8 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         self._checkspace = s_coro.Event()
         self._reloadfuncs = {}  # name -> func
 
+        self.nexslock = asyncio.Lock()
+
         self.conf = self._initCellConf(conf)
 
         self.minfree = self.conf.get('limit:disk:free')
@@ -1127,7 +1129,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         self.ahasvcname = None
         ahaname = self.conf.get('aha:name')
         ahanetw = self._getAhaNetwork()
-        if ahaname is not None:
+        if ahaname is not None and ahanetw is not None:
             self.ahasvcname = f'{ahaname}.{ahanetw}'
 
         # each cell has a guid
@@ -1176,20 +1178,20 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         if self.conf.get('mirror') and not self.conf.get('nexslog:en'):
             self.modCellConf({'nexslog:en': True})
 
-        # construct our nexsroot instance ( but do not start it )
         await s_nexus.Pusher.__anit__(self, self.iden)
 
         self._initCertDir()
 
-        root = await self._ctorNexsRoot()
+        await self.enter_context(s_telepath.loadTeleCell(self.dirn))
 
-        # mutually assured destruction with our nexs root
-        self.onfini(root.fini)
-        root.onfini(self.fini)
+        await self._initCellSlab(readonly=readonly)
 
-        self.setNexsRoot(root)
+        await self.initServiceEarly()
+
+        root = await self._ctorNexsRoot()
+
+        await self.setNexsRoot(root)
 
-        await self._initCellSlab(readonly=readonly)
         self.apikeydb = self.slab.initdb('user:apikeys')  # apikey -> useriden
         self.usermetadb = self.slab.initdb('user:meta')  # useriden + <valu> -> dict valu
         self.rolemetadb = self.slab.initdb('role:meta')  # roleiden + <valu> -> dict valu
@@ -1434,10 +1436,10 @@ def checkFreeSpace(self):
 
     async def _runFreeSpaceLoop(self):
 
-        nexsroot = self.getCellNexsRoot()
-
         while not self.isfini:
 
+            nexsroot = self.getCellNexsRoot()
+
             self._checkspace.clear()
 
             disk = shutil.disk_usage(self.dirn)
@@ -1484,11 +1486,6 @@ async def _runSysctlLoop(self):
 
             await self.waitfini(self.SYSCTL_CHECK_FREQ)
 
-    def _getAhaAdmin(self):
-        name = self.conf.get('aha:admin')
-        if name is not None:
-            return name
-
     async def _initAhaRegistry(self):
 
         # NOTE: This API needs to be safe to call again
@@ -1507,17 +1504,12 @@ async def fini():
 
             self.ahaclient.onfini(fini)
 
-        ahauser = self.conf.get('aha:user')
-
-        ahaadmin = self._getAhaAdmin()
+        ahaadmin = self.conf.get('aha:admin')
         if ahaadmin is not None:
             await self._addAdminUser(ahaadmin)
 
-        if ahauser is not None:
-            await self._addAdminUser(ahauser)
-
     def _getAhaNetwork(self):
-        return self.conf.get('aha:network', 'synapse')
+        return self.conf.get('aha:network')
 
     def _getDmonListen(self):
 
@@ -1545,6 +1537,9 @@ async def _addAdminUser(self, username):
         if user.isLocked():
             await user.setLocked(False, logged=False)
 
+    async def initServiceEarly(self):
+        pass
+
     async def initServiceStorage(self):
         pass
 
@@ -1578,8 +1573,8 @@ async def initServiceNetwork(self):
 
         turl = self._getDmonListen()
         if turl is not None:
-            self.sockaddr = await self.dmon.listen(turl)
             logger.info(f'dmon listening: {turl}')
+            self.sockaddr = await self.dmon.listen(turl)
 
         await self._initAhaService()
 
@@ -1641,13 +1636,16 @@ async def _initAhaService(self):
         if ahaname is None:
             return
 
-        ahalead = self.conf.get('aha:leader')
         ahanetw = self._getAhaNetwork()
+        if ahanetw is None:
+            return
 
         ahainfo = await self.getAhaInfo()
         if ahainfo is None:
             return
 
+        ahalead = self.conf.get('aha:leader')
+
         self.ahasvcname = f'{ahaname}.{ahanetw}'
 
         async def _runAhaRegLoop():
@@ -1804,11 +1802,11 @@ async def handoff(self, turl, timeout=30):
                 mesg = 'Cannot handoff mirror leadership to myself!'
                 raise s_exc.BadArg(mesg=mesg)
 
-            logger.debug(f'HANDOFF: Obtaining nexus applylock ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Obtaining nexus lock ahaname={ahaname}')
 
-            async with self.nexsroot.applylock:
+            async with self.nexslock:
 
-                logger.debug(f'HANDOFF: Obtained nexus applylock ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Obtained nexus lock ahaname={ahaname}')
                 indx = await self.getNexsIndx()
 
                 logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}')
@@ -1829,7 +1827,7 @@ async def handoff(self, turl, timeout=30):
                 logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}')
                 await self.nexsroot.startup()
 
-            logger.debug(f'HANDOFF: Released nexus applylock ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Released nexus lock ahaname={ahaname}')
         logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}')
 
     async def reqAhaProxy(self, timeout=None):
@@ -2129,7 +2127,7 @@ async def _execBackupTask(self, dirn):
 
         try:
 
-            async with self.nexsroot.applylock:
+            async with self.nexslock:
 
                 logger.debug('Syncing LMDB Slabs')
 
@@ -2983,6 +2981,12 @@ async def _initCellHive(self):
 
         return hive
 
+    async def _initSlabFile(self, path, readonly=False):
+        slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
+        slab.addResizeCallback(self.checkFreeSpace)
+        self.onfini(slab)
+        return slab
+
     async def _initCellSlab(self, readonly=False):
 
         s_common.gendir(self.dirn, 'slabs')
@@ -2994,10 +2998,7 @@ async def _initCellSlab(self, readonly=False):
             _slab.initdb('hive')
             await _slab.fini()
 
-        self.slab = await s_lmdbslab.Slab.anit(path, map_size=SLAB_MAP_SIZE, readonly=readonly)
-        self.slab.addResizeCallback(self.checkFreeSpace)
-
-        self.onfini(self.slab.fini)
+        self.slab = await self._initSlabFile(path)
 
     async def _initCellAuth(self):
 
diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py
index 657979c1173..fd9af2bfddd 100644
--- a/synapse/lib/nexus.py
+++ b/synapse/lib/nexus.py
@@ -92,7 +92,6 @@ async def __anit__(self, cell):
         self.writeholds = set()
 
         self.applytask = None
-        self.applylock = asyncio.Lock()
 
         self.ready = asyncio.Event()
         self.donexslog = self.cell.conf.get('nexslog:en')
@@ -231,7 +230,7 @@ def _getResponseFuture(self, iden=None):
 
     async def enNexsLog(self):
 
-        async with self.applylock:
+        async with self.cell.nexslock:
 
             if self.donexslog:
                 return
@@ -345,7 +344,7 @@ async def eat(self, nexsiden, event, args, kwargs, meta):
         if meta is None:
             meta = {}
 
-        async with self.applylock:
+        async with self.cell.nexslock:
             self.reqNotReadOnly()
             # Keep a reference to the shielded task to ensure it isn't GC'd
             self.applytask = asyncio.create_task(self._eat((nexsiden, event, args, kwargs, meta)))
@@ -616,12 +615,12 @@ async def __anit__(self, iden: str, nexsroot: NexsRoot = None):  # type: ignore
         self.nexsroot: Optional[NexsRoot] = None
 
         if nexsroot is not None:
-            self.setNexsRoot(nexsroot)
+            await self.setNexsRoot(nexsroot)
 
         for event, func, passitem in self._regclstupls:  # type: ignore
             self._nexshands[event] = func, passitem
 
-    def setNexsRoot(self, nexsroot):
+    async def setNexsRoot(self, nexsroot):
 
         nexsroot._nexskids[self.nexsiden] = self
 
@@ -629,10 +628,22 @@ def onfini():
             prev = nexsroot._nexskids.pop(self.nexsiden, None)
             assert prev is not None, f'Failed removing {self.nexsiden}'
 
-        self.onfini(onfini)
+        nexsroot.onfini(onfini)
+        self.onfini(nexsroot)
 
         self.nexsroot = nexsroot
 
+    async def modNexsRoot(self, ctor):
+
+        if self.nexsroot is not None:
+            await self.nexsroot.fini()
+
+        nexsroot = await ctor()
+
+        await self.setNexsRoot(nexsroot)
+
+        await self.nexsroot.startup()
+
     @classmethod
     def onPush(cls, event: str, passitem=False) -> Callable:
         '''
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index 0e10fb7bf65..d6f7809adc6 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -166,6 +166,7 @@ async def test_lib_aha(self):
             conf = {
                 'aha:name': '0.cryo',
                 'aha:leader': 'cryo',
+                'aha:network': 'synapse',
                 'aha:admin': 'root@synapse',
                 'aha:registry': [f'tcp://root:hehehaha@127.0.0.1:{port}',
                                  f'tcp://root:hehehaha@127.0.0.1:{port}'],
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index 86fdce2a9c8..15a1aab1b8c 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -1445,11 +1445,14 @@ def mayTestDir(self, dirn):
             yield dirn
 
     @contextlib.asynccontextmanager
-    async def getTestAha(self, conf=None, dirn=None):
+    async def getTestAha(self, conf=None, dirn=None, ctor=None):
 
         if conf is None:
             conf = {}
 
+        if ctor is None:
+            ctor = s_aha.AhaCell.anit
+
         conf = s_msgpack.deepcopy(conf)
 
         hasdmonlisn = 'dmon:listen' in conf
@@ -1465,9 +1468,10 @@ async def getTestAha(self, conf=None, dirn=None):
         conf.setdefault('health:sysctl:checks', False)
 
         with self.withNexusReplay():
+
             with self.mayTestDir(dirn) as dirn:
 
-                async with await s_aha.AhaCell.anit(dirn, conf=conf) as aha:
+                async with await ctor(dirn, conf=conf) as aha:
 
                     mods = {}
                     if not hasdmonlisn and hostname:

From f1d1cad16d26d544d0eee47bb49a33d0ea1fdebc Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 1 Jul 2024 16:55:46 -0400
Subject: [PATCH 08/68] wip

---
 synapse/lib/nexus.py           | 3 +--
 synapse/tests/test_lib_cell.py | 4 ++--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py
index fd9af2bfddd..2f00e49c343 100644
--- a/synapse/lib/nexus.py
+++ b/synapse/lib/nexus.py
@@ -514,8 +514,7 @@ async def runMirrorLoop(self, proxy):
         if self.celliden is not None:
             if self.celliden != await proxy.getCellIden():
                 logger.error('remote cell has different iden!  Aborting mirror sync')
-                await proxy.fini()
-                await self.fini()
+                await self.cell.fini()
                 return
 
         while not proxy.isfini:
diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py
index c7a942dd57e..f9778dbeb53 100644
--- a/synapse/tests/test_lib_cell.py
+++ b/synapse/tests/test_lib_cell.py
@@ -520,7 +520,7 @@ async def test_cell_getinfo(self):
                 self.isin('cortex:defaults', cnfo.get('cellvers', {}))
 
                 # Defaults aha data is
-                self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': 'synapse'})
+                self.eq(cnfo.get('aha'), {'name': None, 'leader': None, 'network': None})
 
                 # Synapse information
                 self.eq(snfo.get('version'), s_version.version)
@@ -1710,7 +1710,7 @@ async def test_mirror_badiden(self):
                                                'has different iden') as stream:
                     async with self.getTestCell(s_cell.Cell, dirn=path01, conf=conf01) as cell01:
                         await stream.wait(timeout=2)
-                        self.true(await cell01.waitfini(6))
+                        self.true(await cell01.nexsroot.waitfini(6))
 
     async def test_backup_restore_base(self):
 

From 9c18de4a5783808dd18994b75a02d0e5a19efe97 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 08:20:08 -0400
Subject: [PATCH 09/68] wip

---
 synapse/lib/aha.py                    | 39 ++++++++++++++-------------
 synapse/lib/cell.py                   | 33 ++++++++++++++---------
 synapse/lib/config.py                 | 26 +++++-------------
 synapse/tests/files/rstorm/testsvc.py |  2 +-
 synapse/tests/test_lib_aha.py         |  2 +-
 synapse/tests/test_lib_cell.py        | 12 ++++-----
 synapse/tests/test_lib_config.py      |  7 +++--
 synapse/tests/utils.py                |  3 ++-
 8 files changed, 60 insertions(+), 64 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index d68160acdb6..66264e8f0b1 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -269,7 +269,6 @@ async def signHostCsr(self, csrtext, signas=None, sans=None):
         return await self.cell.signHostCsr(csrtext, signas=signas, sans=sans)
 
     async def signUserCsr(self, csrtext, signas=None):
-
         await self._reqUserAllowed(('aha', 'csr', 'user'))
         return await self.cell.signUserCsr(csrtext, signas=signas)
 
@@ -428,17 +427,17 @@ async def getUserInfo(self):
         return {
             'aha:urls': await self.aha.getAhaUrls(user=user),
             'aha:user': user,
-            'aha:network': self.aha._getAhaNetwork(),
+            'aha:network': self.aha.conf.req('aha:network'),
         }
 
     async def getCaCert(self):
-        ahanetw = self.aha._getAhaNetwork()
+        ahanetw = self.aha.conf.req('aha:network')
         return self.aha.certdir.getCaCertBytes(ahanetw)
 
     async def signUserCsr(self, byts):
 
         ahauser = self.userinfo.get('name')
-        ahanetw = self.aha._getAhaNetwork()
+        ahanetw = self.aha.conf.req('aha:network')
 
         username = f'{ahauser}@{ahanetw}'
 
@@ -464,7 +463,7 @@ async def getProvInfo(self):
         return self.provinfo
 
     async def getCaCert(self):
-        ahanetw = self.aha._getAhaNetwork()
+        ahanetw = self.aha.conf.req('aha:network')
         return self.aha.certdir.getCaCertBytes(ahanetw)
 
     async def signHostCsr(self, byts):
@@ -686,7 +685,7 @@ async def initServiceRuntime(self):
         if self.isactive:
 
             # bootstrap a CA for our aha:network
-            netw = self._getAhaNetwork()
+            netw = self.conf.req('aha:network')
 
             if self.certdir.getCaCertPath(netw) is None:
                 logger.info(f'Adding CA certificate for {netw}')
@@ -714,9 +713,10 @@ def _getDnsName(self):
         if hostname is not None:
             return hostname
 
-        name = self.conf.get('aha:name')
-        if name is not None:
-            return f'{name}.{self._getAhaNetwork()}'
+        ahaname = self.conf.get('aha:name')
+        ahanetw = self.conf.get('aha:network')
+        if ahaname is not None and ahanetw is not None:
+            return f'{ahaname}.{ahanetw}'
 
     def _getProvListen(self):
 
@@ -737,7 +737,7 @@ def _getDmonListen(self):
         if lisn is not s_common.novalu:
             return lisn
 
-        network = self._getAhaNetwork()
+        network = self.conf.req('aha:network')
         dnsname = self._getDnsName()
         if dnsname is not None:
             return f'ssl://0.0.0.0?hostname={dnsname}&ca={network}'
@@ -753,9 +753,8 @@ def _reqProvListen(self):
     async def initServiceNetwork(self):
 
         # bootstrap CA/host certs first
-        network = self._getAhaNetwork()
-        if network is not None:
-            await self._genCaCert(network)
+        network = self.conf.req('aha:network')
+        await self._genCaCert(network)
 
         hostname = self._getDnsName()
         if hostname is not None and network is not None:
@@ -862,7 +861,7 @@ async def addAhaSvc(self, name, info, network=None):
     def _getAhaName(self, name):
         # the modern version of names is absolute or ...
         if name.endswith('...'):
-            return name[:-2] + self._getAhaNetwork()
+            return name[:-2] + self.conf.req('aha:network')
         return name
 
     async def getAhaPool(self, name):
@@ -1228,18 +1227,20 @@ async def getAhaUrls(self, user='root'):
         if urls is not None:
             return urls
 
-        network = self._getAhaNetwork()
+        network = self.conf.req('aha:network')
+
         urls = []
         for server in await self.getAhaServers():
             host = server.get('host')
             port = server.get('port')
             urls.append(f'ssl://{host}:{port}?certname={user}@{network}')
+
         return urls
 
     def _getAhaUrl(self, user='root'):
         port = self.sockaddr[1]
         host = self._getDnsName()
-        network = self._getAhaNetwork()
+        network = self.conf.req('aha:network')
         return f'ssl://{host}:{port}?certname={user}@{network}'
 
     async def getAhaClone(self, iden):
@@ -1253,7 +1254,7 @@ async def addAhaClone(self, host, port=27492, conf=None):
         if conf is None:
             conf = {}
 
-        network = self._getAhaNetwork()
+        network = self.conf.req('aha:network')
 
         conf['mirror'] = self._getAhaUrl()
 
@@ -1293,7 +1294,7 @@ async def addAhaSvcProv(self, name, provinfo=None):
 
         conf = provinfo.setdefault('conf', {})
 
-        netw = self._getAhaNetwork()
+        netw = self.conf.req('aha:network')
 
         ahaadmin = self.conf.get('aha:admin')
         if ahaadmin is not None: # pragma: no cover
@@ -1418,7 +1419,7 @@ async def addAhaUserEnroll(self, name, userinfo=None, again=False):
             raise s_exc.BadArg(mesg='Empty name values are not allowed for provisioning.')
 
         provurl = self._reqProvListen()
-        ahanetw = self._getAhaNetwork()
+        ahanetw = self.conf.req('aha:network')
 
         username = f'{name}@{ahanetw}'
 
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index a1e6c25ea13..c06497ffe81 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1128,7 +1128,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         # we need to know this pretty early...
         self.ahasvcname = None
         ahaname = self.conf.get('aha:name')
-        ahanetw = self._getAhaNetwork()
+        ahanetw = self.conf.get('aha:network')
         if ahaname is not None and ahanetw is not None:
             self.ahasvcname = f'{ahaname}.{ahanetw}'
 
@@ -1508,8 +1508,14 @@ async def fini():
         if ahaadmin is not None:
             await self._addAdminUser(ahaadmin)
 
-    def _getAhaNetwork(self):
-        return self.conf.get('aha:network')
+    def reqAhaUser(self):
+
+        ahauser = self.conf.get('aha:user')
+        if ahauser is not None:
+            return ahauser
+
+        mesg = 'An aha:user configuration option is required.'
+        raise s_exc.NeedConfValu(mesg=mesg)
 
     def _getDmonListen(self):
 
@@ -1517,10 +1523,11 @@ def _getDmonListen(self):
         if lisn is not s_common.novalu:
             return lisn
 
-        network = self._getAhaNetwork()
         ahaname = self.conf.get('aha:name')
-        if ahaname is not None:
-            return f'ssl://0.0.0.0:0?hostname={hostname}&ca={network}'
+        ahanetw = self.conf.get('aha:network')
+        if ahaname is not None and ahanetw is not None:
+            hostname = f'{ahaname}.{ahanetw}'
+            return f'ssl://0.0.0.0:0?hostname={hostname}&ca={ahanetw}'
 
     async def _addAdminUser(self, username):
         # add the user in a pre-nexus compatible way
@@ -1636,7 +1643,7 @@ async def _initAhaService(self):
         if ahaname is None:
             return
 
-        ahanetw = self._getAhaNetwork()
+        ahanetw = self.conf.get('aha:network')
         if ahanetw is None:
             return
 
@@ -1763,7 +1770,7 @@ async def promote(self, graceful=False):
                 mesg = 'Cannot gracefully promote without aha:name configured.'
                 raise s_exc.BadArg(mesg=mesg)
 
-            ahanetw = self._getAhaNetwork()
+            ahanetw = self.conf.req('aha:network')
 
             myurl = f'aha://{ahaname}.{ahanetw}'
             logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}')
@@ -1861,7 +1868,7 @@ async def _setAhaActive(self):
             await proxy.fini()
             return
 
-        ahanetw = self._getAhaNetwork()
+        ahanetw = self.conf.req('aha:network')
         try:
             await proxy.addAhaSvc(ahalead, ahainfo, network=ahanetw)
         except asyncio.CancelledError:  # pragma: no cover
@@ -2819,7 +2826,7 @@ async def addHttpsPort(self, port, host='0.0.0.0', sslctx=None):
             sslctx = self.initSslCtx(certpath, pkeypath)
 
         kwargs = {
-            'xheaders': self.conf.reqConfValu('https:parse:proxy:remoteip')
+            'xheaders': self.conf.req('https:parse:proxy:remoteip')
         }
         serv = self.wapp.listen(port, address=addr, ssl_options=sslctx, **kwargs)
         self.httpds.append(serv)
@@ -3732,7 +3739,7 @@ async def _bootCellMirror(self, pnfo):
         # this function must assume almost nothing is initialized
         # but that's ok since it will only run rarely.
         # It assumes it has a tuple of (provisioning configuration, provisioning iden) available
-        murl = self.conf.reqConfValu('mirror')
+        murl = self.conf.req('mirror')
         provconf, providen = pnfo
 
         logger.warning(f'Bootstrap mirror from: {murl} (this could take a while!)')
@@ -3896,7 +3903,7 @@ async def _getCellUser(self, link, mesg):
 
             if username.find('@') != -1:
                 userpart, hostpart = username.split('@', 1)
-                if hostpart == self._getAhaNetwork():
+                if hostpart == self.conf.get('aha:network'):
                     user = await self.auth.getUserByName(userpart)
                     if user is not None:
                         if user.isLocked():
@@ -4090,7 +4097,7 @@ async def getCellInfo(self):
                 'aha': {
                     'name': self.conf.get('aha:name'),
                     'leader': self.conf.get('aha:leader'),
-                    'network': self._getAhaNetwork(),
+                    'network': self.conf.get('aha:network'),
                 }
             },
             'features': {
diff --git a/synapse/lib/config.py b/synapse/lib/config.py
index 4e6b18dcd4b..5295a4bac7a 100644
--- a/synapse/lib/config.py
+++ b/synapse/lib/config.py
@@ -392,28 +392,16 @@ def reqConfValid(self):
         else:
             return
 
-    def reqConfValu(self, key):
+    def req(self, name):
         '''
-        Get a configuration value.  If that value is not present in the schema
-        or is not set, then raise an exception.
-
-        Args:
-            key (str): The key to require.
-
-        Returns:
-            The requested value.
+        Return a configuration value or raise NeedConfValu if it is unset.
         '''
-        # Ensure that the key is in self.json_schema
-        if key not in self.json_schema.get('properties', {}):
-            raise s_exc.BadArg(mesg='Required key is not present in the configuration schema.',
-                               key=key)
-
-        # Ensure that the key is present in self.conf
-        if key not in self.conf:
-            raise s_exc.NeedConfValu(mesg='Required key is not present in configuration data.',
-                                     key=key)
+        valu = self.conf.get(name, s_common.novalu)
+        if valu is not s_common.novalu:
+            return valu
 
-        return self.conf.get(key)
+        mesg = f'The {name} configuration option is required.'
+        raise s_exc.NeedConfValu(mesg=mesg, name=name)
 
     def reqKeyValid(self, key, value):
         '''
diff --git a/synapse/tests/files/rstorm/testsvc.py b/synapse/tests/files/rstorm/testsvc.py
index 55f4822d946..678fac85bee 100644
--- a/synapse/tests/files/rstorm/testsvc.py
+++ b/synapse/tests/files/rstorm/testsvc.py
@@ -37,7 +37,7 @@ class Testsvc(s_cell.Cell):
 
     async def __anit__(self, dirn, conf):
         await s_cell.Cell.__anit__(self, dirn, conf=conf)
-        self.secret = self.conf.reqConfValu('secret')
+        self.secret = self.conf.req('secret')
 
     async def test(self):
         return self.secret
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index d6f7809adc6..ffc716e0d4f 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -951,7 +951,7 @@ async def test_aha_restart(self):
                 svc1dirn = s_common.gendir(dirn, 'svc01')
                 async with await s_base.Base.anit() as cm:
 
-                    aconf = {'dns:name': 'aha.loop.vertex.link'}
+                    aconf = {'dns:name': 'aha.loop.vertex.link', 'aha:network': 'synapse'}
 
                     aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf)
                     await cm.enter_context(aha)
diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py
index f9778dbeb53..a202ec1dab0 100644
--- a/synapse/tests/test_lib_cell.py
+++ b/synapse/tests/test_lib_cell.py
@@ -925,11 +925,11 @@ async def test_cell_initargv_conf(self):
                         # 1) cmdline
                         # 2) envars
                         # 3) cell.yaml
-                        self.true(cell.conf.reqConfValu('nexslog:en'))
-                        self.true(cell.conf.reqConfValu('nexslog:async'))
-                        self.none(cell.conf.reqConfValu('dmon:listen'))
-                        self.none(cell.conf.reqConfValu('https:port'))
-                        self.eq(cell.conf.reqConfValu('aha:name'), 'some:cell')
+                        self.true(cell.conf.req('nexslog:en'))
+                        self.true(cell.conf.req('nexslog:async'))
+                        self.none(cell.conf.req('dmon:listen'))
+                        self.none(cell.conf.req('https:port'))
+                        self.eq(cell.conf.req('aha:name'), 'some:cell')
                         root = cell.auth.rootuser
                         self.true(await root.tryPasswd('secret'))
 
@@ -937,7 +937,7 @@ async def test_cell_initargv_conf(self):
                 with self.getTestDir() as dirn:
                     s_common.yamlsave({'nexslog:en': False}, dirn, 'cell.mods.yaml')
                     async with await s_cell.Cell.initFromArgv([dirn]) as cell:
-                        self.false(cell.conf.reqConfValu('nexslog:en'))
+                        self.false(cell.conf.req('nexslog:en'))
                         # We can remove the valu from the overrides file with the pop API
                         # This is NOT reactive API which causes the whole behavior
                         # of the cell to suddenly change. This is intended to be used with
diff --git a/synapse/tests/test_lib_config.py b/synapse/tests/test_lib_config.py
index ec3fb0080c6..cb61a6ae3d4 100644
--- a/synapse/tests/test_lib_config.py
+++ b/synapse/tests/test_lib_config.py
@@ -31,7 +31,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, *args, **kwargs):
         await s_cell.Cell.__anit__(self, dirn, conf, readonly, *args, **kwargs)
         # This captures a design pattern that reduces boilerplate
         # code used by Cell implementators.
-        self.conf.reqConfValu('apikey')
+        self.conf.req('apikey')
 
 
 class ConfTest(s_test.SynTest):
@@ -162,10 +162,9 @@ async def test_config_basics(self):
         })
 
         # We can ensure that certain vars are loaded
-        self.eq('Funky string time!', conf.reqConfValu('key:string'))
+        self.eq('Funky string time!', conf.req('key:string'))
         # And throw if they are not, or if the requested key isn't even schema valid
-        self.raises(s_exc.NeedConfValu, conf.reqConfValu, 'key:bool:nodefval')
-        self.raises(s_exc.BadArg, conf.reqConfValu, 'key:newp')
+        self.raises(s_exc.NeedConfValu, conf.req, 'key:bool:nodefval')
 
         # Since we're an Mutable mapping, we have some dict methods available to us
         self.len(8, conf)  # __len__
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index 15a1aab1b8c..f1487db931d 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -1512,7 +1512,8 @@ async def addSvcToAha(self, aha, svcname, ctor,
             The config data for the cell is pushed into dirn/cell.yaml.
             The cells are created with the ``ctor.anit()`` function.
         '''
-        svcfull = f'{svcname}.{aha._getAhaNetwork()}'
+        ahanetw = aha.conf.req('aha:network')
+        svcfull = f'{svcname}.{ahanetw}'
         onetime = await aha.addAhaSvcProv(svcname, provinfo=provinfo)
 
         conf = self.getCellConf(conf=conf)

From a1664e53088cdc0705b6656814fb9034c24e579e Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 10:27:32 -0400
Subject: [PATCH 10/68] wip

---
 docs/synapse/deploymentguide.rst | 120 +++++++++++++++++++++----------
 synapse/tests/test_tools_aha.py  |   5 ++
 synapse/tools/aha/clone.py       |  42 +++++++++++
 3 files changed, 128 insertions(+), 39 deletions(-)
 create mode 100644 synapse/tools/aha/clone.py

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 2b84671c5d1..106bf0f03ff 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -65,19 +65,6 @@ When using AHA, you may run any of the **other** services on additional hosts as
 directly to the AHA service.  You may also shutdown a service, move it's volume to a different host, and
 start it backup without changing anything.
 
-Decide on a Name
-================
-
-Throughout the examples, we will be using ``<yournetwork>`` as the AHA network name which is also used as the
-common-name (CN) for the CA certificate. This should be changed to an appropriate network name used by your
-synapse deployment such as ``syn.acmecorp.com``. We will use ``<yournetwork>`` in the following configs to
-specify locations which should be replaced with your selected AHA network name. For a **test** deployment which
-runs **all** docker containers on one host, you may use ``loop.vertex.link``.
-
-.. note::
-    It is important that you choose a name and stick with it for a given deployment. Once we begin generating
-    host and service account certificates, changing this name will be difficult.
-
 Deploy AHA Service
 ==================
 
@@ -87,18 +74,43 @@ by name, so it is likely that you need to create a DNS A/AAAA record in your exi
 using AHA, the only host that needs DNS or other external name resolution is the AHA service.
 
 .. note::
-    It is important to ensure that ``aha.<yournetwork>`` is resolvable via DNS or docker container service
-    name resolution from within the container environment! There are configuration options you may use if
-    this is impossible, but the configuration is far simpler if we can make this assumption.
+    FIXME AHA FLAT NETWORK STUFF
+
+Choose an AHA Network Name
+--------------------------
+
+Your AHA network name is separate from DNS and is used by services within the AHA service deployment. It is
+generally a good idea to chose a name that aligns with the use case for the synapse deployment. For example,
+if you plan to have a test/dev/prod deployment, choosing a name like ``prod.synapse`` will make it clear which
+deployment is which. Changing this name later is difficult, so choose carefully! Throughout the examples, we
+will be using ``prod.synapse`` as the AHA network name which is also used as the common-name (CN) for the CA
+certificate.
+
+Configure an AHA DNS Name
+-------------------------
+
+When choosing the DNS name for your AHA server, it is important to keep in mind that you may eventually want to
+deploy AHA mirrors for a high-availability deployment. Choosing a name like ``00.aha.<dns-network>``,
+where ``<dns-network>`` is a DNS zone you control, will allow you to deploy mirrors with a consistent naming
+convention in the future. Once you have selected the DNS name for your AHA server, you will need to ensure that
+all deployed Synapse services will be able to resolve that name.
+
+.. note::
+    It is important to ensure that ``00.aha.<dns-network>`` is resolvable via DNS or docker container service
+    name resolution from within the container environment!
+
+
+Deploy the AHA Container
+------------------------
 
 Create the container directory::
 
-    mkdir -p /srv/syn/aha/storage
+    mkdir -p /srv/syn/00.aha/storage
 
-Create the ``/srv/syn/aha/docker-compose.yaml`` file with contents::
+Create the ``/srv/syn/00.aha/docker-compose.yaml`` file with contents::
 
     services:
-      aha:
+      00.aha:
         user: "999"
         image: vertexproject/synapse-aha:v2.x.x
         network_mode: host
@@ -106,22 +118,23 @@ Create the ``/srv/syn/aha/docker-compose.yaml`` file with contents::
         volumes:
             - ./storage:/vertex/storage
         environment:
+            # disable HTTPS API for now to prevent port collisions
             - SYN_AHA_HTTPS_PORT=null
-            - SYN_AHA_AHA_NAME=aha
-            - SYN_AHA_DNS_NAME=aha.<yuournetwork>
+            - SYN_AHA_AHA_NETWORK=prod.synapse
+            - SYN_AHA_DNS_NAME=00.aha.<yuournetwork>
 
 .. note::
 
-    Don't forget to replace ``<yournetwork>`` with your chosen network name!
+    Don't forget to replace ``<dns-network>`` with your configured DNS suffix.
 
 Change ownership of the storage directory to the user you will use to run the container::
 
-    chown -R 999 /srv/syn/aha/storage
+    chown -R 999 /srv/syn/00.aha/storage
 
 Start the container using ``docker compose``::
 
-    docker compose --file /srv/syn/aha/docker-compose.yaml pull
-    docker compose --file /srv/syn/aha/docker-compose.yaml up -d
+    docker compose --file /srv/syn/00.aha/docker-compose.yaml pull
+    docker compose --file /srv/syn/00.aha/docker-compose.yaml up -d
 
 To view the container logs at any time you may run the following command on the *host* from the
 ``/srv/syn/aha`` directory::
@@ -131,11 +144,40 @@ To view the container logs at any time you may run the following command on the
 You may also execute a shell inside the container using ``docker compose`` from the ``/srv/syn/aha``
 directory on the *host*. This will be necessary for some of the additional provisioning steps::
 
-    docker compose exec aha /bin/bash
-
+    docker compose exec 00.aha /bin/bash
 
 .. _deploy_axon:
 
+Deploy AHA Mirrors (optional)
+=============================
+
+For high-availability deployments, you will want to deploy an AHA mirror or two. Typical Synapse service
+mirroring is configured using AHA based provisioning. Bootstrapping AHA mirrors is simple, but requires
+a slightly different procedure because you cannot bootstrap AHA via AHA :)
+
+For this example, we will assume you chose a DNS name for your primary AHA server similar to the steps
+listed above. If so, you can simply replace ``00`` with sequential numbers and repeat this step to deploy
+however many AHA mirrors you deem appropriate.
+
+NOTE: AHA uses two default ports ETC. The following steps assume you will be running each of your AHA servers
+on a different host. The use of ``network_mode; host`` ETC
+
+Generate a one-time use provisioning URL::
+
+    python -m synapse.tools.aha.clone 01.aha.<dns-network>
+
+You should see output that looks similar to this::
+
+    one-time use URL: ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
+
+Create the container directory::
+
+    mkdir -p /srv/syn/01.aha/storage
+    chown -R 999 /srv/syn/01.aha/storage
+
+::NOTE You can deploy AHA mirrors at any time in the future. Once the mirrors are deployed, each AHA enabled
+       Synapse service will retrieve the updated list of AHA servers the next time it is restarted.
+
 Deploy Axon Service
 ===================
 
@@ -161,11 +203,11 @@ to re-use the one-time use URL!
     We strongly encourage you to use a numbered hierarchical naming convention for services where the
     first part of the name is a 0 padded number and the second part is the service type. The above example
     ``00.axon`` will allow you to deploy mirror instances in the future, such as ``01.axon``, where the AHA
-    name ``axon.<yournetwork>`` will automatically resolve to which ever one is the current leader.
+    name ``axon.prod.synapse`` will automatically resolve to which ever one is the current leader.
 
 You should see output that looks similar to this::
 
-    one-time use URL: ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+    one-time use URL: ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 **On the Host**
 
@@ -187,7 +229,7 @@ Create the ``/srv/syn/00.axon/docker-compose.yaml`` file with contents::
         environment:
             # disable HTTPS API for now to prevent port collisions
             - SYN_AXON_HTTPS_PORT=null
-            - SYN_AXON_AHA_PROVISION=ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+            - SYN_AXON_AHA_PROVISION=ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 .. note::
 
@@ -209,7 +251,7 @@ Generate a one-time use provisioning URL::
 
 You should see output that looks similar to this::
 
-    one-time use URL: ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+    one-time use URL: ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 **On the Host**
 
@@ -231,7 +273,7 @@ Create the ``/srv/syn/00.jsonstor/docker-compose.yaml`` file with contents::
         environment:
             # disable HTTPS API for now to prevent port collisions
             - SYN_JSONSTOR_HTTPS_PORT=null
-            - SYN_JSONSTOR_AHA_PROVISION=ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+            - SYN_JSONSTOR_AHA_PROVISION=ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 .. note::
 
@@ -253,7 +295,7 @@ Generate a one-time use provisioning URL::
 
 You should see output that looks similar to this::
 
-    one-time use URL: ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+    one-time use URL: ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 **On the Host**
 
@@ -275,7 +317,7 @@ Create the ``/srv/syn/00.cortex/docker-compose.yaml`` file with contents::
         environment:
             - SYN_CORTEX_AXON=aha://axon...
             - SYN_CORTEX_JSONSTOR=aha://jsonstor...
-            - SYN_CORTEX_AHA_PROVISION=ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+            - SYN_CORTEX_AHA_PROVISION=ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 .. note::
 
@@ -309,7 +351,7 @@ Generate a one-time use URL for provisioning from *inside the AHA container*::
 
 You should see output that looks similar to this::
 
-    one-time use URL: ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+    one-time use URL: ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 **On the Host**
 
@@ -333,7 +375,7 @@ Create the ``/srv/syn/01.cortex/docker-compose.yaml`` file with contents::
             - SYN_CORTEX_JSONSTOR=aha://jsonstor...
             # disable HTTPS API for now to prevent port collisions
             - SYN_CORTEX_HTTPS_PORT=null
-            - SYN_CORTEX_AHA_PROVISION=ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+            - SYN_CORTEX_AHA_PROVISION=ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 .. note::
 
@@ -372,17 +414,17 @@ following command from **inside the AHA container** to generate a one-time use U
 
 You should see output that looks similar to this::
 
-    one-time use URL: ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+    one-time use URL: ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 Then the **user** may run::
 
-    python -m synapse.tools.aha.enroll ssl://aha.<yournetwork>:27272/<guid>?certhash=<sha256>
+    python -m synapse.tools.aha.enroll ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
 Once they are enrolled, they will have a user certificate located in ``~/.syn/certs/users`` and their telepath
 configuration located in ``~/.syn/telepath.yaml`` will be updated to reflect the use of the AHA server. From there
 the user should be able to use standard Synapse CLI tools using the ``aha://`` URL such as::
 
-    python -m synapse.tools.storm aha://visi@cortex.<yournetwork>
+    python -m synapse.tools.storm aha://visi@cortex.<dns-network>
 
 .. _deployment-guide-storm-pool:
 
diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py
index a4fd7012451..e2c3c633544 100644
--- a/synapse/tests/test_tools_aha.py
+++ b/synapse/tests/test_tools_aha.py
@@ -9,6 +9,7 @@
 import synapse.tests.utils as s_t_utils
 
 import synapse.tools.aha.list as s_a_list
+import synapse.tools.aha.clone as s_a_clone
 import synapse.tools.aha.enroll as s_a_enroll
 import synapse.tools.aha.easycert as s_a_easycert
 import synapse.tools.aha.provision.user as s_a_provision_user
@@ -93,6 +94,10 @@ async def test_aha_enroll(self):
 
         async with self.getTestAha() as aha:
 
+            argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link']
+            retn, outp = await self.execToolMain(s_a_clone.main, argv)
+            self.isin('one-time use URL:', str(outp))
+
             argv = ['--url', aha.getLocalUrl(), 'visi']
             retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
             self.isin('one-time use URL:', str(outp))
diff --git a/synapse/tools/aha/clone.py b/synapse/tools/aha/clone.py
new file mode 100644
index 00000000000..e9b76be96f5
--- /dev/null
+++ b/synapse/tools/aha/clone.py
@@ -0,0 +1,42 @@
+import sys
+import asyncio
+import argparse
+
+import synapse.exc as s_exc
+import synapse.telepath as s_telepath
+
+import synapse.lib.output as s_output
+
+descr = '''
+Generate a new clone URL to deploy an AHA mirror.
+
+Examples:
+
+    python -m synapse.tools.aha.clone 01.aha.loop.vertex.link
+
+'''
+
+async def main(argv, outp=s_output.stdout):
+
+    pars = argparse.ArgumentParser(prog='synapse.tools.aha.clone', description=descr)
+    pars.add_argument('dnsname', help='The DNS name of the new AHA server.')
+    pars.add_argument('--port', type=int, default=27492, help='The port that the new AHA server should listen on.')
+    pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.')
+
+    opts = pars.parse_args(argv)
+
+    async with s_telepath.withTeleEnv():
+
+        try:
+            async with await s_telepath.openurl(opts.url) as aha:
+                curl = await aha.addAhaClone(opts.dnsname, port=opts.port)
+                outp.printf('one-time use URL: {curl}')
+                return 0
+
+        except s_exc.SynErr as e:
+            mesg = e.errinfo.get('mesg', repr(e))
+            outp.printf(f'ERROR: {mesg}')
+            return 1
+
+if __name__ == '__main__':  # pragma: no cover
+    sys.exit(asyncio.run(main(sys.argv[1:])))

From df24baa8b910abdee74991d7f1200e40c263a31c Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 10:48:06 -0400
Subject: [PATCH 11/68] wip

---
 docs/synapse/deploymentguide.rst | 33 +++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 106bf0f03ff..cd84c056ca4 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -20,6 +20,7 @@ orchestration mechanisms such as Kubernetes but for simplicity's sake, this guid
 ``docker compose`` based deployments.
 
 .. note::
+
     Due to `known networking limitations of docker on Mac`_ we do **not** support or recommend the use
     of Docker for Mac for testing or deploying production Synapse instances. Containers run within
     separate ``docker compose`` commands will not be able to reliably communicate with each other.
@@ -29,6 +30,7 @@ within the directory ``/vertex/storage`` which should be a persistent mapped vol
 given volume at a time.
 
 .. note::
+
     To allow hosts to be provisioned on one system, this guide instructs you to disable HTTP API listening
     ports on all services other than the main Cortex. You may remove those configuration options if you are
     running on separate hosts or select alternate ports which do not conflict.
@@ -74,6 +76,7 @@ by name, so it is likely that you need to create a DNS A/AAAA record in your exi
 using AHA, the only host that needs DNS or other external name resolution is the AHA service.
 
 .. note::
+
     FIXME AHA FLAT NETWORK STUFF
 
 Choose an AHA Network Name
@@ -96,6 +99,7 @@ convention in the future. Once you have selected the DNS name for your AHA serve
 all deployed Synapse services will be able to resolve that name.
 
 .. note::
+
     It is important to ensure that ``00.aha.<dns-network>`` is resolvable via DNS or docker container service
     name resolution from within the container environment!
 
@@ -121,7 +125,7 @@ Create the ``/srv/syn/00.aha/docker-compose.yaml`` file with contents::
             # disable HTTPS API for now to prevent port collisions
             - SYN_AHA_HTTPS_PORT=null
             - SYN_AHA_AHA_NETWORK=prod.synapse
-            - SYN_AHA_DNS_NAME=00.aha.<yuournetwork>
+            - SYN_AHA_DNS_NAME=00.aha.<dns-network>
 
 .. note::
 
@@ -155,6 +159,11 @@ For high-availability deployments, you will want to deploy an AHA mirror or two.
 mirroring is configured using AHA based provisioning. Bootstrapping AHA mirrors is simple, but requires
 a slightly different procedure because you cannot bootstrap AHA via AHA :)
 
+.. note::
+
+     You can deploy AHA mirrors at any time in the future. Once the mirrors are deployed, each AHA enabled
+     Synapse service will retrieve the updated list of AHA servers the next time it is restarted.
+
 For this example, we will assume you chose a DNS name for your primary AHA server similar to the steps
 listed above. If so, you can simply replace ``00`` with sequential numbers and repeat this step to deploy
 however many AHA mirrors you deem appropriate.
@@ -175,8 +184,25 @@ Create the container directory::
     mkdir -p /srv/syn/01.aha/storage
     chown -R 999 /srv/syn/01.aha/storage
 
-::NOTE You can deploy AHA mirrors at any time in the future. Once the mirrors are deployed, each AHA enabled
-       Synapse service will retrieve the updated list of AHA servers the next time it is restarted.
+Create the ``/srv/syn/01.aha/docker-compose.yaml`` file with contents::
+
+    services:
+      01.aha:
+        user: "999"
+        image: vertexproject/synapse-aha:v2.x.x
+        network_mode: host
+        restart: unless-stopped
+        volumes:
+            - ./storage:/vertex/storage
+        environment:
+            # disable HTTPS API for now to prevent port collisions
+            - SYN_AHA_HTTPS_PORT=null
+            - SYN_AHA_CLONE=ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
+
+Start the container::
+
+    docker compose --file /srv/syn/01.aha/docker-compose.yaml pull
+    docker compose --file /srv/syn/01.aha/docker-compose.yaml up -d
 
 Deploy Axon Service
 ===================
@@ -404,6 +430,7 @@ Cortex container**::
     python -m synapse.tools.moduser --add --admin true visi
 
 .. note::
+
     If you are a Synapse Enterprise customer, using the Synapse UI with SSO, the admin may now login to the
     Synapse UI. You may skip the following steps if the admin will not be using CLI tools to access the Cortex.
 

From b5a543ea11bed2b811c4d3e5e94c36db1c67f261 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 11:32:05 -0400
Subject: [PATCH 12/68] wip

---
 docs/synapse/deploymentguide.rst | 4 ++++
 synapse/lib/cell.py              | 4 ++++
 synapse/tests/test_lib_config.py | 3 ++-
 3 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index cd84c056ca4..c4bc8b8b54f 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -171,6 +171,8 @@ however many AHA mirrors you deem appropriate.
 NOTE: AHA uses two default ports ETC. The following steps assume you will be running each of your AHA servers
 on a different host. The use of ``network_mode; host`` ETC
 
+**Inside the AHA container**
+
 Generate a one-time use provisioning URL::
 
     python -m synapse.tools.aha.clone 01.aha.<dns-network>
@@ -179,6 +181,8 @@ You should see output that looks similar to this::
 
     one-time use URL: ssl://00.aha.<dns-network>:27272/<guid>?certhash=<sha256>
 
+**On the Host**
+
 Create the container directory::
 
     mkdir -p /srv/syn/01.aha/storage
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index c06497ffe81..20f1621338d 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1496,6 +1496,10 @@ async def _initAhaRegistry(self):
             if self.ahaclient is not None:
                 await self.ahaclient.fini()
 
+            async def onlink(proxy):
+                ahauser = self.conf.req('aha:user')
+                ahaurls = await proxy.getAhaUrls(user=ahauser)
+
             self.ahaclient = await s_telepath.Client.anit(ahaurl)
             self.onfini(self.ahaclient)
 
diff --git a/synapse/tests/test_lib_config.py b/synapse/tests/test_lib_config.py
index cb61a6ae3d4..f04de15b829 100644
--- a/synapse/tests/test_lib_config.py
+++ b/synapse/tests/test_lib_config.py
@@ -285,7 +285,8 @@ async def test_config_fromcell(self):
                 # Trying to make a cell with a missing key it wants fails
                 async with await SchemaCell.anit(dirn, conf={}) as cell:
                     pass
-            self.eq(cm.exception.get('key'), 'apikey')
+
+            self.eq(cm.exception.get('name'), 'apikey')
 
     def test_hideconf(self):
         hide_schema = {

From d891c82a477913590ea441445c164bfbda36f916 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 11:47:42 -0400
Subject: [PATCH 13/68] wip

---
 synapse/lib/cell.py  | 12 +++++++-----
 synapse/lib/nexus.py | 11 +++++------
 2 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 20f1621338d..a02be00bce8 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1190,7 +1190,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
 
         root = await self._ctorNexsRoot()
 
-        await self.setNexsRoot(root)
+        self.setNexsRoot(root)
 
         self.apikeydb = self.slab.initdb('user:apikeys')  # apikey -> useriden
         self.usermetadb = self.slab.initdb('user:meta')  # useriden + <valu> -> dict valu
@@ -1489,16 +1489,18 @@ async def _runSysctlLoop(self):
     async def _initAhaRegistry(self):
 
         # NOTE: This API needs to be safe to call again
-        ahaurl = self.conf.get('aha:registry')
-        if ahaurl is not None:
+        ahaurls = self.conf.get('aha:registry')
+        if ahaurls is not None:
 
-            info = await s_telepath.addAhaUrl(ahaurl)
+            info = await s_telepath.addAhaUrl(ahaurls)
             if self.ahaclient is not None:
                 await self.ahaclient.fini()
 
             async def onlink(proxy):
                 ahauser = self.conf.req('aha:user')
-                ahaurls = await proxy.getAhaUrls(user=ahauser)
+                newurls = await proxy.getAhaUrls(user=ahauser)
+                if newurls and newurls != self.conf.get('aha:registry'):
+                    self.modCellConf({'aha:registry': newurls})
 
             self.ahaclient = await s_telepath.Client.anit(ahaurl)
             self.onfini(self.ahaclient)
diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py
index 2f00e49c343..e3c6f4dca2d 100644
--- a/synapse/lib/nexus.py
+++ b/synapse/lib/nexus.py
@@ -614,12 +614,12 @@ async def __anit__(self, iden: str, nexsroot: NexsRoot = None):  # type: ignore
         self.nexsroot: Optional[NexsRoot] = None
 
         if nexsroot is not None:
-            await self.setNexsRoot(nexsroot)
+            self.setNexsRoot(nexsroot)
 
         for event, func, passitem in self._regclstupls:  # type: ignore
             self._nexshands[event] = func, passitem
 
-    async def setNexsRoot(self, nexsroot):
+    def setNexsRoot(self, nexsroot):
 
         nexsroot._nexskids[self.nexsiden] = self
 
@@ -627,8 +627,7 @@ def onfini():
             prev = nexsroot._nexskids.pop(self.nexsiden, None)
             assert prev is not None, f'Failed removing {self.nexsiden}'
 
-        nexsroot.onfini(onfini)
-        self.onfini(nexsroot)
+        self.onfini(onfini)
 
         self.nexsroot = nexsroot
 
@@ -639,9 +638,9 @@ async def modNexsRoot(self, ctor):
 
         nexsroot = await ctor()
 
-        await self.setNexsRoot(nexsroot)
+        self.setNexsRoot(nexsroot)
 
-        await self.nexsroot.startup()
+        await nexsroot.startup()
 
     @classmethod
     def onPush(cls, event: str, passitem=False) -> Callable:

From f31f1af2924c3a478b5677c6acf43c4280fecc17 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 11:54:29 -0400
Subject: [PATCH 14/68] wip

---
 synapse/lib/cell.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index a02be00bce8..f8105eef443 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1488,7 +1488,6 @@ async def _runSysctlLoop(self):
 
     async def _initAhaRegistry(self):
 
-        # NOTE: This API needs to be safe to call again
         ahaurls = self.conf.get('aha:registry')
         if ahaurls is not None:
 
@@ -1514,15 +1513,6 @@ async def fini():
         if ahaadmin is not None:
             await self._addAdminUser(ahaadmin)
 
-    def reqAhaUser(self):
-
-        ahauser = self.conf.get('aha:user')
-        if ahauser is not None:
-            return ahauser
-
-        mesg = 'An aha:user configuration option is required.'
-        raise s_exc.NeedConfValu(mesg=mesg)
-
     def _getDmonListen(self):
 
         lisn = self.conf.get('dmon:listen', s_common.novalu)

From 7e2fb641d3189f8e82a4cc56ca418332a4bb4770 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 12:09:29 -0400
Subject: [PATCH 15/68] wip

---
 synapse/lib/aha.py  | 4 ----
 synapse/lib/cell.py | 2 +-
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 66264e8f0b1..0fd1272dccd 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -126,10 +126,6 @@ async def get(self):
 
 class AhaApi(s_cell.CellApi):
 
-    @s_cell.adminapi()
-    async def getAhaClone(self, iden):
-        return await self.cell.getAhaClone(iden)
-
     @s_cell.adminapi()
     async def addAhaClone(self, host, port=27492, conf=None):
         return await self.cell.addAhaClone(host, port=port, conf=conf)
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index f8105eef443..a94dfafa5dd 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1496,7 +1496,7 @@ async def _initAhaRegistry(self):
                 await self.ahaclient.fini()
 
             async def onlink(proxy):
-                ahauser = self.conf.req('aha:user')
+                ahauser = self.conf.get('aha:user', 'root')
                 newurls = await proxy.getAhaUrls(user=ahauser)
                 if newurls and newurls != self.conf.get('aha:registry'):
                     self.modCellConf({'aha:registry': newurls})

From 1d5e0df683737bc61c79897e0fd9eec961d93a27 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 14:00:37 -0400
Subject: [PATCH 16/68] wip

---
 synapse/lib/cell.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index a94dfafa5dd..101e1940021 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1501,11 +1501,11 @@ async def onlink(proxy):
                 if newurls and newurls != self.conf.get('aha:registry'):
                     self.modCellConf({'aha:registry': newurls})
 
-            self.ahaclient = await s_telepath.Client.anit(ahaurl)
+            self.ahaclient = await s_telepath.Client.anit(ahaurls)
             self.onfini(self.ahaclient)
 
             async def fini():
-                await s_telepath.delAhaUrl(ahaurl)
+                await s_telepath.delAhaUrl(ahaurls)
 
             self.ahaclient.onfini(fini)
 

From ff884dcb83d7bcf042824395b8f3b00f1b7d7e5d Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 14:31:55 -0400
Subject: [PATCH 17/68] wip

---
 synapse/lib/cell.py     |  2 +-
 synapse/lib/lmdbslab.py |  2 ++
 synapse/lib/nexus.py    | 11 ++++-------
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 101e1940021..dc09a99156e 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -904,7 +904,7 @@ class Cell(s_nexus.Pusher, s_telepath.Aware):
         },
         'nexslog:async': {
             'default': True,
-            'description': 'Set to false to disable async memory mapping of the nexus change log.',
+            'description': 'Deprecated. This option ignored.',
             'type': 'boolean',
             'hidedocs': True,
             'hidecmdl': True,
diff --git a/synapse/lib/lmdbslab.py b/synapse/lib/lmdbslab.py
index c1148d3ad77..8aeb0b19fa3 100644
--- a/synapse/lib/lmdbslab.py
+++ b/synapse/lib/lmdbslab.py
@@ -709,6 +709,8 @@ async def __anit__(self, path, **kwargs):
         kwargs.setdefault('lockmemory', False)
         kwargs.setdefault('map_async', True)
 
+        assert kwargs.get('map_async')
+
         opts = kwargs
 
         self.path = path
diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py
index 9a4f1071379..805d602ea06 100644
--- a/synapse/lib/nexus.py
+++ b/synapse/lib/nexus.py
@@ -109,9 +109,7 @@ async def __anit__(self, cell):
 
         logpath = s_common.genpath(self.dirn, 'slabs', 'nexuslog')
 
-        self.map_async = self.cell.conf.get('nexslog:async')
-        self.nexsslab = await s_lmdbslab.Slab.anit(path, map_async=self.map_async)
-        self.nexsslab.addResizeCallback(cell.checkFreeSpace)
+        self.nexsslab = await cell._initSlabFile(path)
 
         self.nexshot = await self.nexsslab.getHotCount('nexs:indx')
 
@@ -125,8 +123,7 @@ async def __anit__(self, cell):
         elif vers != 2:
             raise s_exc.BadStorageVersion(mesg=f'Got nexus log version {vers}.  Expected 2.  Accidental downgrade?')
 
-        slabopts = {'map_async': self.map_async}
-        self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, slabopts=slabopts, cell=cell)
+        self.nexslog = await s_multislabseqn.MultiSlabSeqn.anit(logpath, cell=cell)
 
         # just in case were previously configured differently
         logindx = self.nexslog.index()
@@ -194,8 +191,7 @@ async def _migrateV1toV2(self, nexspath, logpath):
 
         # Open a fresh slab where the old one used to be
         logger.warning(f'Re-opening fresh nexslog slab at {nexspath} for nexshot')
-        self.nexsslab = await s_lmdbslab.Slab.anit(nexspath, map_async=self.map_async)
-        self.nexsslab.addResizeCallback(self.cell.checkFreeSpace)
+        self.nexsslab = await self.cell._initSlabFile(nexspath)
 
         self.nexshot = await self.nexsslab.getHotCount('nexs:indx')
 
@@ -633,6 +629,7 @@ def onfini():
             assert prev is not None, f'Failed removing {self.nexsiden}'
 
         self.onfini(onfini)
+        self.onfini(nexsroot)
 
         self.nexsroot = nexsroot
 

From 562cea715941bfbcd48ac6e5b31e846738ad205d Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 15:27:10 -0400
Subject: [PATCH 18/68] wip

---
 synapse/cortex.py            | 12 ++++++++----
 synapse/lib/cell.py          |  9 +++++++--
 synapse/lib/layer.py         |  5 -----
 synapse/lib/lmdbslab.py      |  4 +---
 synapse/lib/nexus.py         |  3 ---
 synapse/tests/test_cortex.py | 19 +++----------------
 synapse/tests/utils.py       |  8 +-------
 7 files changed, 20 insertions(+), 40 deletions(-)

diff --git a/synapse/cortex.py b/synapse/cortex.py
index 168d18ba865..0350872ec94 100644
--- a/synapse/cortex.py
+++ b/synapse/cortex.py
@@ -773,13 +773,17 @@ class Cortex(s_oauth.OAuthMixin, s_cell.Cell):  # type: ignore
         },
         'layer:lmdb:map_async': {
             'default': True,
-            'description': 'Set the default lmdb:map_async value in LMDB layers.',
-            'type': 'boolean'
+            'description': 'Deprecated. This value is ignored.',
+            'type': 'boolean',
+            'hidecmdl': True,
+            'hideconf': True,
         },
         'layer:lmdb:max_replay_log': {
             'default': 10000,
-            'description': 'Set the max size of the replay log for all layers.',
-            'type': 'integer'
+            'description': 'Deprecated. This value is ignored.',
+            'type': 'integer',
+            'hidecmdl': True,
+            'hideconf': True,
         },
         'layers:lockmemory': {
             'default': False,
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index dc09a99156e..e1af664d45a 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1188,9 +1188,14 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
 
         await self.initServiceEarly()
 
-        root = await self._ctorNexsRoot()
+        nexsroot = await self._ctorNexsRoot()
 
-        self.setNexsRoot(root)
+        self.setNexsRoot(nexsroot)
+
+        async def fini():
+            await self.nexsroot.fini()
+
+        self.onfini(fini)
 
         self.apikeydb = self.slab.initdb('user:apikeys')  # apikey -> useriden
         self.usermetadb = self.slab.initdb('user:meta')  # useriden + <valu> -> dict valu
diff --git a/synapse/lib/layer.py b/synapse/lib/layer.py
index 5ae38bf3c9d..e7f4063fee9 100644
--- a/synapse/lib/layer.py
+++ b/synapse/lib/layer.py
@@ -1409,9 +1409,6 @@ async def __anit__(self, core, layrinfo):
         self.growsize = self.layrinfo.get('growsize')
         self.logedits = self.layrinfo.get('logedits')
 
-        self.mapasync = core.conf.get('layer:lmdb:map_async')
-        self.maxreplaylog = core.conf.get('layer:lmdb:max_replay_log')
-
         # slim hooks to avoid async/fire
         self.nodeAddHook = None
         self.nodeDelHook = None
@@ -2704,8 +2701,6 @@ async def _initLayerStorage(self):
         slabopts = {
             'readahead': True,
             'lockmemory': self.lockmemory,
-            'map_async': self.mapasync,
-            'max_replay_log': self.maxreplaylog,
         }
 
         if self.growsize is not None:
diff --git a/synapse/lib/lmdbslab.py b/synapse/lib/lmdbslab.py
index 8aeb0b19fa3..02023fa2bcc 100644
--- a/synapse/lib/lmdbslab.py
+++ b/synapse/lib/lmdbslab.py
@@ -696,7 +696,7 @@ async def getSlabStats(clas):
                 'recovering': slab.recovering,
                 'maxsize': slab.maxsize,
                 'growsize': slab.growsize,
-                'mapasync': slab.mapasync,
+                'mapasync': True,
 
             })
         return retn
@@ -755,8 +755,6 @@ async def __anit__(self, path, **kwargs):
                 logger.info(f'SYN_LOCKMEM_DISABLE envar set, skipping lockmem for {self.path}')
                 self.lockmemory = False
 
-        self.mapasync = opts.setdefault('map_async', True)
-
         self.mapsize = _mapsizeround(mapsize)
         if self.maxsize is not None:
             self.mapsize = min(self.mapsize, self.maxsize)
diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py
index 805d602ea06..fbebc3efdf2 100644
--- a/synapse/lib/nexus.py
+++ b/synapse/lib/nexus.py
@@ -304,7 +304,6 @@ async def issue(self, nexsiden, event, args, kwargs, meta=None):
         If I'm not a follower, mutate, otherwise, ask the leader to make the change and wait for the follower loop
         to hand me the result through a future.
         '''
-
         # pick up a reference to avoid race when we eventually can promote
         client = self.client
 
@@ -629,8 +628,6 @@ def onfini():
             assert prev is not None, f'Failed removing {self.nexsiden}'
 
         self.onfini(onfini)
-        self.onfini(nexsroot)
-
         self.nexsroot = nexsroot
 
     async def modNexsRoot(self, ctor):
diff --git a/synapse/tests/test_cortex.py b/synapse/tests/test_cortex.py
index bf2b9ca9413..e265ec605bc 100644
--- a/synapse/tests/test_cortex.py
+++ b/synapse/tests/test_cortex.py
@@ -4410,10 +4410,7 @@ async def test_cortex_nexslogen_off(self):
         '''
         Everything still works when no nexus log is kept
         '''
-        conf = {'layer:lmdb:map_async': True,
-                'nexslog:en': False,
-                'layers:logedits': True,
-                }
+        conf = {'nexslog:en': False, 'layers:logedits': True}
         async with self.getTestCore(conf=conf) as core:
             self.len(2, await core.nodes('[test:str=foo test:str=bar]'))
             self.len(2, await core.nodes('test:str'))
@@ -4422,10 +4419,7 @@ async def test_cortex_logedits_off(self):
         '''
         Everything still works when no layer log is kept
         '''
-        conf = {'layer:lmdb:map_async': True,
-                'nexslog:en': True,
-                'layers:logedits': False,
-                }
+        conf = {'nexslog:en': True, 'layers:logedits': False}
         async with self.getTestCore(conf=conf) as core:
             self.len(2, await core.nodes('[test:str=foo test:str=bar]'))
             self.len(2, await core.nodes('test:str'))
@@ -4443,18 +4437,12 @@ async def test_cortex_layer_settings(self):
         '''
         Make sure settings make it down to the slab
         '''
-        conf = {
-            'layer:lmdb:map_async': False,
-            'layer:lmdb:max_replay_log': 500,
-            'layers:lockmemory': True,
-        }
+        conf = {'layers:lockmemory': True}
         async with self.getTestCore(conf=conf) as core:
             layr = core.getLayer()
             slab = layr.layrslab
 
             self.true(slab.lockmemory)
-            self.eq(500, slab.max_xactops_len)
-            self.true(500, slab.mapasync)
 
     async def test_feed_syn_nodes(self):
 
@@ -8141,7 +8129,6 @@ async def initNexusSubsystem(self):
                 return ret
 
         conf = {
-            'layer:lmdb:map_async': True,
             'nexslog:en': True,
             'layers:logedits': True,
         }
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index f1487db931d..f1cf8019fc4 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -1311,14 +1311,8 @@ async def getTestCore(self, conf=None, dirn=None):
 
         with self.withNexusReplay():
 
-            if dirn is not None:
-
-                async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
-                    yield core
-
-                return
+            with self.mayTestDir(dirn) as dirn:
 
-            with self.getTestDir() as dirn:
                 async with await s_cortex.Cortex.anit(dirn, conf=conf) as core:
                     yield core
 

From 0fc325733d182a2cc58d8afa9f2a44e2a32c5268 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 15:45:37 -0400
Subject: [PATCH 19/68] wip

---
 synapse/tests/test_lib_cell.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py
index a202ec1dab0..fb7d488f9c8 100644
--- a/synapse/tests/test_lib_cell.py
+++ b/synapse/tests/test_lib_cell.py
@@ -606,7 +606,6 @@ async def coro(prox, offs):
             async with self.getTestCell(s_cell.Cell, dirn=dir0, conf=conf) as cell00, \
                 cell00.getLocalProxy() as prox00:
 
-                self.true(cell00.nexsroot.map_async)
                 self.true(cell00.nexsroot.donexslog)
 
                 await prox00.addUser('test')

From 27cfd9f6407df82bf3012c57138a43141b9e417c Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 5 Jul 2024 18:17:36 -0400
Subject: [PATCH 20/68] improve coverage

---
 synapse/telepath.py             | 5 ++++-
 synapse/tests/test_lib_nexus.py | 8 ++++++++
 synapse/tests/test_tools_aha.py | 4 ++++
 synapse/tools/aha/clone.py      | 7 +++++--
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/synapse/telepath.py b/synapse/telepath.py
index 63a98875b8f..76508f061aa 100644
--- a/synapse/telepath.py
+++ b/synapse/telepath.py
@@ -1543,7 +1543,7 @@ async def openinfo(info):
             path, name = path.split(':')
         link = await s_link.unixconnect(path)
 
-    else:
+    elif scheme in ('tcp', 'ssl'):
 
         path = info.get('path')
         name = info.get('name', path[1:])
@@ -1592,6 +1592,9 @@ async def openinfo(info):
 
         link = await s_link.connect(host, port, linkinfo=linkinfo)
 
+    else:
+        raise s_exc.BadUrl(mesg=f'Invalid URL scheme: {scheme}')
+
     prox = await Proxy.anit(link, name)
     prox.onfini(link)
 
diff --git a/synapse/tests/test_lib_nexus.py b/synapse/tests/test_lib_nexus.py
index 81f5ea3eb70..e4b1521b155 100644
--- a/synapse/tests/test_lib_nexus.py
+++ b/synapse/tests/test_lib_nexus.py
@@ -111,6 +111,14 @@ async def test_nexus(self):
                     stream.seek(0)
                     self.isin('while replaying log', stream.read())
 
+    async def test_nexus_modroot(self):
+
+        async with self.getTestCell(s_cell.Cell) as cell:
+            await cell.sync()
+            async with cell.nexslock:
+                cell.modNexsRoot(cell._ctorNexsRoot)
+            await cell.sync()
+
     async def test_nexus_mixin(self):
         with self.getTestDir() as dirn:
             dir1 = s_common.genpath(dirn, 'nexus1')
diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py
index e2c3c633544..e3ca70d4ba0 100644
--- a/synapse/tests/test_tools_aha.py
+++ b/synapse/tests/test_tools_aha.py
@@ -98,6 +98,10 @@ async def test_aha_enroll(self):
             retn, outp = await self.execToolMain(s_a_clone.main, argv)
             self.isin('one-time use URL:', str(outp))
 
+            argv = ['--url', 'newp://1.2.3.4', '01.aha.loop.vertex.link']
+            retn, outp = await self.execToolMain(s_a_clone.main, argv)
+            self.isin('ERROR: Invalid URL scheme: newp', str(outp))
+
             argv = ['--url', aha.getLocalUrl(), 'visi']
             retn, outp = await self.execToolMain(s_a_provision_user.main, argv)
             self.isin('one-time use URL:', str(outp))
diff --git a/synapse/tools/aha/clone.py b/synapse/tools/aha/clone.py
index e9b76be96f5..878b08f7425 100644
--- a/synapse/tools/aha/clone.py
+++ b/synapse/tools/aha/clone.py
@@ -33,8 +33,11 @@ async def main(argv, outp=s_output.stdout):
                 outp.printf('one-time use URL: {curl}')
                 return 0
 
-        except s_exc.SynErr as e:
-            mesg = e.errinfo.get('mesg', repr(e))
+        except Exception as e:
+            mesg = repr(e)
+            if isinstance(e, s_exc.SynErr):
+                mesg = e.errinfo.get('mesg', repr(e))
+
             outp.printf(f'ERROR: {mesg}')
             return 1
 

From b9bb3f06f58ec94a0bd79207c58cff51cc0f0c9c Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Sat, 6 Jul 2024 10:19:06 -0400
Subject: [PATCH 21/68] wip

---
 synapse/lib/cell.py           |   6 +-
 synapse/lib/nexus.py          |   3 +
 synapse/telepath.py           |  40 ++++++----
 synapse/tests/test_lib_aha.py | 138 +++++++++++-----------------------
 4 files changed, 76 insertions(+), 111 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index e1af664d45a..b6281b42e76 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1503,10 +1503,12 @@ async def _initAhaRegistry(self):
             async def onlink(proxy):
                 ahauser = self.conf.get('aha:user', 'root')
                 newurls = await proxy.getAhaUrls(user=ahauser)
-                if newurls and newurls != self.conf.get('aha:registry'):
+                oldurls = tuple(self.conf.get('aha:registry'))
+                if newurls and newurls != oldurls:
                     self.modCellConf({'aha:registry': newurls})
+                    self.ahaclient.setBootUrls(newurls)
 
-            self.ahaclient = await s_telepath.Client.anit(ahaurls)
+            self.ahaclient = await s_telepath.Client.anit(ahaurls, onlink=onlink)
             self.onfini(self.ahaclient)
 
             async def fini():
diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py
index fbebc3efdf2..8322f5d8f17 100644
--- a/synapse/lib/nexus.py
+++ b/synapse/lib/nexus.py
@@ -571,6 +571,9 @@ async def runMirrorLoop(self, proxy):
                         if respfutu is not None:
                             respfutu.set_result(retn)
 
+            except s_exc.LinkShutDown:
+                logger.warning(f'mirror loop: leader closed the connection.')
+
             except Exception as exc:  # pragma: no cover
                 logger.exception(f'error in mirror loop: {exc}')
 
diff --git a/synapse/telepath.py b/synapse/telepath.py
index 76508f061aa..5c538bd8a03 100644
--- a/synapse/telepath.py
+++ b/synapse/telepath.py
@@ -997,12 +997,6 @@ async def __anit__(self, urlinfo, onlink=None):
 
         await s_base.Base.__anit__(self)
 
-        # some ugly stuff in order to be backward compatible...
-        if not isinstance(urlinfo, (list, tuple)):
-            urlinfo = (urlinfo,)
-
-        urlinfo = [chopurl(u) for u in urlinfo]
-
         self.aha = None
 
         self.clients = {}
@@ -1012,9 +1006,9 @@ async def __anit__(self, urlinfo, onlink=None):
 
         self.onlink = onlink
 
-        self.booturls = urlinfo
         self.bootdeque = collections.deque()
-        self.bootdeque.extend(self.booturls)
+
+        self.setBootUrls(urlinfo)
 
         self.ready = asyncio.Event()
         self.deque = collections.deque()
@@ -1033,6 +1027,16 @@ async def fini():
 
         self.schedCoro(self._initBootProxy())
 
+    def setBootUrls(self, urlinfo):
+
+        if not isinstance(urlinfo, (list, tuple)):
+            urlinfo = (urlinfo,)
+
+        self.booturls = [chopurl(u) for u in urlinfo]
+
+        self.bootdeque.clear()
+        self.bootdeque.extend(self.booturls)
+
     def getNextBootUrl(self):
         if not self.bootdeque:
             self.bootdeque.extend(self.booturls)
@@ -1210,20 +1214,16 @@ async def __anit__(self, urlinfo, opts=None, conf=None, onlink=None):
 
         await s_base.Base.__anit__(self)
 
-        if isinstance(urlinfo, (str, dict)):
-            urlinfo = (urlinfo,)
-
-        urlinfo = [chopurl(u) for u in urlinfo]
-
         if conf is None:
             conf = {}
 
         if opts is None:
             opts = {}
 
-        self._t_urlinfo = urlinfo
         self._t_urldeque = collections.deque()
 
+        self.setBootUrls(urlinfo)
+
         self._t_opts = opts
         self._t_conf = conf
 
@@ -1247,6 +1247,16 @@ async def fini():
 
         await self._fireLinkLoop()
 
+    def setBootUrls(self, urlinfo):
+
+        if not isinstance(urlinfo, (list, tuple)):
+            urlinfo = (urlinfo,)
+
+        self._t_urlinfo = [chopurl(u) for u in urlinfo]
+
+        self._t_urldeque.clear()
+        self._t_urldeque.extend(self._t_urlinfo)
+
     def _getNextUrl(self):
         if not self._t_urldeque:
             self._t_urldeque.extend(self._t_urlinfo)
@@ -1281,7 +1291,7 @@ async def _teleLinkLoop(self):
                 now = time.monotonic()
                 if now > lastlog + 60.0:  # don't logspam the disconnect message more than 1/min
                     url = s_urlhelp.sanitizeUrl(zipurl(urlinfo))
-                    logger.exception(f'telepath client ({url}) encountered an error: {e}')
+                    logger.warning(f'telepath client ({url}) encountered an error: {e}')
                     lastlog = now
                 await self.waitfini(timeout=self._t_conf.get('retrysleep', 0.2))
 
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index ffc716e0d4f..083c0ffd55b 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -131,7 +131,7 @@ async def test_lib_aha_offon(self):
                 svc = await aha.getAhaSvc('0.cryo...')
                 self.notin('online', svc.get('svcinfo'))
 
-    async def test_lib_aha(self):
+    async def test_lib_aha_basics(self):
 
         with self.raises(s_exc.NoSuchName):
             await s_telepath.getAhaProxy({})
@@ -159,40 +159,26 @@ async def test_lib_aha(self):
 
             cryo0_dirn = s_common.gendir(aha.dirn, 'cryo0')
 
-            host, port = await aha.dmon.listen('tcp://127.0.0.1:0')
-            await aha.auth.rootuser.setPasswd('hehehaha')
+            ahaurls = await aha.getAhaUrls()
 
             wait00 = aha.waiter(1, 'aha:svcadd')
-            conf = {
-                'aha:name': '0.cryo',
-                'aha:leader': 'cryo',
-                'aha:network': 'synapse',
-                'aha:admin': 'root@synapse',
-                'aha:registry': [f'tcp://root:hehehaha@127.0.0.1:{port}',
-                                 f'tcp://root:hehehaha@127.0.0.1:{port}'],
-                'dmon:listen': 'tcp://0.0.0.0:0/',
-            }
-            async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
 
-                await cryo.auth.rootuser.setPasswd('secret')
-
-                ahaadmin = await cryo.auth.getUserByName('root@synapse')
-                self.nn(ahaadmin)
-                self.true(ahaadmin.isAdmin())
+            conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')}
+            async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
 
                 await wait00.wait(timeout=2)
 
                 with self.raises(s_exc.NoSuchName):
                     await s_telepath.getAhaProxy({'host': 'hehe.haha'})
 
-                async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
+                async with await s_telepath.openurl('aha://cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 with self.raises(s_exc.BadArg):
                     _proxy = await cryo.ahaclient.proxy(timeout=2)
                     await _proxy.modAhaSvcInfo('cryo...', {'newp': 'newp'})
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
+                async with await s_telepath.openurl('aha://0.cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 # force a reconnect...
@@ -201,7 +187,7 @@ async def test_lib_aha(self):
                 await proxy.fini()
                 self.nn(await waiter.wait(timeout=6))
 
-                async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
+                async with await s_telepath.openurl('aha://cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 waiter = aha.waiter(1, 'aha:svcadd')
@@ -209,72 +195,51 @@ async def test_lib_aha(self):
                 await cryo.setCellActive(False)
 
                 with self.raises(s_exc.NoSuchName):
-                    async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
+                    async with await s_telepath.openurl('aha://cryo...') as proxy:
                         pass
 
                 self.nn(await waiter.wait(timeout=6))
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo...') as proxy:
+                async with await s_telepath.openurl('aha://0.cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 await cryo.setCellActive(True)
 
-                async with await s_telepath.openurl('aha://root:secret@cryo...') as proxy:
+                async with await s_telepath.openurl('aha://cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
                 # some coverage edge cases...
                 cryo.conf.pop('aha:leader', None)
                 await cryo.setCellActive(False)
-
-                # lock the aha:admin account so we can confirm it is unlocked upon restart
                 # remove the admin flag from the account.
-                self.false(ahaadmin.isLocked())
-                await ahaadmin.setLocked(True, logged=False)
-                self.true(ahaadmin.isLocked())
-                # remove the admin status so we can confirm its an admin upon restart
-                await ahaadmin.setAdmin(False, logged=False)
-                self.false(ahaadmin.isAdmin())
-
-            async with self.getTestCryo(dirn=cryo0_dirn, conf=conf) as cryo:
-                ahaadmin = await cryo.auth.getUserByName('root@synapse')
-                # And we should be unlocked and admin now
-                self.false(ahaadmin.isLocked())
-                self.true(ahaadmin.isAdmin())
 
             wait01 = aha.waiter(1, 'aha:svcadd')
-            conf = {
-                'aha:name': '0.cryo',
-                'aha:leader': 'cryo',
-                'aha:network': 'foo',
-                'aha:registry': f'tcp://root:hehehaha@127.0.0.1:{port}',
-                'dmon:listen': 'tcp://0.0.0.0:0/',
-            }
+            conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')}
             async with self.getTestCryo(conf=conf) as cryo:
 
                 info = await cryo.getCellInfo()
                 cnfo = info.get('cell')
                 anfo = cnfo.get('aha')
-                self.eq(cnfo.get('aha'), {'name': '0.cryo', 'leader': 'cryo', 'network': 'foo'})
-
-                await cryo.auth.rootuser.setPasswd('secret')
+                self.eq(cnfo.get('aha'), {'name': '0.cryo', 'leader': 'cryo', 'network': 'synapse'})
 
                 await wait01.wait(timeout=2)
 
-                async with await s_telepath.openurl('aha://root:secret@cryo.foo') as proxy:
+                async with await s_telepath.openurl('aha://cryo.synapse') as proxy:
                     self.nn(await proxy.getCellIden())
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo.foo') as proxy:
+                async with await s_telepath.openurl('aha://0.cryo.synapse') as proxy:
                     self.nn(await proxy.getCellIden())
                     await proxy.puts('hehe', ('hehe', 'haha'))
 
-                async with await s_telepath.openurl('aha://root:secret@0.cryo.foo/*/hehe') as proxy:
+                async with await s_telepath.openurl('aha://0.cryo.synapse/*/hehe') as proxy:
                     self.nn(await proxy.iden())
 
-                async with await s_telepath.openurl(f'tcp://root:hehehaha@127.0.0.1:{port}') as ahaproxy:
-                    svcs = [x async for x in ahaproxy.getAhaSvcs('foo')]
+                async with aha.getLocalProxy() as ahaproxy:
+
+                    svcs = [x async for x in ahaproxy.getAhaSvcs('synapse')]
                     self.len(2, svcs)
                     names = [s['name'] for s in svcs]
-                    self.sorteq(('cryo.foo', '0.cryo.foo'), names)
+                    self.sorteq(('cryo.synapse', '0.cryo.synapse'), names)
 
                     self.none(await ahaproxy.getCaCert('vertex.link'))
                     cacert0 = await ahaproxy.genCaCert('vertex.link')
@@ -301,20 +266,10 @@ async def test_lib_aha(self):
                     self.nn(usercert01)
                     self.ne(usercert00, usercert01)
 
-            async with await s_telepath.openurl(f'tcp://root:hehehaha@127.0.0.1:{port}') as ahaproxy:
-                await ahaproxy.delAhaSvc('cryo', network='foo')
-                await ahaproxy.delAhaSvc('0.cryo', network='foo')
-                self.none(await ahaproxy.getAhaSvc('cryo.foo'))
-                self.none(await ahaproxy.getAhaSvc('0.cryo.foo'))
-                self.len(2, [s async for s in ahaproxy.getAhaSvcs()])
-
-                with self.raises(s_exc.BadArg):
-                    info = {'urlinfo': {'host': '127.0.0.1', 'port': 8080, 'scheme': 'tcp'}}
-                    await ahaproxy.addAhaSvc('newp', info, network=None)
-
             # We can use HTTP API to get the registered services
             await aha.addUser('lowuser', passwd='lowuser')
             await aha.auth.rootuser.setPasswd('secret')
+
             host, httpsport = await aha.addHttpsPort(0)
             svcsurl = f'https://localhost:{httpsport}/api/v1/aha/services'
 
@@ -359,37 +314,32 @@ async def test_lib_aha(self):
                     self.eq(info.get('status'), 'err')
                     self.eq(info.get('code'), 'AuthDeny')
 
-        # The aha service can also be configured with a set of URLs that could represent itself.
-        urls = ('cell://home0', 'cell://home1')
-        conf = {'aha:urls': urls}
-        async with self.getTestAha(conf=conf) as aha:
             async with aha.getLocalProxy() as ahaproxy:
-                aurls = await ahaproxy.getAhaUrls()
-                self.eq(urls, aurls)
+                await ahaproxy.delAhaSvc('cryo', network='synapse')
+                await ahaproxy.delAhaSvc('0.cryo', network='synapse')
+                self.none(await ahaproxy.getAhaSvc('cryo.synapse'))
+                self.none(await ahaproxy.getAhaSvc('0.cryo.synapse'))
+                self.len(0, [s async for s in ahaproxy.getAhaSvcs()])
 
-        with self.getTestDir() as dirn:
-            conf = {
-                'aha:name': '0.test',
-                'aha:leader': 'test',
-                'aha:network': 'foo',
-                'aha:registry': f'tcp://root:hehehaha@127.0.0.1:{port}',
-                'dmon:listen': f'unix://{dirn}/sock'
-            }
-            async with self.getTestAha(conf=conf) as aha:
-                ahainfo = await aha.getAhaInfo()
-                uinfo = ahainfo.get('urlinfo', {})
-                self.eq(uinfo.get('scheme'), 'unix')
-                self.none(uinfo.get('port'))
-                self.len(1, await aha.getAhaUrls())
-
-            conf['dmon:listen'] = 'tcp://0.0.0.0:0/'
-            async with self.getTestAha(conf=conf) as aha:
-                ahainfo = await aha.getAhaInfo()
-                ahaurls = await aha.getAhaUrls()
-                uinfo = ahainfo.get('urlinfo', {})
-                self.eq(uinfo.get('scheme'), 'tcp')
-                self.gt(uinfo.get('port'), 0)
-                self.eq(ahaurls[0], f'ssl://00.aha.loop.vertex.link:{aha.sockaddr[1]}?certname=root@foo')
+                with self.raises(s_exc.BadArg):
+                    info = {'urlinfo': {'host': '127.0.0.1', 'port': 8080, 'scheme': 'tcp'}}
+                    await ahaproxy.addAhaSvc('newp', info, network=None)
+
+            # test that services get updated aha server list
+            with self.getTestDir() as dirn:
+
+                conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')}
+
+                async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell:
+                    self.len(1, cell.conf.get('aha:registry'))
+
+                await aha.addAhaServer({'host': '01.aha.loop.vertex.link'})
+
+                self.len(2, await aha.getAhaServers())
+
+                async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell:
+                    await cell.ahaclient.proxy()
+                    self.len(2, cell.conf.get('aha:registry'))
 
     async def test_lib_aha_loadenv(self):
 

From 92b498dd447aeb0412555a1cccbd01be976f2c1b Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Sat, 6 Jul 2024 11:11:53 -0400
Subject: [PATCH 22/68] wip

---
 synapse/lib/base.py           | 19 +++++++--
 synapse/tests/test_lib_aha.py | 73 ++++++++++++++---------------------
 2 files changed, 46 insertions(+), 46 deletions(-)

diff --git a/synapse/lib/base.py b/synapse/lib/base.py
index 1507a11bfe8..8fe9d6bb7a0 100644
--- a/synapse/lib/base.py
+++ b/synapse/lib/base.py
@@ -590,7 +590,7 @@ async def main(self):
         await self.addSignalHandlers()
         return await self.waitfini()
 
-    def waiter(self, count, *names):
+    def waiter(self, count, *names, timeout=None):
         '''
         Construct and return a new Waiter for events on this base.
 
@@ -615,16 +615,17 @@ def waiter(self, count, *names):
             race conditions with this mechanism ;)
 
         '''
-        return Waiter(self, count, *names)
+        return Waiter(self, count, *names, timeout=timeout)
 
 class Waiter:
     '''
     A helper to wait for a given number of events on a Base.
     '''
-    def __init__(self, base, count, *names):
+    def __init__(self, base, count, *names, timeout=None):
         self.base = base
         self.names = names
         self.count = count
+        self.timeout = timeout
         self.event = asyncio.Event()
 
         self.events = []
@@ -656,6 +657,9 @@ async def wait(self, timeout=None):
                 doStuff(evnt)
 
         '''
+        if timeout is None:
+            timeout = self.timeout
+
         try:
 
             retn = await s_coro.event_wait(self.event, timeout)
@@ -676,6 +680,15 @@ def fini(self):
             self.base.unlink(self._onWaitEvent)
         del self.event
 
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, exc, cls, tb):
+        if exc is None:
+            if await self.wait() is None:
+                mesg = f'timeout waiting for {self.count} events {self.names}'
+                raise s_exc.TimeOut(mesg=mesg)
+
 class BaseRef(Base):
     '''
     An object for managing multiple Base instances by name.
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index 083c0ffd55b..7db50afc29d 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -182,45 +182,38 @@ async def test_lib_aha_basics(self):
                     self.nn(await proxy.getCellIden())
 
                 # force a reconnect...
-                waiter = aha.waiter(1, 'aha:svcadd')
                 proxy = await cryo.ahaclient.proxy(timeout=2)
-                await proxy.fini()
-                self.nn(await waiter.wait(timeout=6))
+                async with aha.waiter(2, 'aha:svcadd'):
+                    await proxy.fini()
 
                 async with await s_telepath.openurl('aha://cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
-                waiter = aha.waiter(1, 'aha:svcadd')
                 # force the service into passive mode...
-                await cryo.setCellActive(False)
+                async with aha.waiter(3, 'aha:svcdown', 'aha:svcadd', timeout=6):
+                    await cryo.setCellActive(False)
 
                 with self.raises(s_exc.NoSuchName):
                     async with await s_telepath.openurl('aha://cryo...') as proxy:
                         pass
 
-                self.nn(await waiter.wait(timeout=6))
-
                 async with await s_telepath.openurl('aha://0.cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
-                await cryo.setCellActive(True)
+                async with aha.waiter(1, 'aha:svcadd', timeout=6):
+                    await cryo.setCellActive(True)
 
                 async with await s_telepath.openurl('aha://cryo...') as proxy:
                     self.nn(await proxy.getCellIden())
 
-                # some coverage edge cases...
-                cryo.conf.pop('aha:leader', None)
-                await cryo.setCellActive(False)
-                # remove the admin flag from the account.
+            wait01 = aha.waiter(2, 'aha:svcadd')
 
-            wait01 = aha.waiter(1, 'aha:svcadd')
             conf = {'aha:provision': await aha.addAhaSvcProv('0.cryo')}
             async with self.getTestCryo(conf=conf) as cryo:
 
                 info = await cryo.getCellInfo()
-                cnfo = info.get('cell')
-                anfo = cnfo.get('aha')
-                self.eq(cnfo.get('aha'), {'name': '0.cryo', 'leader': 'cryo', 'network': 'synapse'})
+
+                self.eq(info['cell']['aha'], {'name': '0.cryo', 'leader': 'cryo', 'network': 'synapse'})
 
                 await wait01.wait(timeout=2)
 
@@ -986,13 +979,12 @@ async def test_aha_service_pools(self):
                     # Pool has no members....
                     pool = await s_telepath.open('aha://pool00...')
                     self.eq(0, pool.size())
-                    waiter = pool.waiter(0, 'svc:add')
 
-                    msgs = await core00.stormlist('aha.pool.svc.add pool00... 00...')
-                    self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('AHA service (00...) added to service pool (pool00.synapse)', msgs)
+                    async with pool.waiter(1, 'svc:add', timeout=12):
+                        msgs = await core00.stormlist('aha.pool.svc.add pool00... 00...')
+                        self.stormHasNoWarnErr(msgs)
+                        self.stormIsInPrint('AHA service (00...) added to service pool (pool00.synapse)', msgs)
 
-                    self.len(1, await waiter.wait(timeout=12))
                     prox = await pool.proxy(timeout=12)
                     info = await prox.getCellInfo()
                     self.eq('00', info.get('cell').get('aha').get('name'))
@@ -1017,17 +1009,15 @@ async def test_aha_service_pools(self):
                         replay = s_common.envbool('SYNDEV_NEXUS_REPLAY')
                         nevents = 5 if replay else 3
 
-                        waiter = pool.waiter(nevents, 'svc:add')
-
-                        msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...')
-                        self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs)
+                        async with pool.waiter(nevents, 'svc:add', timeout=3):
 
-                        msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...')
-                        self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs)
+                            msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...')
+                            self.stormHasNoWarnErr(msgs)
+                            self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs)
 
-                        await waiter.wait(timeout=3)
+                            msgs = await core00.stormlist('aha.pool.svc.add pool00... 01...')
+                            self.stormHasNoWarnErr(msgs)
+                            self.stormIsInPrint('AHA service (01...) added to service pool (pool00.synapse)', msgs)
 
                         poolinfo = await aha.getAhaPool('pool00...')
                         self.len(2, poolinfo['services'])
@@ -1050,24 +1040,21 @@ async def test_aha_service_pools(self):
 
                         waiter = pool.waiter(1, 'pool:reset')
 
-                        ahaproxy = await pool.aha.proxy()
-                        await ahaproxy.fini()
-
-                        await waiter.wait(timeout=3)
+                        async with pool.waiter(1, 'pool:reset', timeout=3):
+                            ahaproxy = await pool.aha.proxy()
+                            await ahaproxy.fini()
 
                         # wait for the pool to be notified of the topology change
-                        waiter = pool.waiter(1, 'svc:del')
+                        async with pool.waiter(1, 'svc:del', timeout=3):
 
-                        msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
-                        self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('AHA service (00.synapse) removed from service pool (pool00.synapse)',
-                                            msgs)
+                            msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
+                            self.stormHasNoWarnErr(msgs)
+                            self.stormIsInPrint('AHA service (00.synapse) removed from service pool (pool00.synapse)', msgs)
 
-                        msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
-                        self.stormHasNoWarnErr(msgs)
-                        self.stormIsInPrint('Did not remove (00...) from the service pool.', msgs)
+                            msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
+                            self.stormHasNoWarnErr(msgs)
+                            self.stormIsInPrint('Did not remove (00...) from the service pool.', msgs)
 
-                        await waiter.wait(timeout=3)
                         run00 = await (await pool.proxy(timeout=3)).getCellRunId()
                         self.eq(run00, await (await pool.proxy(timeout=3)).getCellRunId())
 

From 40c5e0478e7f65c40c0bf01ffb055a9dcc278261 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Sun, 7 Jul 2024 09:47:50 -0400
Subject: [PATCH 23/68] wip

---
 synapse/lib/nexus.py          |  7 ++++-
 synapse/telepath.py           |  1 +
 synapse/tests/test_lib_aha.py | 55 +++++++++++++++++++----------------
 3 files changed, 37 insertions(+), 26 deletions(-)

diff --git a/synapse/lib/nexus.py b/synapse/lib/nexus.py
index 8322f5d8f17..8418232e8a5 100644
--- a/synapse/lib/nexus.py
+++ b/synapse/lib/nexus.py
@@ -143,6 +143,9 @@ async def fini():
 
         self.onfini(fini)
 
+    def getNexsKids(self):
+        return list(self._nexskids.values())
+
     async def _migrateV1toV2(self, nexspath, logpath):
         '''
         Close the slab, move it to the new multislab location, then copy out the nexshot
@@ -635,12 +638,14 @@ def onfini():
 
     async def modNexsRoot(self, ctor):
 
+        kids = [self]
         if self.nexsroot is not None:
+            kids = self.nexsroot.getNexsKids()
             await self.nexsroot.fini()
 
         nexsroot = await ctor()
 
-        self.setNexsRoot(nexsroot)
+        [kid.setNexsRoot(nexsroot) for kid in kids]
 
         await nexsroot.startup()
 
diff --git a/synapse/telepath.py b/synapse/telepath.py
index 5c538bd8a03..388bc2b108c 100644
--- a/synapse/telepath.py
+++ b/synapse/telepath.py
@@ -1273,6 +1273,7 @@ async def offlink(self, func):
     async def _fireLinkLoop(self):
         self._t_proxy = None
         self._t_ready.clear()
+        await self.fire('tele:client:linkloop')
         self.schedCoro(self._teleLinkLoop())
 
     async def _teleLinkLoop(self):
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index 7db50afc29d..e6ebb5988c2 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -53,11 +53,12 @@ async def test_lib_aha_clone(self):
 
             async with self.getTestAha(dirn=dir0) as aha0:
 
+                ahacount = len(await aha0.getAhaUrls())
                 async with aha0.getLocalProxy() as proxy0:
-                    self.len(1, await proxy0.getAhaUrls())
-                    self.len(1, await proxy0.getAhaServers())
+                    self.len(ahacount, await proxy0.getAhaUrls())
+                    self.len(ahacount, await proxy0.getAhaServers())
 
-                    purl = await proxy0.addAhaClone('01.aha.loop.vertex.link')
+                    purl = await proxy0.addAhaClone('zoinks.aha.loop.vertex.link')
 
                 conf1 = {'clone': purl}
                 async with self.getTestAha(conf=conf1, dirn=dir1) as aha1:
@@ -70,7 +71,7 @@ async def test_lib_aha_clone(self):
                     serv0 = await aha0.getAhaServers()
                     serv1 = await aha1.getAhaServers()
 
-                    self.len(2, serv0)
+                    self.len(ahacount + 1, serv0)
                     self.eq(serv0, serv1)
 
                     # ensure some basic functionality is being properly mirrored
@@ -89,16 +90,17 @@ async def test_lib_aha_clone(self):
                     mnfo = await aha1.getAhaSvc('test.example.net')
                     self.eq(mnfo.get('name'), 'test.example.net')
 
-                    wait00 = aha0.waiter(1, 'aha:svcdown')
-                    await aha0.setAhaSvcDown('test', iden, network='example.net')
-                    self.isin(len(await wait00.wait(timeout=6)), (1, 2))
+                    async with aha0.waiter(1, 'aha:svcdown', timeout=6):
+                        await aha0.setAhaSvcDown('test', iden, network='example.net')
 
                     await aha1.sync()
+
                     mnfo = await aha1.getAhaSvc('test.example.net')
                     self.notin('online', mnfo)
 
                     await aha0.delAhaSvc('test', network='example.net')
                     await aha1.sync()
+
                     mnfo = await aha1.getAhaSvc('test.example.net')
                     self.none(mnfo)
 
@@ -323,16 +325,19 @@ async def test_lib_aha_basics(self):
 
                 conf = {'aha:provision': await aha.addAhaSvcProv('00.cell')}
 
+                # can't assume just one due to enterprise tests with raft...
+                ahacount = len(await aha.getAhaUrls())
+
                 async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell:
-                    self.len(1, cell.conf.get('aha:registry'))
+                    self.len(ahacount, cell.conf.get('aha:registry'))
 
-                await aha.addAhaServer({'host': '01.aha.loop.vertex.link'})
+                await aha.addAhaServer({'host': 'zoinks.aha.loop.vertex.link'})
 
-                self.len(2, await aha.getAhaServers())
+                self.len(ahacount + 1, await aha.getAhaServers())
 
                 async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell:
                     await cell.ahaclient.proxy()
-                    self.len(2, cell.conf.get('aha:registry'))
+                    self.len(ahacount + 1, cell.conf.get('aha:registry'))
 
     async def test_lib_aha_loadenv(self):
 
@@ -389,9 +394,11 @@ async def test_lib_aha_finid_cell(self):
 
                 proxy = await cryo.ahaclient.proxy()
 
-                await aha.fini()
+                # avoid race to notify client...
+                async with cryo.ahaclient.waiter(1, 'tele:client:linkloop', timeout=2):
+                    await aha.fini()
+                    self.true(await proxy.waitfini(timeout=10))
 
-                self.true(await proxy.waitfini(timeout=10))
                 with self.raises(asyncio.TimeoutError):
                     await cryo.ahaclient.proxy(timeout=0.1)
 
@@ -436,12 +443,12 @@ async def test_lib_aha_bootstrap(self):
                 }
 
                 async with self.getTestAha(dirn=dirn, conf=conf) as aha:
-                    self.true(os.path.isfile(os.path.join(dirn, 'certs', 'cas', 'do.vertex.link.crt')))
-                    self.true(os.path.isfile(os.path.join(dirn, 'certs', 'cas', 'do.vertex.link.key')))
-                    self.true(os.path.isfile(os.path.join(dirn, 'certs', 'hosts', 'aha.do.vertex.link.crt')))
-                    self.true(os.path.isfile(os.path.join(dirn, 'certs', 'hosts', 'aha.do.vertex.link.key')))
-                    self.true(os.path.isfile(os.path.join(dirn, 'certs', 'users', 'root@do.vertex.link.crt')))
-                    self.true(os.path.isfile(os.path.join(dirn, 'certs', 'users', 'root@do.vertex.link.key')))
+                    self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'cas', 'do.vertex.link.crt')))
+                    self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'cas', 'do.vertex.link.key')))
+                    self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'hosts', 'aha.do.vertex.link.crt')))
+                    self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'hosts', 'aha.do.vertex.link.key')))
+                    self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'users', 'root@do.vertex.link.crt')))
+                    self.true(os.path.isfile(os.path.join(aha.dirn, 'certs', 'users', 'root@do.vertex.link.key')))
 
                     host, port = await aha.dmon.listen('ssl://127.0.0.1:0?hostname=aha.do.vertex.link&ca=do.vertex.link')
 
@@ -471,10 +478,7 @@ async def test_lib_aha_provision(self):
 
         with self.getTestDir() as dirn:
 
-            conf = {
-                'aha:name': 'aha',
-                'dns:name': 'aha.loop.vertex.link',
-            }
+            conf = {'dns:name': 'aha.loop.vertex.link'}
             async with self.getTestAha(dirn=dirn, conf=conf) as aha:
 
                 ahaport = aha.sockaddr[1]
@@ -556,7 +560,8 @@ async def test_lib_aha_provision(self):
                     self.eq('00.axon', yamlconf.get('aha:name'))
                     self.eq('synapse', yamlconf.get('aha:network'))
                     self.none(yamlconf.get('aha:admin'))
-                    self.eq((f'ssl://aha.loop.vertex.link:{ahaport}?certname=root@synapse',), yamlconf.get('aha:registry'))
+
+                    self.eq(await aha.getAhaUrls(), yamlconf.get('aha:registry'))
                     self.eq(f'ssl://0.0.0.0:0?hostname=00.axon.synapse&ca=synapse', yamlconf.get('dmon:listen'))
 
                     unfo = await axon.addUser('visi')
@@ -583,7 +588,7 @@ async def test_lib_aha_provision(self):
 
                         teleyaml = s_common.yamlload(syndir, 'telepath.yaml')
                         self.eq(teleyaml.get('version'), 1)
-                        self.eq(teleyaml.get('aha:servers'), (f'ssl://aha.loop.vertex.link:{ahaport}?certname=visi@synapse',))
+                        self.sorteq(teleyaml.get('aha:servers'), await aha.getAhaUrls(user='visi'))
 
                         certdir = s_telepath.s_certdir.CertDir(os.path.join(syndir, 'certs'))
                         async with await s_telepath.openurl('aha://visi@axon...', certdir=certdir) as prox:

From 2c063ed7365c3d06e0c33dca090a56b975b26db8 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Sun, 7 Jul 2024 14:33:52 -0400
Subject: [PATCH 24/68] wip

---
 synapse/lib/aha.py            |  49 ++++++++++-
 synapse/lib/cell.py           |  33 ++++----
 synapse/tests/test_lib_aha.py | 151 ++++++++++++----------------------
 synapse/tests/utils.py        |   3 +-
 4 files changed, 117 insertions(+), 119 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 0fd1272dccd..9f9c03f2454 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -534,14 +534,15 @@ def getEnvPrefix(cls):
 
     async def _initCellBoot(self):
 
-        path = s_common.genpath(self.dirn, 'cell.guid')
-        if os.path.isfile(path):
-            return
-
         curl = self.conf.get('clone')
         if curl is None:
             return
 
+        path = s_common.genpath(self.dirn, 'cell.guid')
+        if os.path.isfile(path):
+            logger.info('Cloneing AHA: cell.guid detected. Skipping.')
+            return
+
         logger.warning(f'Cloning AHA: {curl}')
 
         async with await s_telepath.openurl(curl) as proxy:
@@ -788,6 +789,43 @@ async def _clearInactiveSessions(self):
         # Wait until we are cancelled or the cell is fini.
         await self.waitfini()
 
+    async def _waitAhaSvcOnline(self, name, timeout=None):
+
+        name = self._getAhaName(name)
+        print(f'WAITING FOR ONLINE {name}')
+
+        while True:
+
+            async with self.nexslock:
+
+                retn = await self.getAhaSvc(name)
+                if retn['svcinfo'].get('online') is not None:
+                    return retn
+
+                waiter = self.waiter(1, f'aha:svcadd:{name}')
+
+            if await waiter.wait(timeout=timeout) is None:
+                raise s_exc.TimeOut(mesg=f'Timeout waiting for aha:svcadd:{name}')
+
+    async def _waitAhaSvcDown(self, name, timeout=None):
+
+        name = self._getAhaName(name)
+        print(f'WAITING FOR DOWN {name}')
+
+        while True:
+
+            async with self.nexslock:
+
+                retn = await self.getAhaSvc(name)
+                online = retn['svcinfo'].get('online')
+                if online is None:
+                    return retn
+
+                waiter = self.waiter(1, f'aha:svcdown:{name}')
+
+            if await waiter.wait(timeout=timeout) is None:
+                raise s_exc.TimeOut(mesg=f'Timeout waiting for aha:svcdown:{name}')
+
     async def getAhaSvcs(self, network=None):
         path = ('aha', 'services')
         if network is not None:
@@ -1003,8 +1041,10 @@ async def setAhaSvcDown(self, name, linkiden, network=None):
 
     @s_nexus.Pusher.onPush('aha:svc:down')
     async def _setAhaSvcDown(self, name, linkiden, network=None):
+
         svcname, svcnetw, svcfull = self._nameAndNetwork(name, network)
         path = ('aha', 'services', svcnetw, svcname)
+
         if await self.jsonstor.cmpDelPathObjProp(path, 'svcinfo/online', linkiden):
             await self.jsonstor.setPathObjProp(path, 'svcinfo/ready', False)
 
@@ -1016,6 +1056,7 @@ async def _setAhaSvcDown(self, name, linkiden, network=None):
                 await link.fini()
 
         await self.fire('aha:svcdown', svcname=svcname, svcnetw=svcnetw)
+        await self.fire(f'aha:svcdown:{svcfull}', svcname=svcname, svcnetw=svcnetw)
 
         logger.info(f'Set [{svcfull}] offline.',
                         extra=await self.getLogExtra(name=svcname, netw=svcnetw))
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index b6281b42e76..c32898ab894 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1105,6 +1105,7 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
         self._reloadfuncs = {}  # name -> func
 
         self.nexslock = asyncio.Lock()
+        self.netready = asyncio.Event()
 
         self.conf = self._initCellConf(conf)
 
@@ -1186,6 +1187,11 @@ async def __anit__(self, dirn, conf=None, readonly=False, parent=None):
 
         await self._initCellSlab(readonly=readonly)
 
+        # initialize network daemons (but do not listen yet)
+        # to allow registration of callbacks and shared objects
+        await self._initCellHttp()
+        await self._initCellDmon()
+
         await self.initServiceEarly()
 
         nexsroot = await self._ctorNexsRoot()
@@ -1267,12 +1273,6 @@ async def fini():
         # initialize network backend infrastructure
         await self._initAhaRegistry()
 
-        # initialize network daemons (but do not listen yet)
-        # to allow registration of callbacks and shared objects
-        # within phase 2/4.
-        await self._initCellHttp()
-        await self._initCellDmon()
-
         # phase 2 - service storage
         await self.initServiceStorage()
         # phase 3 - nexus subsystem
@@ -1562,7 +1562,11 @@ async def initNexusSubsystem(self):
             if self.minfree is not None:
                 self.schedCoro(self._runFreeSpaceLoop())
 
-    async def initServiceNetwork(self):
+    async def _bindDmonListen(self):
+
+        # functionalized so Raft code can bind early...
+        if self.sockaddr is not None:
+            return
 
         # start a unix local socket daemon listener
         sockpath = os.path.join(self.dirn, 'sock')
@@ -1570,24 +1574,25 @@ async def initServiceNetwork(self):
 
         try:
             await self.dmon.listen(sockurl)
-        except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-            raise
         except OSError as e:
             logger.error(f'Failed to listen on unix socket at: [{sockpath}][{e}]')
             logger.error('LOCAL UNIX SOCKET WILL BE UNAVAILABLE')
         except Exception:  # pragma: no cover
             logging.exception('Unknown dmon listen error.')
-            raise
-
-        self.sockaddr = None
 
         turl = self._getDmonListen()
         if turl is not None:
             logger.info(f'dmon listening: {turl}')
             self.sockaddr = await self.dmon.listen(turl)
 
+    async def initServiceNetwork(self):
+
+        await self._bindDmonListen()
+
         await self._initAhaService()
 
+        self.netready.set()
+
         port = self.conf.get('https:port')
         if port is not None:
             await self.addHttpsPort(port)
@@ -2975,9 +2980,7 @@ def fini():
 
     async def _initCellDmon(self):
 
-        ahainfo = {
-            'name': self.ahasvcname
-        }
+        ahainfo = {'name': self.ahasvcname}
 
         self.dmon = await s_daemon.Daemon.anit(ahainfo=ahainfo)
         self.dmon.share('*', self)
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index e6ebb5988c2..24f82a28598 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -891,30 +891,26 @@ async def test_aha_util_helpers(self):
                     # This should teardown cleanly.
 
     async def test_aha_restart(self):
-        with self.withNexusReplay() as stack:
 
-            with self.getTestDir() as dirn:
-                ahadirn = s_common.gendir(dirn, 'aha')
-                svc0dirn = s_common.gendir(dirn, 'svc00')
-                svc1dirn = s_common.gendir(dirn, 'svc01')
-                async with await s_base.Base.anit() as cm:
+        with self.getTestDir() as dirn:
 
-                    aconf = {'dns:name': 'aha.loop.vertex.link', 'aha:network': 'synapse'}
+            ahadirn = s_common.gendir(dirn, 'aha')
+            svc0dirn = s_common.gendir(dirn, 'svc00')
+            svc1dirn = s_common.gendir(dirn, 'svc01')
 
-                    aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf)
-                    await cm.enter_context(aha)
+            async with await s_base.Base.anit() as cm:
 
-                    onetime = await aha.addAhaSvcProv('00.svc', provinfo=None)
-                    sconf = {'aha:provision': onetime}
-                    s_common.yamlsave(sconf, svc0dirn, 'cell.yaml')
-                    svc0 = await s_cell.Cell.anit(svc0dirn, conf=sconf)
-                    await cm.enter_context(svc0)
+                async with self.getTestAha(dirn=ahadirn) as aha:
 
-                    onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'})
-                    sconf = {'aha:provision': onetime}
-                    s_common.yamlsave(sconf, svc1dirn, 'cell.yaml')
-                    svc1 = await s_cell.Cell.anit(svc1dirn, conf=sconf)
-                    await cm.enter_context(svc1)
+                    async with aha.waiter(3, 'aha:svcadd', timeout=10):
+
+                        onetime = await aha.addAhaSvcProv('00.svc', provinfo=None)
+                        conf = {'aha:provision': onetime}
+                        svc0 = await cm.enter_context(self.getTestCell(conf=conf))
+
+                        onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'})
+                        conf = {'aha:provision': onetime}
+                        svc1 = await cm.enter_context(self.getTestCell(conf=conf))
 
                     # Ensure that services have connected
                     await asyncio.wait_for(svc1.nexsroot._mirready.wait(), timeout=6)
@@ -922,39 +918,22 @@ async def test_aha_restart(self):
 
                     # Get Aha services
                     snfo = await aha.getAhaSvc('01.svc...')
-                    svcinfo = snfo.get('svcinfo')
-                    ready = svcinfo.get('ready')
-                    self.true(ready)
+                    self.true(snfo['svcinfo']['ready'])
 
-                    # Fini the Aha service.
-                    await aha.fini()
-
-                    # Restart aha
-                    aha = await s_aha.AhaCell.anit(ahadirn, conf=aconf)
-                    await cm.enter_context(aha)
-
-                    # services are cleared
-                    snfo = await aha.getAhaSvc('01.svc...')
-                    svcinfo = snfo.get('svcinfo')
-                    ready = svcinfo.get('ready')
-                    online = svcinfo.get('online')
-                    self.none(online)
-                    self.false(ready)  # Ready is cleared upon restart / setting service down.
+                    online = snfo['svcinfo']['online']
+                    self.nn(online)
 
-                    n = 3
-                    if len(stack._exit_callbacks) > 0:
-                        n = n * 2
+                # Restart aha
+                async with self.getTestAha(dirn=ahadirn) as aha:
 
-                    waiter = aha.waiter(n, 'aha:svcadd')
-                    self.ge(len(await waiter.wait(timeout=12)), n)
+                    snfo = await aha._waitAhaSvcDown('01.svc...', timeout=10)
+                    self.none(snfo['svcinfo'].get('online'))
+                    self.false(snfo['svcinfo']['ready'])
 
                     # svc01 has reconnected and the ready state has been re-registered
-                    snfo = await aha.getAhaSvc('01.svc...')
-                    svcinfo = snfo.get('svcinfo')
-                    ready = svcinfo.get('ready')
-                    online = svcinfo.get('online')
-                    self.nn(online)
-                    self.true(ready)
+                    snfo = await aha._waitAhaSvcOnline('01.svc...', timeout=10)
+                    self.nn(snfo['svcinfo']['online'])
+                    self.true(snfo['svcinfo']['ready'])
 
     async def test_aha_service_pools(self):
 
@@ -1050,7 +1029,7 @@ async def test_aha_service_pools(self):
                             await ahaproxy.fini()
 
                         # wait for the pool to be notified of the topology change
-                        async with pool.waiter(1, 'svc:del', timeout=3):
+                        async with pool.waiter(1, 'svc:del', timeout=10):
 
                             msgs = await core00.stormlist('aha.pool.svc.del pool00... 00...')
                             self.stormHasNoWarnErr(msgs)
@@ -1071,83 +1050,57 @@ async def test_aha_service_pools(self):
                     self.stormIsInPrint('Removed AHA service pool: pool00.synapse', msgs)
 
     async def test_aha_reprovision(self):
+
         with self.withNexusReplay() as stack:
             with self.getTestDir() as dirn:
+
                 aha00dirn = s_common.gendir(dirn, 'aha00')
                 aha01dirn = s_common.gendir(dirn, 'aha01')
                 svc0dirn = s_common.gendir(dirn, 'svc00')
                 svc1dirn = s_common.gendir(dirn, 'svc01')
+
                 async with await s_base.Base.anit() as cm:
-                    aconf = {
-                        'aha:name': 'aha',
-                        'aha:network': 'loop.vertex.link',
-                        'dns:name': 'aha.loop.vertex.link',
-                    }
-                    name = aconf.get('aha:name')
-                    netw = aconf.get('aha:network')
 
-                    aha = await s_aha.AhaCell.anit(aha00dirn, conf=aconf)
-                    await cm.enter_context(aha)
+                    aha = await cm.enter_context(self.getTestAha(dirn=aha00dirn))
 
-                    onetime = await aha.addAhaSvcProv('00.svc', provinfo=None)
-                    sconf = {'aha:provision': onetime}
-                    s_common.yamlsave(sconf, svc0dirn, 'cell.yaml')
-                    svc0 = await s_cell.Cell.anit(svc0dirn, conf=sconf)
-                    await cm.enter_context(svc0)
+                    async with aha.waiter(2, 'aha:svcadd', timeout=6):
+                        purl = await aha.addAhaSvcProv('00.svc')
+                        svc0 = await s_cell.Cell.anit(svc0dirn, conf={'aha:provision': purl})
+                        await cm.enter_context(svc0)
 
-                    onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'})
-                    sconf = {'aha:provision': onetime}
-                    s_common.yamlsave(sconf, svc1dirn, 'cell.yaml')
-                    svc1 = await s_cell.Cell.anit(svc1dirn, conf=sconf)
-                    await cm.enter_context(svc1)
+                    async with aha.waiter(1, 'aha:svcadd', timeout=6):
+                        purl = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'})
+                        svc1 = await s_cell.Cell.anit(svc1dirn, conf={'aha:provision': purl})
+                        await cm.enter_context(svc1)
 
-                    # Ensure that services have connected
                     await asyncio.wait_for(svc1.nexsroot._mirready.wait(), timeout=6)
                     await svc1.sync()
 
-                    # Get Aha services
-                    snfo = await aha.getAhaSvc('01.svc.loop.vertex.link')
-                    svcinfo = snfo.get('svcinfo')
-                    ready = svcinfo.get('ready')
-                    self.true(ready)
-
-                    await aha.fini()
+                    snfo = self.nn(await aha.getAhaSvc('01.svc...'))
+                    self.true(snfo['svcinfo']['ready'])
 
                 # Now re-deploy the AHA Service and re-provision the two cells
                 # with the same AHA configuration
                 async with await s_base.Base.anit() as cm:
-                    aconf = {
-                        'aha:name': 'aha',
-                        'aha:network': 'loop.vertex.link',
-                        'dns:name': 'aha.loop.vertex.link',
-                    }
-                    name = aconf.get('aha:name')
-                    netw = aconf.get('aha:network')
 
-                    aha = await s_aha.AhaCell.anit(aha01dirn, conf=aconf)
-                    await cm.enter_context(aha)
+                    aha = await cm.enter_context(self.getTestAha(dirn=aha01dirn))
 
-                    onetime = await aha.addAhaSvcProv('00.svc', provinfo=None)
-                    sconf = {'aha:provision': onetime}
-                    s_common.yamlsave(sconf, svc0dirn, 'cell.yaml')
-                    svc0 = await s_cell.Cell.anit(svc0dirn, conf=sconf)
-                    await cm.enter_context(svc0)
+                    async with aha.waiter(2, 'aha:svcadd', timeout=6):
+                        purl = await aha.addAhaSvcProv('00.svc')
+                        svc0 = await s_cell.Cell.anit(svc0dirn, conf={'aha:provision': purl})
+                        await cm.enter_context(svc0)
 
-                    onetime = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'})
-                    sconf = {'aha:provision': onetime}
-                    s_common.yamlsave(sconf, svc1dirn, 'cell.yaml')
-                    svc1 = await s_cell.Cell.anit(svc1dirn, conf=sconf)
-                    await cm.enter_context(svc1)
+                    async with aha.waiter(1, 'aha:svcadd', timeout=6):
+                        purl = await aha.addAhaSvcProv('01.svc', provinfo={'mirror': 'svc'})
+                        svc1 = await s_cell.Cell.anit(svc1dirn, conf={'aha:provision': purl})
+                        await cm.enter_context(svc1)
 
-                    # Ensure that services have connected
                     await asyncio.wait_for(svc1.nexsroot._mirready.wait(), timeout=6)
                     await svc1.sync()
 
                     # Get Aha services
-                    snfo = await aha.getAhaSvc('01.svc.loop.vertex.link')
-                    svcinfo = snfo.get('svcinfo')
-                    ready = svcinfo.get('ready')
-                    self.true(ready)
+                    snfo = self.nn(await aha.getAhaSvc('01.svc...'))
+                    self.true(snfo['svcinfo']['ready'])
 
     async def test_aha_provision_longname(self):
         # Run a long network name and try provisioning with values that would exceed CSR
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index f1cf8019fc4..d346d59a482 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -1386,7 +1386,7 @@ async def getTestDmon(self):
             s_certdir.delCertPath(certpath)
 
     @contextlib.asynccontextmanager
-    async def getTestCell(self, ctor, conf=None, dirn=None):
+    async def getTestCell(self, ctor=s_cell.Cell, conf=None, dirn=None):
         '''
         Get a test Cell.
         '''
@@ -2019,6 +2019,7 @@ def nn(self, x, msg=None):
         Assert X is not None
         '''
         self.assertIsNotNone(x, msg=msg)
+        return x
 
     def none(self, x, msg=None):
         '''

From 08d3534a9633eb3074408f9a50091df84504f40f Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Sun, 7 Jul 2024 15:08:40 -0400
Subject: [PATCH 25/68] wip

---
 synapse/lib/aha.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 9f9c03f2454..d3ce45e1d39 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -792,7 +792,6 @@ async def _clearInactiveSessions(self):
     async def _waitAhaSvcOnline(self, name, timeout=None):
 
         name = self._getAhaName(name)
-        print(f'WAITING FOR ONLINE {name}')
 
         while True:
 
@@ -810,7 +809,6 @@ async def _waitAhaSvcOnline(self, name, timeout=None):
     async def _waitAhaSvcDown(self, name, timeout=None):
 
         name = self._getAhaName(name)
-        print(f'WAITING FOR DOWN {name}')
 
         while True:
 

From 97509359c1ba3a98564bb7d851d99e00e00a5a28 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 8 Jul 2024 05:22:38 -0400
Subject: [PATCH 26/68] wip

---
 synapse/tests/test_lib_base.py  | 5 +++++
 synapse/tests/test_lib_nexus.py | 4 ++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py
index 742438d8f0b..90563f7ee3d 100644
--- a/synapse/tests/test_lib_base.py
+++ b/synapse/tests/test_lib_base.py
@@ -201,6 +201,7 @@ async def woot(mesg):
         self.eq(data['count'], 1)
 
     async def test_base_waiter(self):
+
         base0 = await s_base.Base.anit()
 
         wait0 = base0.waiter(3, 'foo:bar')
@@ -224,6 +225,10 @@ async def test_base_waiter(self):
         evts = await wait2.wait(1)
         self.len(2, evts)
 
+        with self.raises(s_exc.TimeOut):
+            async with base0.waiter(1, 'newp', timeout=0.01):
+                await asyncio.sleep(1)
+
     async def test_baseref(self):
 
         bref = await s_base.BaseRef.anit()
diff --git a/synapse/tests/test_lib_nexus.py b/synapse/tests/test_lib_nexus.py
index e4b1521b155..8270e1a61c6 100644
--- a/synapse/tests/test_lib_nexus.py
+++ b/synapse/tests/test_lib_nexus.py
@@ -113,10 +113,10 @@ async def test_nexus(self):
 
     async def test_nexus_modroot(self):
 
-        async with self.getTestCell(s_cell.Cell) as cell:
+        async with self.getTestCell() as cell:
             await cell.sync()
             async with cell.nexslock:
-                cell.modNexsRoot(cell._ctorNexsRoot)
+                await cell.modNexsRoot(cell._ctorNexsRoot)
             await cell.sync()
 
     async def test_nexus_mixin(self):

From 2ee4285d1994c04f07f9a797628e4ec15e11d369 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 8 Jul 2024 07:57:49 -0400
Subject: [PATCH 27/68] duh

---
 synapse/tests/test_lib_base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py
index 90563f7ee3d..3bd9b1be99b 100644
--- a/synapse/tests/test_lib_base.py
+++ b/synapse/tests/test_lib_base.py
@@ -227,7 +227,7 @@ async def test_base_waiter(self):
 
         with self.raises(s_exc.TimeOut):
             async with base0.waiter(1, 'newp', timeout=0.01):
-                await asyncio.sleep(1)
+                pass
 
     async def test_baseref(self):
 

From 1e715c94adad73c580c9da1eaf309b2449ec39e6 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 8 Jul 2024 08:46:58 -0400
Subject: [PATCH 28/68] wip

---
 docs/synapse/deploymentguide.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index c4bc8b8b54f..26b16a9ae12 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -455,7 +455,7 @@ Once they are enrolled, they will have a user certificate located in ``~/.syn/ce
 configuration located in ``~/.syn/telepath.yaml`` will be updated to reflect the use of the AHA server. From there
 the user should be able to use standard Synapse CLI tools using the ``aha://`` URL such as::
 
-    python -m synapse.tools.storm aha://visi@cortex.<dns-network>
+    python -m synapse.tools.storm aha://visi@cortex...
 
 .. _deployment-guide-storm-pool:
 

From b01d3bfb201a7a9f498993ccd1f16f7b66aa375b Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 8 Jul 2024 10:36:13 -0400
Subject: [PATCH 29/68] speed up tests

---
 docker/rmlist.txt                             |  1 +
 synapse/tests/files/aha/certs/cas/synapse.crt | 28 ++++++++++
 synapse/tests/files/aha/certs/cas/synapse.key | 51 +++++++++++++++++++
 .../certs/hosts/00.aha.loop.vertex.link.crt   | 30 +++++++++++
 .../certs/hosts/00.aha.loop.vertex.link.key   | 51 +++++++++++++++++++
 .../files/aha/certs/users/root@synapse.crt    | 29 +++++++++++
 .../files/aha/certs/users/root@synapse.key    | 51 +++++++++++++++++++
 synapse/tests/utils.py                        |  6 +++
 8 files changed, 247 insertions(+)
 create mode 100644 synapse/tests/files/aha/certs/cas/synapse.crt
 create mode 100644 synapse/tests/files/aha/certs/cas/synapse.key
 create mode 100644 synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt
 create mode 100644 synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key
 create mode 100644 synapse/tests/files/aha/certs/users/root@synapse.crt
 create mode 100644 synapse/tests/files/aha/certs/users/root@synapse.key

diff --git a/docker/rmlist.txt b/docker/rmlist.txt
index 88d30b7a22b..382394960ab 100644
--- a/docker/rmlist.txt
+++ b/docker/rmlist.txt
@@ -1 +1,2 @@
 /usr/local/lib/python3.11/dist-packages/synapse/tests/files/certdir/
+/usr/local/lib/python3.11/dist-packages/synapse/tests/files/aha/certs/
diff --git a/synapse/tests/files/aha/certs/cas/synapse.crt b/synapse/tests/files/aha/certs/cas/synapse.crt
new file mode 100644
index 00000000000..ee927854c7f
--- /dev/null
+++ b/synapse/tests/files/aha/certs/cas/synapse.crt
@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIEvzCCAqegAwIBAgIRAIdu+9lQ3Z2nydXmG1/p2BAwDQYJKoZIhvcNAQELBQAw
+EjEQMA4GA1UEAwwHc3luYXBzZTAeFw0yNDA3MDgxMzAxNDBaFw0zNDA3MDYxMzAx
+NDBaMBIxEDAOBgNVBAMMB3N5bmFwc2UwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw
+ggIKAoICAQDQ8AFyEbiRUsKdRFlm6WS5X2cp9339H3A0qsXLSRYsCZGc6oJtLX0/
+52xPGCrKlXde2jxHU8DMDiEzQ3iVJjzDbJWXgAoNVfwrNVS1zjlSZyExVfRrCbzo
+XpGeuNAIY/WCpsbSxM7mCVPuEZtbX2rY4XmdyBWUpd4dqoDDIK2wU329daJEVNpd
+y9wdbZ2lLNPmKGS6Kb7Th3mio+sHpHEi6gcfYzH+mOFdWFVAh602OFMUp+fLq3nG
+MyFHHLDC2vvPqVWf+T7Z1E8k85Zqw43TSUTwdlG6u8egC6soKQsEze6oi4fMAt0l
+a4bgMMWDMIKJuQ4f6THEq5vKwPj0E78aT6C4C7qSKeHZSf8IOj7l3ukdgl7Sbrga
+JCjpL1V5+EB8LTpxjjK+TDbXUwU/eKIJp/I2i7dd//nMRs9Wr2k6aIvAt9PbopA2
+kG4tCS93TmzCza3TCtc4BruFHz92ToLfxvgcNq6N/BboDVjDIeAKtrWpe9VHPEIy
+Eh23tAPcgPXmlOBZj4DKJ5YoGNbwYQaFDrLIHh1JzRGQbAh00JuaqDPH3kb1Q6sm
+VjUx4QpX34qj231Bpfgu2eZX2kiLvtW+qPPfxWY2RCms1Yb+GMeKAE+9E8rak8Nq
+oPvdqgdUUbneW1PRgBaalu7xDALH/DQm5R+baly5PEYi8gk/vsyaQwIDAQABoxAw
+DjAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQB4Asv4TuAgF2vcOdxl
+mjPBwYueBDJrbbP2I6qjE9laO0xMyIN4Dp8ExbJyBKl/RlkSA/1cvJ7EyhlJnY1S
+Dps8EgKS5lQw0D/bFg8JQNgNec6jCxjfytRgYkhoikabz8HR2gmUIb0On6X+dvht
+L3Hv1/41Q4+Tig1JPRxP6C/hHcfLnCvmIsrMgKRbTGzb1AKfP8N93eZqDcA0nZo5
+Q0p+AdLQ6ilnh8895wQexoKs7mWImEzxUdTMZlf/YSQ9Nct3OKxtYg0IOk1KW9Fe
+sMstTDHg+/NJr2297vCmeNydYRGOTidLefebaNq0YnTm9cI3kPi/9etfUGeWHbrW
+aXXJCQ02uiWjrL9ykiifB6f0nve0m4ENWI11FypZ3nReMNDFIYSQfwlat6E/KTQ/
+5ih/NEgbY5KnMcUJqEKCBftfvfZyueE0frHClLZNi+YQNXD3ui6lmXzn8vmvaHeR
+dDfCifQ5TGHOlNmk9LMOv1Szx9CNswUwKDpSLklFp2lzI0JHTXm+NYCqeE+3cnrN
+vbsq55txmImj/Ekos5/kuv3Z2myJsDd2xT2eZ/QEkb7FOeO5RPrya3yXoSGNlkgs
+XG8vvCDNTeD6BJPWHjbu0ksZ6nxBFRsEmA2Ni8Y9ROUXUuJtwX+sYqHfAAVMLbiR
+wSujqDSopMoJSGr0hUT9kjbUSg==
+-----END CERTIFICATE-----
diff --git a/synapse/tests/files/aha/certs/cas/synapse.key b/synapse/tests/files/aha/certs/cas/synapse.key
new file mode 100644
index 00000000000..c734ec8f972
--- /dev/null
+++ b/synapse/tests/files/aha/certs/cas/synapse.key
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEA0PABchG4kVLCnURZZulkuV9nKfd9/R9wNKrFy0kWLAmRnOqC
+bS19P+dsTxgqypV3Xto8R1PAzA4hM0N4lSY8w2yVl4AKDVX8KzVUtc45UmchMVX0
+awm86F6RnrjQCGP1gqbG0sTO5glT7hGbW19q2OF5ncgVlKXeHaqAwyCtsFN9vXWi
+RFTaXcvcHW2dpSzT5ihkuim+04d5oqPrB6RxIuoHH2Mx/pjhXVhVQIetNjhTFKfn
+y6t5xjMhRxywwtr7z6lVn/k+2dRPJPOWasON00lE8HZRurvHoAurKCkLBM3uqIuH
+zALdJWuG4DDFgzCCibkOH+kxxKubysD49BO/Gk+guAu6kinh2Un/CDo+5d7pHYJe
+0m64GiQo6S9VefhAfC06cY4yvkw211MFP3iiCafyNou3Xf/5zEbPVq9pOmiLwLfT
+26KQNpBuLQkvd05sws2t0wrXOAa7hR8/dk6C38b4HDaujfwW6A1YwyHgCra1qXvV
+RzxCMhIdt7QD3ID15pTgWY+AyieWKBjW8GEGhQ6yyB4dSc0RkGwIdNCbmqgzx95G
+9UOrJlY1MeEKV9+Ko9t9QaX4LtnmV9pIi77Vvqjz38VmNkQprNWG/hjHigBPvRPK
+2pPDaqD73aoHVFG53ltT0YAWmpbu8QwCx/w0JuUfm2pcuTxGIvIJP77MmkMCAwEA
+AQKCAgAxPqgwkQGt6tIoy//AVDkjwdsoVodQ3hSNrURiNe8uYPD7iYBFKEgRhEOQ
+XtNTHShd6FT1wMU7swbbNMdabAE9VD3rz8dOvlnpey/ki98Rz3HQ1X/+rHRkVkm/
+HbMWjyzB5voMmktjh4ZLIcY6ooIl7PrDl/GSMAfqeRHRK8YUPZFw0qV0soUnP1G5
+c+kIkci9wf5/rDAoXhFqpnTSP81Um9Ei8jfJ2JGhdRze0TufgGYAg9SLufZBIzLw
+NlBpFMDuAGzIgC/ymmou/OSSdFXcmzPO8ywvNWwHCkkEdav9rWXXPs+6Y2BpHe9T
+rtsWoRvbRw0Ps2BCCOp2vsXOjUrohP4I2iKb5ExHAQoKmewro2b/txJw39rnaWa5
+/E9OG5hvbXEt0o5dfeLgtQLXcTsjTGqbOO3bi4dS42Zz9DdnnHf9oNes5odxW2lp
+TCPJQozZuzY9J5toEltz2RBgX4ECWaXE1UnY6E4nm1zkDRc1xx7V1bXp6MsXWYx6
+MO62tprWb+zFz7M6yIz7fsB9339X5Ky+xtDPKBJk8qnOVFpMbHkefpiHs6tPlmyu
+1wIjeL+3Om2pAd1nEPuk02pOhiurthiqI6wwHI8X03RnF5HU9tKUKLMCjnD1PJuM
+Ysqj7pDdaezRic+mrQAyBqebLzEcZ+PRwOpPVCX3lo2G/BD1bQKCAQEA8u70QHiX
+l/OlcxTG4nVn62ljsWCY3FYTyfDtWrP3uZQ7zaLF+wlGX7MXo416zVHPvW4YtCkG
+1n6KdaQguuSbE+CgvzyCmCLtSYiXuA35dVa9a6TDG5nPvwPJeT99HPMg1xRXxCaC
+f8ilJ6WILD9sG6383a0fWC7ZlXpIFB+ZXmNaj82vS421BrbciH+sLO9jk94Drxnw
+aXe81G6Rkabw3w852y+0TgqOoiVis4rCBpCL1e7Oz+s6+kmpNvc/4NI9LZTk2B/l
+M++9aTi11p4Z/1paDffOPinmgKXOhb1fud6+j7XbNmvU+mgDc+95H93zA2rp37CT
+CU9jo5i/hNUDFQKCAQEA3Czy3iKpvDKtsq9cDStfg1Ysr1raqOKHr4cS01M3u5Xa
+Ku3ORGkiVcQJAfgWik5nFfbzbiX2SNUFB9jLQvIGnAxRTen6empfCCds9+HNS4PI
+E8Byyepr9H6LFpqLr/GDrTA4cXWez8r4BMZwsuYa/cOEZt9Uoq97DFlCEwL4YDD2
+ZDNLSLry7LhH4XN0oYU+7imj4fKebWDY2m/qc+OJ0+xMDg/dioFfWaIW5UnpteNY
+6YZFTOygosSRrvEKGKnaszVpDRLPcHKcEzIKX8Naf03898R1m+dM1Atc/UZa9pet
+6kpZpqpKoGHrXg7rNWWdPyJfm4bH1MoNqjABUIhd9wKCAQAV6xtcicTbr974oCJF
+omQq6EpXYajJEHcenD8+FMjAFLDEn/AO80pHLihu2EABMGV26O0PrDfyuF4TuSg+
+1IttYrH+Lx51TYltPga6U4BzZs0WXjpATkNhL51I9EJ8jy8iWLKGfxb9IoRMLHI5
+08sUQEF1Wr5ePXPiObMxJZy32Gz+Vod/YJy5q1wAcMx/DWZFnB1m+gcn7Oa7n/JA
+WviWl5AXx5kUBX3TAV6DZnyVDQug1LgSKF4c4PKEhBBeX3mnmCyBl3cdlX7YdIZr
+g75CvMstQXN5RlyGtO8KQAjYA1HcM4NAyL/hi+rr1epuxp67azUIuqy5hVEvHIQD
+Hxj1AoIBAQC8knrIKiP5neYKvgo29UjusaW/4i6YqrvPZ/6FpCZ9sRCT5+zbxrez
+gRy95P9ZIWFE/KbtVfIj2t5eJB2ijqt+h0YzVwxCQEx4LVw0yd4MqSd5U0B9Exu2
+4ZK6n064OD+w2zXcZwLHsWzOmi736gCACy6g9PIGDAl1QBVJNygHKqg8lXoLJqLc
+f9CAlWP02qxVSrCj2io6P9I689N3wg/Pw/g3qvrxn3BM0niNlMpoD/mcuHUuNxQ1
+k+m6TZN6IC/BgSMiIVQtWNu3zQn5jtU5Z1Ab3NVl26p/iePwwIsz3CEGIvu5tOwJ
+hRQTEO/+YbNV2VjNWZhY9VzSwB7AHKttAoIBADaDQIZC+2JcOAZnLCwfToYb0Ga4
+Hs0NFcO8/EH7PNTTlhbD5bVPW/6IlTkSDemMp89zdYoB3EAfevWK5pMp5XOpQfsA
+Puc2gueO/ICSb76YcabAdnOyB73Kb2PvqQ7yl6YP9U9IK3PBuZbDxiOcsM1h1yfL
+XlMjDH08/e5efDBaxeVDTXS63haFXTgRKHYdnEm63AzeR0PXvpD/az++aaxoxkAV
+SV/zudxSzx/O8zrVjl6lz1vb35jQ1s4X2T8EfGEjuXAGx7yV9MaEwLWH/RhHAUgD
+vHoSEcB5Q3hbSj0QBqHiSkKcGn5VYP3kXSGMF2438flUahpdMpeP4QTa3tA=
+-----END RSA PRIVATE KEY-----
diff --git a/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt
new file mode 100644
index 00000000000..b1fc57b2310
--- /dev/null
+++ b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.crt
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFJTCCAw2gAwIBAgIRANebBnGvTlRt7CIGrH6WJagwDQYJKoZIhvcNAQELBQAw
+EjEQMA4GA1UEAwwHc3luYXBzZTAeFw0yNDA3MDgxMzAxNDFaFw0zNDA3MDYxMzAx
+NDFaMCIxIDAeBgNVBAMMFzAwLmFoYS5sb29wLnZlcnRleC5saW5rMIICIjANBgkq
+hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAvXp4PAfmdzu5t6ADh/l3iVTRYbpPVnQz
+7bySmQlhTtNQFRDOHgEyfttpd0zTWoQZBiQr9H4ZjXE5W3XHdptHcPE718rzBcP3
+G+DDIbp5nbkccUb7lvMVt39/4n9lK64K5YdnRop4SGsnps2WJvrLv/7qhgXPedjF
+Yh/E7iR54C+wns4aKvD2gZV9vYudu6PcB3S/b/r0Vqykpsy6dFL1bKTvJCPyD4b3
+ZAXpHr8Cws9Chae8DPt3PkUFCMnRl4opG7i8w/aJABRiXS9whccP1H0uGMvRF69P
+E1eSjXytwFcBUOI3rTwnAWBOfQ1zwRAQ6nCQsjH/CKZffrKro6M7kT+rfE0EiDPG
+kqf0owa+av4Rv2D0qRXhOBsq/jWgCtznk4286JweaRW8Omk+ViG5JO0Pxczp0bpg
+qiExqpJ/O8HTAEKbI9MB2+qRcUVnJanJ2KXmPUWdTtJQuEtCoLfDExudzOJgMwb8
+i2cmXTeVwas2QKA5pD+OOlGKciQc0T2xPIE/Gm/3CWRBs1wj9fqJ85NG9zbtCPpT
+7yuruYyiuRJTAo9s8D4AGaoeQLhQ8JE4Xkfol9Pva/UTIaHoKux6aHYsXBUbEeZb
+RqO3BGViV86Po+1+t2lBHHfD/s/YTnT7cjm+dYQdS9GqrwzZr7lp538lFkmFeX7t
+wTZakkXWIGMCAwEAAaNmMGQwEQYJYIZIAYb4QgEBBAQDAgZAMAsGA1UdDwQEAwIF
+oDATBgNVHSUEDDAKBggrBgEFBQcDATAJBgNVHRMEAjAAMCIGA1UdEQQbMBmCFzAw
+LmFoYS5sb29wLnZlcnRleC5saW5rMA0GCSqGSIb3DQEBCwUAA4ICAQDFSKMqM/A9
+maLx5zMO917Ysw1odf1DsemYeDjoZPRMSC/PVIjJnaZqB7UH3/mUxX0uqk9NMjLD
+aq0u8vJqNnjxBzFWfcb5vcH05WC1lBIJj9Bd/tR+0jXL31COqZ5usljv/uL74hII
+MH9qiRqDE7eQq7Mcxz6qbXwGGqY5570iOBfvLConR8zCy0WEuDf3yB8/8XKaDsxA
+6TiaY35OZgR6yNA7q+1rpBUGCsX14mJMOVWtIaY5V2F4RHc+7HzdkW0o8XDHk4lS
+Tq/SAfjFsGt1bDLgmJ3HtI+lUBZtW+uKLZVHNnV4v7rAebu4TLUtbZXl7eeUBWpF
+mSP2rIs0RbR7Y3jAXg4WMu4fIeZkP/jUvPpgUfkiF5jevwc0GvG7h8u+6ybg9nrS
+09vlTXc08CRgtp2eYTnsSR7aJCr0kYo7ZTWbeo96gf/vJIVJcNnRZnvvfl1vyrZT
+QK13S6TzUbF1oAVHMH5nSun200D2CSzyjOYDnGpTGIdfbamI2KsphyUS/sVCETDP
+f8fC01rRBpjFc7w+ubYuRyiXEJpzbRJe2CWQZPxq0CZ7Yifgz7hIot5/F1EuNE5z
+elh6vtjcudEoQjOgUI4HABQe9nI6TUDfAKPlo/ffB4iHMBmC49WaCdu/6gQs0s7j
+KjhaMLwW7zaGUYFkdR1vcn3qeFnAM9cmBQ==
+-----END CERTIFICATE-----
diff --git a/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key
new file mode 100644
index 00000000000..6db7bae2262
--- /dev/null
+++ b/synapse/tests/files/aha/certs/hosts/00.aha.loop.vertex.link.key
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEAvXp4PAfmdzu5t6ADh/l3iVTRYbpPVnQz7bySmQlhTtNQFRDO
+HgEyfttpd0zTWoQZBiQr9H4ZjXE5W3XHdptHcPE718rzBcP3G+DDIbp5nbkccUb7
+lvMVt39/4n9lK64K5YdnRop4SGsnps2WJvrLv/7qhgXPedjFYh/E7iR54C+wns4a
+KvD2gZV9vYudu6PcB3S/b/r0Vqykpsy6dFL1bKTvJCPyD4b3ZAXpHr8Cws9Chae8
+DPt3PkUFCMnRl4opG7i8w/aJABRiXS9whccP1H0uGMvRF69PE1eSjXytwFcBUOI3
+rTwnAWBOfQ1zwRAQ6nCQsjH/CKZffrKro6M7kT+rfE0EiDPGkqf0owa+av4Rv2D0
+qRXhOBsq/jWgCtznk4286JweaRW8Omk+ViG5JO0Pxczp0bpgqiExqpJ/O8HTAEKb
+I9MB2+qRcUVnJanJ2KXmPUWdTtJQuEtCoLfDExudzOJgMwb8i2cmXTeVwas2QKA5
+pD+OOlGKciQc0T2xPIE/Gm/3CWRBs1wj9fqJ85NG9zbtCPpT7yuruYyiuRJTAo9s
+8D4AGaoeQLhQ8JE4Xkfol9Pva/UTIaHoKux6aHYsXBUbEeZbRqO3BGViV86Po+1+
+t2lBHHfD/s/YTnT7cjm+dYQdS9GqrwzZr7lp538lFkmFeX7twTZakkXWIGMCAwEA
+AQKCAgAgRonPk/ruiYZvoHqpgVWa149ZCc7055Nm5i3EksP4FOe5xuSNWN/cmwxi
+jXwdGY5XrPatzYMVxFkkWrIw4m9vbjAm6IOwEjr4DTe/+Y84zizpoNE/W8XxvW6v
+ysqVf66MfZ2adwDZOSOGdtOibSsi183kKX43f7TTq5y0ghMenJEF5A6yDNy4oxnJ
+nUwvh9B1lq37abCQSRU88ne6U91Jdejka5kSiwd+CsG0go36WCq5MKLIRVeBDGm4
+nwQsP5UUC0pgSRD6Kf69Z9TPfOKV7ALbp3BFDBA4t7yXjErejhODzxzzzeDJC7oi
+9BUpKE7xWF5VdE3Aj/KJVu8Ez0vYfTTr+IvhAOi6mOxmqYhcODzwehwiD5o0CiG8
+Igb3YzZjvgjjIhX0n5GsKSOWlxjXOMiuB221WvLIx/qlCXCJzR71XoX4YwPYn7GP
+LjjTVrMiLqN8UNEZ1AYH1sP9xlL/Dzg164EMBsRUIgQJo7DHXtpcEEpioyDRfLmm
+LsoT3MaNsd9EoNDKlTpNz1BrJtJR0GAQCvBOKOCIpXihiLRWj3d9gFe4ZO4JdNOd
+B+m1eZzXltus3Rc/7tGGKAfNt212IGKOalZsIpYN40C7C/iFr2miST6SUyBLhVaj
+3zbexSIbgj7QfT6IIMYZYrZBb2FfBALJa58ye/enNRGlcnFtwQKCAQEA8h7COxN8
+uotwC4T2PIFzjusGGCeAKx6PuRKfl4P8nzBic4BjvbKVrLc2W4hqqAu15uk5RkAk
+sdjd2IFpQzPsdbUTmKWNKYDclpMctqTifdxyjOSxIx9Hg22UkZKbVKyrjHKvlKkx
+o2E+p6XijAc8LvFLty7pSAe3uHv61bzEmEc+rJEoH+k5sQ1TQP43rbOLjvWNz4oS
+gFv1/437+uFaMglsgfP14fBYwaTQZc3mPIEbeO3e2D95kZ6+ptOnUagS+LDNNcCk
+F0YrFePPDP2GKZPLjfqoN4EGASSdXogcMDdQfVM/BmiZFYWAmSPKzwzm874mSRZA
+fXyNxH+0FsMzowKCAQEAyFcqVXTbKTRzTgssjnRIrfgwp7bDT9xoMG4PWPqrNKax
+f/Kg8CU/iJErkcriwcbheWKxKy6d8MgDcMZuDEM4gsfQxIeTLUPeiNvVCdtk8x0/
+twaX7R7KBRWR5Q4/SGZqoI5ZVZTNROpn1YwFlhJJXdJv0l0LxF2NGYSLXIqG99g2
+2kOlVBeAm4evoY6xHsp9dxVa41Bc3+KK/vrZezF9bQTq2NmtHyIUJ2jIMlgr+eAc
+Bq+yJLLHQ3c1NR8jKNK76E59dUkawsxPnSEc0KWh3g3L2udxy/+1eE3r+jj0jCII
+2MoiTFVCNnJ7LMdsGISkax5ZJX1blUflikIA72IsQQKCAQEAwLQsgRp8fni2f+Se
+mv+pOsniOt1NjIQxferNrKk3Kng3E5jPSc9Wg3X6xJVp1kAj0ho0JK6uxgJGZ6hw
+YDV2cSTi6O5y0OKoLwv9oXzQa75GSc9HER43K+rOgaJ/EMCxdQJeruKPCGtAk+xa
+yHqFsxMH4U9sCpFh72p19SHeExk5T93kYqmc6kchySvMouqxG+JisRlCqnkG7RRT
+xpUP1Z1ciH3kaKSD7/O+jhh3tBZKCFDCubijiHwhX+Q7Wql8GAWX/r1JnOCTMEP1
+qnAqFPN14pXqxuphHg3HVtLcJKAR5v2XvwEHPnLYLIqpQ2wQcVUZYbhdMcMtjoTZ
+j/hjIwKCAQEApmGdyvMNwJ7K1Bn7myN/6NuirObgNkb6UJ5XKLKl1UhLSdObTVXh
++e12ndI9mGkvgLwyH4bLrNiv4s0pQA3jtNl1zII7/O/MtSS9PT50DGRSMhLLwiY7
+6RUM4Yp/jAVisI0ILEc0YvO54GQ1j3kIbV8Dd1XHHAIF2Rd3FhgGF3f9ti9P8xLB
+wGljt2zmNIg+wtN9dCOdvmJKxZBXZjSn0g6vbAD8AksvKbuf6A/KFe/F1te7vzaq
+vqEWE1QUwyag4EGvd+SK0RUVWY3SfIXSdLRIhTiKDb4EXDF6tYjvsCHj7weQjIyS
+PN2+5mWIpKQkWMIPj08Y7FWVkMlYNXb3AQKCAQEAuXMzr9YD8r3kaF7vQmRiF3f7
+725mWjjBwcKrUW/ZgLMkfx2Pcn9Lt+1+ULVcEGwIuI8eoS/9LhI4WR6j6H8kkvOH
+TQ//5ZFiOJS2MaS9kSEVs5RVu0MhaWshrCiYOIcq3z1v4AiSxh/jNGc3Uc3eEiMy
+sxPA3YLKPahPTt3QEtwRagCsgFfBvj52RJg8TON3ZQoOACeUSRJlCgV7mxh+gRor
+1D8eevuSd+7IRlWnM7UCQQTtNie5mbOgd3fRuNI+vGFIwQnHmkg55iFE5TCRg4oS
+UeOhOVF+5BZp7qOn9j3FVWzkaGwfIredFWnfh9oENIW5orIGB/SbwEfWm8UtUw==
+-----END RSA PRIVATE KEY-----
diff --git a/synapse/tests/files/aha/certs/users/root@synapse.crt b/synapse/tests/files/aha/certs/users/root@synapse.crt
new file mode 100644
index 00000000000..7b73a8def75
--- /dev/null
+++ b/synapse/tests/files/aha/certs/users/root@synapse.crt
@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE9jCCAt6gAwIBAgIRAJGsran8ISfoaXuIMN7vAIAwDQYJKoZIhvcNAQELBQAw
+EjEQMA4GA1UEAwwHc3luYXBzZTAeFw0yNDA3MDgxMzAxNDFaFw0zNDA3MDYxMzAx
+NDFaMBcxFTATBgNVBAMMDHJvb3RAc3luYXBzZTCCAiIwDQYJKoZIhvcNAQEBBQAD
+ggIPADCCAgoCggIBAL42zY6E1q0RJIr9j+T6bPb7/f0RbwLJ2hza0/4qjAisNGQs
+t1WkWvpNR/l0+JHmXw78KQ1k8/WP+niIXZmakZdrMNQOf+BBY+MoNqFHtA1KoP0p
+b7f0hyQyRprYihSllA3uFZEagaylx9z5TliwclnCvB77whRZFU8f5PXH+wibWFvJ
+n4ycdFVLg93Rb6IiliMytc3CA4TEemDiL5410nZNipWGcYRmTUruWZCADYLSh6D+
+VJvY/sfoteBX1cvhCdL6DORKaxZnXsPthzkDPJYgLn+1E9hJ4vhuqgXIJ7VjqrPZ
+0wJtciZYb9YcJM6OqiWavW8AeLOEySEKO4NEGU0JduFpu4/9xnfhv/Atdv9YwK6j
+mJmxl/bV9dPhjszNG/D+RbCeBdhWebL1zcnt8indigLxSI9UgulBnjCJg4xHrJPO
+9tiXPi4ipFL4tIpiW4NYCWnJXAkCNgPBFGeFaEmzAxoiSh/iBIMfUgfMV26HzOUS
+2MAzq04/2nH4MjsZYtIKNx8sNZvC3QnDVH1ncGC8JcbSVZPH2jBmMRJJjhxDT7wM
+JbgJJomJ1mIP3a03FJpLTCxP8IbBTmq3TuOpqmhmiVHjBunACbNf6l6fhOJG3pMk
++VpWtIUiKyNMXvjxgM5bKi6sx1ivInxmPk0f2BPBTrYeLFCTj711A4XuHxSzAgMB
+AAGjQjBAMBEGCWCGSAGG+EIBAQQEAwIHgDALBgNVHQ8EBAMCB4AwEwYDVR0lBAww
+CgYIKwYBBQUHAwIwCQYDVR0TBAIwADANBgkqhkiG9w0BAQsFAAOCAgEANZtBnHeo
+yatrA/s2BzOU0k4WGEUaFMrCi6cDyVAWQ6mhcQHqJ2C8Yi2xCd5Si2BEC0hiyJ4K
+wOpi/bkoyAh67bX/RchzqLTwfUYyM17ZH5yiqbPufjpJQAitAVaywCI0jXDpaYlM
+o2u6b3u8eUKhlfHSIzTJn1sVc0p0Ql0okhrHwwUfwN1lwkVDR/e+jF5GKAwjnkNc
+yu3yYYz0oHAg6ZOcQaaFASX7oBBpYnHfDvou3uoJTGfKevRgI5NwQDRG3gipudSi
+vIwWR2iyBjTZaICChmaBX7zVHLwOFN0LRv73tgEBcJwZ4oCvE64POpAL2pO+U52d
+fIFlIOPSuILtTTtBRvfTuIjEvVCF44g4JkPgPpaN9yPcdAOnBQkP7RkrMlQ1pKZ3
+S6uZh9Ar0jNx59ijYq2Y8o/yyEVveXIKbp3HLogfcLeEWUqjL3syP72WkfBWRWx9
+QELaYrFFxgtT4bKZSmyK4C4qwgSknm89Xp/JZFyKWAjYEmu9BAHDI29IBVbLzQAS
+ezrtG2CmyhejnGhwSMYNmv+dC+ahSeP5FQiixrrxaTJ0s3Ax5mvbQQqv9lBvkn9Z
+WpVNN5MHdHEWnvbK4kfFnVQIBfv3lvPl0C0vk3VSWZexXIOjNtkUK6/ALHV2n5AA
+BEJmC3OPyT3eNrlr7gCxuMoV6uDMeTCRpTY=
+-----END CERTIFICATE-----
diff --git a/synapse/tests/files/aha/certs/users/root@synapse.key b/synapse/tests/files/aha/certs/users/root@synapse.key
new file mode 100644
index 00000000000..f426d5dac89
--- /dev/null
+++ b/synapse/tests/files/aha/certs/users/root@synapse.key
@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKgIBAAKCAgEAvjbNjoTWrREkiv2P5Pps9vv9/RFvAsnaHNrT/iqMCKw0ZCy3
+VaRa+k1H+XT4keZfDvwpDWTz9Y/6eIhdmZqRl2sw1A5/4EFj4yg2oUe0DUqg/Slv
+t/SHJDJGmtiKFKWUDe4VkRqBrKXH3PlOWLByWcK8HvvCFFkVTx/k9cf7CJtYW8mf
+jJx0VUuD3dFvoiKWIzK1zcIDhMR6YOIvnjXSdk2KlYZxhGZNSu5ZkIANgtKHoP5U
+m9j+x+i14FfVy+EJ0voM5EprFmdew+2HOQM8liAuf7UT2Eni+G6qBcgntWOqs9nT
+Am1yJlhv1hwkzo6qJZq9bwB4s4TJIQo7g0QZTQl24Wm7j/3Gd+G/8C12/1jArqOY
+mbGX9tX10+GOzM0b8P5FsJ4F2FZ5svXNye3yKd2KAvFIj1SC6UGeMImDjEesk872
+2Jc+LiKkUvi0imJbg1gJaclcCQI2A8EUZ4VoSbMDGiJKH+IEgx9SB8xXbofM5RLY
+wDOrTj/acfgyOxli0go3Hyw1m8LdCcNUfWdwYLwlxtJVk8faMGYxEkmOHENPvAwl
+uAkmiYnWYg/drTcUmktMLE/whsFOardO46mqaGaJUeMG6cAJs1/qXp+E4kbekyT5
+Wla0hSIrI0xe+PGAzlsqLqzHWK8ifGY+TR/YE8FOth4sUJOPvXUDhe4fFLMCAwEA
+AQKCAgAO2f6QuyR73NPKmmOq0DbDzgcVxE+zmwkRqbBVrwLCBpgMnkUuRafo1THi
+c1SZJ7CSXfPinNhDextmX9dXD++AMBle5Ubsvo5LBS/Gxe9z/ji1q4+SgGAw3lAO
+9QtK82as88LxUm1/im4hfcG1QQmcoozHqoaLyizUwbvByPx7vo1WlVaExw56Pmws
+XJbnxOWOF/6D8zsNGb4oZGCtbKxjGkjQxe+FE+vDBvstQiJL7Z4zXshCZt31w+C3
+hIQn6Ak5VNa+2GSmGioXCCu/stX/LsISrJPBFnvHQ/S4gNlA22hPpJlKHjuEbPFG
+eWoSSqdUSlGyIgpKkfqWPtNIu39yFtj2GKQPotwnGA9CVk4HjjEpBEYBSz8cGGvc
+jw9/YjUUEpna8Ab+RJcUJ0Oc+RdMZvnmEtGs2w2dHHXFWU/0nMe7oBeSR1DLYRW3
+X++je8IqLDvHVZ6+LgcdQOcMM5Myw9rPsu89+YOf76wtOwb8T014XmoLyNC0sulN
+WbEdx49yQSkSX0lFq9fvSmfaptl+2PXIAqoRmK5Vkg2f/OQBmFX9GM6mLLXR/eMH
+kurXGrnZ/RRqb3TIO1Gy+v7vTCSL0nZC4krQj1AWlUA3lhTEIo9qJtlL2D+e1pkR
+Ujhcz0U73SAFbkVOMzv/loCsSIRHdfhh4vRNEGZRW6lx6xdxiQKCAQEA7KMqiDJt
++xn0hQjWOBL8nnbNiXKKNU+ENwedUXnmZnkpaWbXEh5+gbmUsSNwTvVCncQbjh/a
+sNwvVi2yalSEaQIji7PUmOJV4Pw/UXhEaGYhsQqs/snnKpnfzqdHXcZhhqmVXALm
+eVJPYjvmt49obd91lC+sCc/iH+wkn1Anj8UvAlU8bXoUNEwR5ZcNG9Rc8QgfpMTE
+CM0lgjYqcxiJ65YUUoEuz8QeAteRPdSNgIDy3A1q4PbPyFwjwAQYAxW4BhEyqnJx
+EAf8BMZw4t4qbaN/j5Nc+KzyRmA0vcQVzReZj3cRo+qx9gkiDDsD0zz/Nx2iYFCl
+XrQwGquq0VMZXwKCAQEAzcc1uaeJbMvvEI2OphNroMmxmkXM++M4qQnhXeZAHkU1
+9lTKvmMCM1arVW/anIXb4PPUCkT9C5a1adHCW43KPzR/x9Rh2q8oqNy8Eb9HdRkn
+5gjQogsQJLC9CSyuwdpSrtjqZZMSv533vRSu53GoCCDIM9iL8MyWhMo3yOzouoh7
+09Xm7fpeU7OU8PGqmCWcT1ykNDcyKp4Qn+FjfAGt8RX81dJp8Bd9s97fu1oFK1jv
+FHjDiofdFveqVucFuxnEydOohhcnyKOEKLiGHH1iFP0BKNzGTswiboiMCPFJUgkX
+3F18Rpx9eM0qzHeD38OOGTu9kwYK7XJaRiQlh33BLQKCAQEAkWtL7dqfv5mZrE9b
+5aW1XGRBtt+Ok2hEJdUmFjXFIt/+VOl/7YCT2YCEIb9Xew45W95x2Is2x0zoQhte
+8vzxSd6onWdrlHAyukoJVzWDRGzO849N9F17E7Dv2nzt3HDW6fw47wxROekdI48Q
+H2mfkTWred855+W6Xket0cFeLnnMg7CBq/1DgUfmD6MdySZd3zlQyN6qYYZAMJ68
+w3Im/4GBB5qfe8pXbwOuG8MNGhUkKZqNPaYu1j/ZMFzTrMGiDwhujFQ7qtBdTUnt
+gV1p0WqSL1ct4RvW3uysPJnNk/WWxpUzX/oQVtQR+lYh6aQoamRprqqglWxBy4IH
+PQ/GHQKCAQEAgGfBl6L6tyAegobCr9FIhOG26JyLx9ZG4fxfXBe0hO8NODQ6+0iO
+8/guG7cbhDZnOV0NQIdCG7wkYe7ZSAYXYQ9ieRCHK03Hom9zD+P7NA/JfNqdTbB3
+l12N6C6wshiJJcAHq/B1b4qkL6G7boLWppPGOcvzrwQLHLuOydtDcjp+gnZkjrV/
+rN1PoHY8zIoTmtYlRMv4iu1tVEbxrEzj/J+K0OZm4sHV16rIk4Ed3qm1LmIZKtWS
+2aUb7vRj+BehW4uvClVipumaK0rRrO90h8JEufnh6QOqrKyaLW5bIUy5TfIr/Wfj
+R8FG5qXhbXz6q7ZCmn7XzMlhmYkuvfhz1QKCAQEAvCLx84l0WONH1jq0XTXssQev
+CVj0k3izQ4/KRsEJT8elWU/94dMqS2vLfhaF2/ySJkH7/Dr+UpmUbLZa6xlO1Rp0
+Vw0EuYlWgoSvD11YGC+8LoSjNgcePyQFE0Kv+i7UHBGibI23ooyOGbap/Nq94Fhk
+mI9JNKgiQ1G4sqkDpfvNEb5a95OdWDCKwZWdaRttYmu5fb1xxpTur+r0QrsJR81y
+FgFbV9OU1IBI8HirMO5JpTicenatszi+fCif0ldDAnkiM6apzuqOuM7y9mlWt6cg
+8HQuTwBKqTNbj35Red79YcdBnPbLkbwPWi6NfPdQpC3icEDlwOzVjiMFOuex+Q==
+-----END RSA PRIVATE KEY-----
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index d346d59a482..7ed11691b00 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -1465,6 +1465,12 @@ async def getTestAha(self, conf=None, dirn=None, ctor=None):
 
             with self.mayTestDir(dirn) as dirn:
 
+                if conf.get('aha:network') == 'synapse':
+                    dstpath = os.path.join(dirn, 'certs')
+                    if not os.path.isdir(dstpath):
+                        srcpath = self.getTestFilePath('aha/certs')
+                        shutil.copytree(srcpath, os.path.join(dirn, 'certs'))
+
                 async with await ctor(dirn, conf=conf) as aha:
 
                     mods = {}

From f9489ecfb244616925217375b7e0e469ef93c71b Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 9 Jul 2024 11:17:27 -0400
Subject: [PATCH 30/68] WIP: AHA Leader Terms

---
 synapse/lib/aha.py     | 94 ++++++++++++++++++++++++++++++++++++++++++
 synapse/lib/schemas.py | 13 ++++++
 2 files changed, 107 insertions(+)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index d3ce45e1d39..9270d0a62c5 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -61,6 +61,9 @@
 }
 provSvcSchema = s_config.getJsValidator(_provSvcSchema)
 
+STATE_LEAD = 0      # lead...
+STATE_FOLLOW = 1    # follow...
+STATE_INVALID = 2   # or get out of the way!
 
 class AhaProvisionServiceV1(s_httpapi.Handler):
 
@@ -580,6 +583,8 @@ async def fini():
         self.slab.initdb('aha:provs')
         self.slab.initdb('aha:enrolls')
 
+        self.slab.initdb('aha:leadterms')
+
         self.slab.initdb('aha:clones')
         self.slab.initdb('aha:servers')
 
@@ -1255,6 +1260,95 @@ async def signUserCsr(self, csrtext, signas=None):
 
         return self.certdir._certToByts(cert).decode()
 
+    async def getLeadTerm(self, iden):
+        '''
+        Get the current leader term for the specified service cluster.
+        '''
+        lkey = s_common.uhex(termdef.get('iden'))
+        byts = self.slab.get(b'\x00' + lkey, db='aha:leadterms')
+        if byts is not None:
+            return s_msgpack.un(byts)
+
+    @s_cell.from_leader()
+    async def setLeadTerm(self, termdef):
+        '''
+        Set the current leader term for the specified service cluster.
+        '''
+        # TODO schema
+        termdef['time'] = s_common.now()
+        s_schema.reqValidLeadTerm(termdef)
+        return await self._push('aha:lead:set', termdef)
+
+    @s_nexus.Pusher.onPush('aha:lead:set')
+    async def _setLeadTerm(self, termdef):
+        s_schema.reqValidLeadTerm(termdef)
+        lkey = s_common.uhex(termdef.get('iden'))
+        self.slab.put(b'\x00' + lkey, s_msgpack.en(termdef), db='aha:leadterms')
+
+    @s_cell.from_leader()
+    async def nextLeadTerm(self, iden, name, nexs):
+        '''
+        Force a change in leadership for the given cluster iden.
+        '''
+        async with self.nexslock:
+
+            term = 0
+            leadterm = self.getLeadTerm(iden)
+            if leadterm is not None:
+                term = leadterm.get('term') + 1
+
+            newterm = {
+                'iden': iden,
+                'name': name,
+                'term': term,
+                'nexs': nexs,
+            }
+            return await self.addLeadTerm(leadterm)
+
+    @s_cell.from_leader()
+    async def mayLeadTerm(self, iden, name, term, nexs):
+        '''
+        Determine if the caller is still the valid leader.
+        Returns: (<status>, <termdef>) tuple.
+
+        The <status> may be one of the following values:
+        STATE_LEAD: you are the leader
+        STATE_FOLLOW: you must become a follower
+        STATE_INVALID: you are divergent and must reprovision.
+        '''
+
+        # we hold the nexus lock so this may only be run on the leader
+        async with self.nexslock:
+
+            leadterm = await self.getLeadTerm(iden)
+            if leadterm is None:
+                leadterm = {
+                    'iden': iden,
+                    'name': ahaname,
+                    'term': term,
+                    'nexs': nexs,
+                }
+                await self.addLeadTerm(leadterm)
+                return (STATE_LEAD, {})
+
+            # if we were the last known leader, we can jump right in
+            if leadterm.get('name') == ahaname:
+                return (STATE_LEAD, leadterm)
+
+            # has someone else taken the lead in our absense?
+            if leadterm.get('term') > term:
+
+                # if someone else was promoted and their nexs index
+                # was less than ours at the time, we are divergent :(
+                if leadterm.get('nexs') < nexs:
+                    return (STATE_INVALID, leadterm)
+
+                # if they took over but had an equal nexs index we can
+                # become a follower...
+                return (STATE_FOLLOW, leadterm)
+
+            return (STATE_INVALID, leadterm)
+
     async def getAhaUrls(self, user='root'):
 
         # for backward compat...
diff --git a/synapse/lib/schemas.py b/synapse/lib/schemas.py
index 95d1b0bdd97..22228328008 100644
--- a/synapse/lib/schemas.py
+++ b/synapse/lib/schemas.py
@@ -3,6 +3,19 @@
 import synapse.lib.msgpack as s_msgpack
 
 
+leadTermSchema = {
+    'type': 'object',
+    'properties': {
+        'iden': {'type': 'string', 'pattern': s_config.re_iden},
+        'name': {'type': 'string'},
+        'term': {'type': 'number'},
+        'nexs': {'type': 'number'},
+        'time': {'type': 'number'},
+    },
+    'required': ['iden', 'term', 'nexs', 'name', 'time'],
+}
+reqValidLeadTerm = s_config.getJsValidator(leadTermSchema)
+
 easyPermSchema = {
     'type': 'object',
     'properties': {

From be760d2a8286b78d1cb38b23a86b39e74d3065ca Mon Sep 17 00:00:00 2001
From: invisig0th <visi@vertex.link>
Date: Tue, 9 Jul 2024 12:50:12 -0400
Subject: [PATCH 31/68] Update docs/synapse/deploymentguide.rst

Co-authored-by: Cisphyx <cisphyx@vertex.link>
---
 docs/synapse/deploymentguide.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 26b16a9ae12..e92774e9830 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -83,7 +83,7 @@ Choose an AHA Network Name
 --------------------------
 
 Your AHA network name is separate from DNS and is used by services within the AHA service deployment. It is
-generally a good idea to chose a name that aligns with the use case for the synapse deployment. For example,
+generally a good idea to chose a name that aligns with the use case for the Synapse deployment. For example,
 if you plan to have a test/dev/prod deployment, choosing a name like ``prod.synapse`` will make it clear which
 deployment is which. Changing this name later is difficult, so choose carefully! Throughout the examples, we
 will be using ``prod.synapse`` as the AHA network name which is also used as the common-name (CN) for the CA

From a7c5e3d6896297e8df4218ccfd5d947b663fada5 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 9 Jul 2024 15:18:19 -0400
Subject: [PATCH 32/68] wip

---
 synapse/tests/test_lib_agenda.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/tests/test_lib_agenda.py b/synapse/tests/test_lib_agenda.py
index 7497319ace1..5280cec22e4 100644
--- a/synapse/tests/test_lib_agenda.py
+++ b/synapse/tests/test_lib_agenda.py
@@ -850,7 +850,7 @@ async def task():
 
     async def test_cron_kill_pool(self):
 
-        async with self.getTestAhaProv() as aha:
+        async with self.getTestAha() as aha:
 
             import synapse.cortex as s_cortex
             import synapse.lib.base as s_base
@@ -872,11 +872,11 @@ async def test_cron_kill_pool(self):
 
                     msgs = await core00.stormlist('aha.pool.add pool00...')
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('Created AHA service pool: pool00.loop.vertex.link', msgs)
+                    self.stormIsInPrint('Created AHA service pool: pool00.synapse', msgs)
 
                     msgs = await core00.stormlist('aha.pool.svc.add pool00... 01.core...')
                     self.stormHasNoWarnErr(msgs)
-                    self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.loop.vertex.link)', msgs)
+                    self.stormIsInPrint('AHA service (01.core...) added to service pool (pool00.synapse)', msgs)
 
                     msgs = await core00.stormlist('cortex.storm.pool.set --connection-timeout 1 --sync-timeout 1 aha://pool00...')
                     self.stormHasNoWarnErr(msgs)

From 90b7959bc167e09312b6e4e73400b1fb71606bed Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 9 Jul 2024 16:51:29 -0400
Subject: [PATCH 33/68] wip

---
 synapse/lib/aha.py            |  4 +--
 synapse/lib/cell.py           | 53 +++++++++++++++++------------------
 synapse/tests/test_lib_aha.py |  9 ++++++
 3 files changed, 36 insertions(+), 30 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index d3ce45e1d39..a5a406d4b17 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -1272,7 +1272,7 @@ async def getAhaUrls(self, user='root'):
 
         return urls
 
-    def _getAhaUrl(self, user='root'):
+    def getMyUrl(self, user='root'):
         port = self.sockaddr[1]
         host = self._getDnsName()
         network = self.conf.req('aha:network')
@@ -1291,7 +1291,7 @@ async def addAhaClone(self, host, port=27492, conf=None):
 
         network = self.conf.req('aha:network')
 
-        conf['mirror'] = self._getAhaUrl()
+        conf['mirror'] = self.getMyUrl()
 
         conf['dns:name'] = host
         conf['aha:network'] = network
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index c32898ab894..3de3adf57c5 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1759,6 +1759,11 @@ async def waitFor(turl_sani, prox_):
     async def setNexsIndx(self, indx):
         return await self.nexsroot.setindex(indx)
 
+    def getMyUrl(self, user='root'):
+        host = self.conf.req('aha:name')
+        network = self.conf.req('aha:network')
+        return f'aha://{host}.{network}'
+
     async def promote(self, graceful=False):
         '''
         Transform this cell from a passive follower to
@@ -1769,45 +1774,36 @@ async def promote(self, graceful=False):
             mesg = 'promote() called on non-mirror'
             raise s_exc.BadConfValu(mesg=mesg)
 
-        ahaname = self.conf.get('aha:name')
-        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful} ahaname={ahaname}')
+        myurl = self.getMyUrl()
+        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}.')
 
         if graceful:
 
-            if ahaname is None: # pragma: no cover
-                mesg = 'Cannot gracefully promote without aha:name configured.'
-                raise s_exc.BadArg(mesg=mesg)
-
-            ahanetw = self.conf.req('aha:network')
-
-            myurl = f'aha://{ahaname}.{ahanetw}'
-            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff to ahaname={ahaname}')
+            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff.')
             async with await s_telepath.openurl(mirurl) as lead:
-                logger.debug(f'PROMOTION: Requesting leadership handoff to ahaname={ahaname}')
                 await lead.handoff(myurl)
-                logger.warning(f'PROMOTION: Completed leadership handoff to ahaname={ahaname}')
+                logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}')
                 return
 
-        logger.debug(f'PROMOTION: Clearing mirror configuration for ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Clearing mirror configuration.')
         self.modCellConf({'mirror': None})
 
-        logger.debug(f'PROMOTION: Promoting the nexus root for ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Promoting the nexus root.')
         await self.nexsroot.promote()
 
-        logger.debug(f'PROMOTION: Setting the cell as active ahaname={ahaname}')
+        logger.debug(f'PROMOTION: Setting the cell as active.')
         await self.setCellActive(True)
 
-        logger.warning(f'PROMOTION: Finished leadership promotion ahaname={ahaname}')
+        logger.warning(f'PROMOTION: Finished leadership promotion!')
 
     async def handoff(self, turl, timeout=30):
         '''
         Hand off leadership to a mirror in a transactional fashion.
         '''
-        ahaname = self.conf.get("aha:name")
-        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
+        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}.')
         async with await s_telepath.openurl(turl) as cell:
 
-            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)} from ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}.')
 
             if self.iden != await cell.getCellIden(): # pragma: no cover
                 mesg = 'Mirror handoff remote cell iden does not match!'
@@ -1817,33 +1813,34 @@ async def handoff(self, turl, timeout=30):
                 mesg = 'Cannot handoff mirror leadership to myself!'
                 raise s_exc.BadArg(mesg=mesg)
 
-            logger.debug(f'HANDOFF: Obtaining nexus lock ahaname={ahaname}')
+            logger.debug(f'HANDOFF: Obtaining nexus lock.')
 
             async with self.nexslock:
 
-                logger.debug(f'HANDOFF: Obtained nexus lock ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Obtained nexus lock.')
                 indx = await self.getNexsIndx()
 
-                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}, ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}.')
                 if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover
                     mndx = await cell.getNexsIndx()
                     mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.'
                     raise s_exc.NotReady(mesg=mesg)
 
-                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion.')
                 await cell.promote()
 
-                logger.debug(f'HANDOFF: Setting the service as inactive ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Setting the service as inactive.')
                 await self.setCellActive(False)
 
-                logger.debug(f'HANDOFF: Configuring service to use the new leader as its mirror ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Configuring service to sync from new leader.')
                 self.modCellConf({'mirror': turl})
 
-                logger.debug(f'HANDOFF: Restarting the nexus ahaname={ahaname}')
+                logger.debug(f'HANDOFF: Restarting the nexus.')
                 await self.nexsroot.startup()
 
-            logger.debug(f'HANDOFF: Released nexus lock ahaname={ahaname}')
-        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)} ahaname={self.conf.get("aha:name")}')
+            logger.debug(f'HANDOFF: Released nexus lock.')
+
+        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}.')
 
     async def reqAhaProxy(self, timeout=None):
         if self.ahaclient is None:
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index 24f82a28598..a66edc2b131 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -104,6 +104,15 @@ async def test_lib_aha_clone(self):
                     mnfo = await aha1.getAhaSvc('test.example.net')
                     self.none(mnfo)
 
+                    self.true(aha0.isactive)
+                    self.false(aha1.isactive)
+
+                    async with aha1.getLocalProxy() as proxy:
+                        await proxy.promote(graceful=True)
+
+                    self.false(aha0.isactive)
+                    self.true(aha1.isactive)
+
     async def test_lib_aha_offon(self):
         with self.getTestDir() as dirn:
             cryo0_dirn = s_common.gendir(dirn, 'cryo0')

From 25af8b63b8836af54702bf3db9b9d3a2b366d3e1 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 9 Jul 2024 17:34:23 -0400
Subject: [PATCH 34/68] wip

---
 synapse/lib/cell.py           | 3 ++-
 synapse/tests/test_lib_aha.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 3de3adf57c5..ddc6d563269 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1774,11 +1774,12 @@ async def promote(self, graceful=False):
             mesg = 'promote() called on non-mirror'
             raise s_exc.BadConfValu(mesg=mesg)
 
-        myurl = self.getMyUrl()
         logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}.')
 
         if graceful:
 
+            myurl = self.getMyUrl()
+
             logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff.')
             async with await s_telepath.openurl(mirurl) as lead:
                 await lead.handoff(myurl)
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index a66edc2b131..c37bfe6f621 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -884,7 +884,7 @@ async def test_aha_connect_back(self):
         async with self.getTestAha() as aha:  # type: s_aha.AhaCell
 
             async with self.addSvcToAha(aha, '00.exec', ExecTeleCaller) as conn:
-                ahaurl = aha._getAhaUrl()
+                ahaurl = aha.getMyUrl()
                 await conn.exectelecall(ahaurl, 'getNexsIndx')
 
             self.true(conn.ahaclient.isfini)

From e4dea7bd398b5da2fb3050764074b2a6306949bb Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Wed, 10 Jul 2024 08:41:17 -0400
Subject: [PATCH 35/68] make timeout message a bit prettier

---
 synapse/lib/base.py            | 3 ++-
 synapse/tests/test_lib_base.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/synapse/lib/base.py b/synapse/lib/base.py
index 8fe9d6bb7a0..8040fe60c75 100644
--- a/synapse/lib/base.py
+++ b/synapse/lib/base.py
@@ -686,7 +686,8 @@ async def __aenter__(self):
     async def __aexit__(self, exc, cls, tb):
         if exc is None:
             if await self.wait() is None:
-                mesg = f'timeout waiting for {self.count} events {self.names}'
+                events = ','.join(self.names)
+                mesg = f'timeout waiting for {self.count} event(s): {events}'
                 raise s_exc.TimeOut(mesg=mesg)
 
 class BaseRef(Base):
diff --git a/synapse/tests/test_lib_base.py b/synapse/tests/test_lib_base.py
index 3bd9b1be99b..f7d746b35fa 100644
--- a/synapse/tests/test_lib_base.py
+++ b/synapse/tests/test_lib_base.py
@@ -226,7 +226,7 @@ async def test_base_waiter(self):
         self.len(2, evts)
 
         with self.raises(s_exc.TimeOut):
-            async with base0.waiter(1, 'newp', timeout=0.01):
+            async with base0.waiter(1, 'newp', 'nuuh', timeout=0.01):
                 pass
 
     async def test_baseref(self):

From c7f1973091411248755f89340d49903ae922aaef Mon Sep 17 00:00:00 2001
From: invisig0th <visi@vertex.link>
Date: Wed, 10 Jul 2024 09:06:17 -0400
Subject: [PATCH 36/68] Update synapse/lib/aha.py

Co-authored-by: Cisphyx <cisphyx@vertex.link>
---
 synapse/lib/aha.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index a5a406d4b17..665e791375a 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -540,7 +540,7 @@ async def _initCellBoot(self):
 
         path = s_common.genpath(self.dirn, 'cell.guid')
         if os.path.isfile(path):
-            logger.info('Cloneing AHA: cell.guid detected. Skipping.')
+            logger.info('Cloning AHA: cell.guid detected. Skipping.')
             return
 
         logger.warning(f'Cloning AHA: {curl}')

From f62ee425ea10a6d263a174ee74d0b91487857248 Mon Sep 17 00:00:00 2001
From: epiphyte <epiphyte@vertex.link>
Date: Wed, 10 Jul 2024 13:14:00 +0000
Subject: [PATCH 37/68] Do container branch build

---
 .circleci/config.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index bf7ce9db25d..0234a01bf6b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -603,6 +603,7 @@ workflows:
             branches:
               only:
                 - master
+                - visi-aha-defnet
 
       - build_docker_tag:
           requires:

From 21314af56226ac14aca86d15b636c9bbb8eeec19 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Thu, 11 Jul 2024 13:33:19 -0400
Subject: [PATCH 38/68] wip

---
 synapse/lib/cell.py | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index ddc6d563269..cb4eff7bfb7 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1209,6 +1209,7 @@ async def fini():
 
         # for runtime cell configuration values
         self.slab.initdb('cell:conf')
+        self.slab.initdb('cell:meta')
 
         self._sslctx_cache = s_cache.FixedCache(self._makeCachedSslCtx, size=SSLCTX_CACHE_SIZE)
 
@@ -1286,6 +1287,17 @@ async def fini():
         # phase 5 - service networking
         await self.initServiceNetwork()
 
+    def getCellMeta(self, name, defv=None):
+        byts = self.slab.get(name.encode(), s_msgpack.en(valu), db='cell:meta')
+        if byts is not None:
+            return s_msgpack.un(byts)
+        return defv
+
+    def setCellMeta(self, name, valu):
+        # NOTE: these changes are NOT nexus enabled!
+        self.slab.put(name.encode(), s_msgpack.en(valu), db='cell:meta')
+        return valu
+
     def getPermDef(self, perm):
         perm = tuple(perm)
         if self.permlook is None:
@@ -1557,6 +1569,12 @@ async def initNexusSubsystem(self):
         if self.cellparent is None:
             await self.nexsroot.recover()
             await self.nexsroot.startup()
+
+            mirurl = self.conf.get('mirror')
+            if mirurl is None:
+                # we think we should lead...
+                nexs
+
             await self.setCellActive(self.conf.get('mirror') is None)
 
             if self.minfree is not None:

From 5954fa4f3ee91744cc9c2c00a854c4f856b12e60 Mon Sep 17 00:00:00 2001
From: epiphyte <epiphyte@vertex.link>
Date: Fri, 12 Jul 2024 14:41:59 +0000
Subject: [PATCH 39/68] Update docker test script for required value.

---
 docker/scripts/test_all.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/scripts/test_all.sh b/docker/scripts/test_all.sh
index 2a3b809803e..b72d8616996 100755
--- a/docker/scripts/test_all.sh
+++ b/docker/scripts/test_all.sh
@@ -28,7 +28,7 @@ echo "Spinning up images"
 docker run --rm -it --entrypoint python vertexproject/synapse:${TAG} -m synapse.servers.cortex --help
 dstatus00=$?
 if [ $dstatus00 != "0" ]; then exit 1; fi
-docker run --rm -d --name test-aha vertexproject/synapse-aha:${TAG}
+docker run --rm -d --name test-aha -e "SYN_AHA_AHA_NETWORK=synapse.ci" vertexproject/synapse-aha:${TAG}
 docker run --rm -d --name test-axon vertexproject/synapse-axon:${TAG}
 docker run --rm -d --name test-cortex vertexproject/synapse-cortex:${TAG}
 docker run --rm -d --name test-cryotank vertexproject/synapse-cryotank:${TAG}

From c7145e3bed4f01797ee3d17813d46fc945350681 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 12 Jul 2024 13:07:29 -0400
Subject: [PATCH 40/68] wip

---
 synapse/lib/aha.py    | 4 ++--
 synapse/lib/config.py | 4 ++++
 synapse/telepath.py   | 2 +-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 665e791375a..7e677f298d7 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -611,7 +611,6 @@ async def _addAhaServer(self, server):
         host = server.get('host')
         port = server.get('port')
 
-        oldv = None
         lkey = s_msgpack.en((host, port))
 
         byts = self.slab.get(lkey, db='aha:servers')
@@ -751,7 +750,6 @@ async def initServiceNetwork(self):
 
         # bootstrap CA/host certs first
         network = self.conf.req('aha:network')
-        await self._genCaCert(network)
 
         hostname = self._getDnsName()
         if hostname is not None and network is not None:
@@ -1155,6 +1153,8 @@ async def genCaCert(self, network):
 
     async def _genCaCert(self, network):
 
+        # generate a CA cert if one does not exist
+        # ( but don't read it in if it does )
         if os.path.isfile(os.path.join(self.dirn, 'certs', 'cas', f'{network}.crt')):
             return
 
diff --git a/synapse/lib/config.py b/synapse/lib/config.py
index 5295a4bac7a..d03a950e0bf 100644
--- a/synapse/lib/config.py
+++ b/synapse/lib/config.py
@@ -392,6 +392,10 @@ def reqConfValid(self):
         else:
             return
 
+    def reqConfValu(self, name):
+        s_common.deprecated('reqConfValu()')
+        return self.req(name)
+
     def req(self, name):
         '''
         Return a configuration value or raise NeedConfValu if it is unset.
diff --git a/synapse/telepath.py b/synapse/telepath.py
index 388bc2b108c..ba45546d406 100644
--- a/synapse/telepath.py
+++ b/synapse/telepath.py
@@ -1292,7 +1292,7 @@ async def _teleLinkLoop(self):
                 now = time.monotonic()
                 if now > lastlog + 60.0:  # don't logspam the disconnect message more than 1/min
                     url = s_urlhelp.sanitizeUrl(zipurl(urlinfo))
-                    logger.warning(f'telepath client ({url}) encountered an error: {e}')
+                    logger.warning(f'telepath client ({url}) encountered an error: {e}', exc_info=e)
                     lastlog = now
                 await self.waitfini(timeout=self._t_conf.get('retrysleep', 0.2))
 

From 4282ac8f738eee26c8c9b391f686ad600b496597 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 12 Jul 2024 13:58:55 -0400
Subject: [PATCH 41/68] wip

---
 synapse/lib/aha.py  |  2 +-
 synapse/lib/cell.py | 39 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 1 deletion(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index bc496416b64..d82369d449d 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -1329,7 +1329,7 @@ async def mayLeadTerm(self, iden, name, term, nexs):
                     'nexs': nexs,
                 }
                 await self.addLeadTerm(leadterm)
-                return (STATE_LEAD, {})
+                return (STATE_LEAD, leadterm)
 
             # if we were the last known leader, we can jump right in
             if leadterm.get('name') == ahaname:
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 62bb2fabb5c..677b68fa660 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1503,6 +1503,39 @@ async def _runSysctlLoop(self):
 
             await self.waitfini(self.SYSCTL_CHECK_FREQ)
 
+    async def _askAhaToLead(self, proxy):
+
+        async with self.nexslock:
+
+            name = self.conf.get('aha:name')
+            nexs = self.getNexsIndx()
+            term = self.getCellMeta('aha:term', 0)
+
+            state, leadterm = await proxy.mayLeadTerm(self.iden, name, term, nexs)
+
+            if state == s_aha.STATE_LEAD:
+                self.conf.modCellConf({'mirror': None})
+                await self.setCellActive(True)
+                await self.nexsroot.startup()
+                return
+
+            if state == s_aha.STATE_FOLLOW:
+                lead = leadterm.get('name')
+                user = self.conf.get('aha:user', 'root')
+                self.conf.modCellConf({'mirror': f'aha://{user}@{lead}...'})
+                await self.setCellActive(False)
+                await self.nexsroot.startup()
+                return
+
+            if state == s_aha.STATE_INVALID:
+                await self._terStateInvalid()
+
+    async def _termStateInvalid(self):
+        # hook point for enterprise behavior
+        logger.error('Leadership schism detected!')
+        logger.error('See: FIXME DOCS URL')
+        await self.fini()
+
     async def _initAhaRegistry(self):
 
         ahaurls = self.conf.get('aha:registry')
@@ -1520,6 +1553,12 @@ async def onlink(proxy):
                     self.modCellConf({'aha:registry': newurls})
                     self.ahaclient.setBootUrls(newurls)
 
+                if self.conf.get('mirror') is None:
+
+                    # if i think i'm the leader, lets check with AHA
+                    ahainfo = await proxy.getCellInfo()
+                    if ahainfo['features'].get('aha:leadterm'):
+
             self.ahaclient = await s_telepath.Client.anit(ahaurls, onlink=onlink)
             self.onfini(self.ahaclient)
 

From f2c4faeb5e652d98ac2dbcf60feab54c6b8393b3 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 15 Jul 2024 09:10:57 -0400
Subject: [PATCH 42/68] wip

---
 synapse/lib/cell.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 32c41be28cc..5be7acb6932 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -3849,15 +3849,12 @@ async def initFromArgv(cls, argv, outp=None):
 
         try:
 
-            if 'dmon:listen' not in cell.conf:
-                await cell.dmon.listen(opts.telepath)
-                logger.info(f'...{cell.getCellType()} API (telepath): {opts.telepath}')
-            else:
-                lisn = cell.conf.get('dmon:listen')
-                if lisn is None:
-                    lisn = cell.getLocalUrl()
+            turl = cell._getDmonListen()
+            if turl is None:
+                turl = opts.telepath
+                await cell.dmon.listen(turl)
 
-                logger.info(f'...{cell.getCellType()} API (telepath): {lisn}')
+            logger.info(f'...{cell.getCellType()} API (telepath): {turl}')
 
             if 'https:port' not in cell.conf:
                 await cell.addHttpsPort(opts.https)

From ef41b3eb09bcd1de2c2389e3c2ad0038a4cfd1b2 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 15 Jul 2024 14:48:57 -0400
Subject: [PATCH 43/68] wip

---
 synapse/lib/aha.py  |  11 ++++-
 synapse/lib/cell.py | 103 ++++++++++++++++++++++++++------------------
 2 files changed, 70 insertions(+), 44 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index d82369d449d..3272028aadd 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -362,6 +362,15 @@ async def clearAhaUserEnrolls(self):
         '''
         return await self.cell.clearAhaUserEnrolls()
 
+    async def mayLeadTerm(self, iden, name, term, nexs):
+        network = self.cell.conf.get('aha:network')
+        # If we could register the service, we can lead the service...
+        await self._reqUserAllowed(('aha', 'service', 'add', network, name))
+        return await self.cell.mayLeadTerm(iden, name, term, nexs)
+
+    async def getLeadTerm(self, iden):
+        return await self.cell.getLeadTerm(iden)
+
 class ProvDmon(s_daemon.Daemon):
 
     async def __anit__(self, aha):
@@ -566,7 +575,6 @@ async def _initCellBoot(self):
 
     async def initServiceStorage(self):
 
-        # TODO plumb using a remote jsonstor?
         dirn = s_common.gendir(self.dirn, 'slabs', 'jsonstor')
 
         slab = await s_lmdbslab.Slab.anit(dirn)
@@ -1274,7 +1282,6 @@ async def setLeadTerm(self, termdef):
         '''
         Set the current leader term for the specified service cluster.
         '''
-        # TODO schema
         termdef['time'] = s_common.now()
         s_schema.reqValidLeadTerm(termdef)
         return await self._push('aha:lead:set', termdef)
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 677b68fa660..58518e4708a 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1505,35 +1505,43 @@ async def _runSysctlLoop(self):
 
     async def _askAhaToLead(self, proxy):
 
-        async with self.nexslock:
+        while not self.isfini:
+            try:
+                proxy = self.ahaclient.proxy(timeout=4)
+            except Exception as e:
+                logger.error(f'Error getting AHA proxy to check leadership: {e}')
 
-            name = self.conf.get('aha:name')
-            nexs = self.getNexsIndx()
-            term = self.getCellMeta('aha:term', 0)
+            async with self.nexslock:
 
-            state, leadterm = await proxy.mayLeadTerm(self.iden, name, term, nexs)
+                name = self.conf.get('aha:name')
+                nexs = self.getNexsIndx()
+                term = self.getCellMeta('aha:term', 0)
 
-            if state == s_aha.STATE_LEAD:
-                self.conf.modCellConf({'mirror': None})
-                await self.setCellActive(True)
-                await self.nexsroot.startup()
-                return
+                state, leadterm = await proxy.mayLeadTerm(self.iden, name, term, nexs)
 
-            if state == s_aha.STATE_FOLLOW:
-                lead = leadterm.get('name')
-                user = self.conf.get('aha:user', 'root')
-                self.conf.modCellConf({'mirror': f'aha://{user}@{lead}...'})
-                await self.setCellActive(False)
-                await self.nexsroot.startup()
-                return
+                realterm = leadterm.get('term')
+                if realterm > term:
+                    self.setCellMeta('aha:term', realterm)
+
+                if state == s_aha.STATE_LEAD:
+                    await self.setMirror(None)
+                    return
+
+                if state == s_aha.STATE_FOLLOW:
+                    lead = leadterm.get('name')
+                    user = self.conf.get('aha:user', 'root')
+                    await self.setMirror(f'aha://{user}@{lead}')
+                    return
 
-            if state == s_aha.STATE_INVALID:
-                await self._terStateInvalid()
+                if state == s_aha.STATE_INVALID:
+                    await self._termStateInvalid()
 
     async def _termStateInvalid(self):
         # hook point for enterprise behavior
         logger.error('Leadership schism detected!')
         logger.error('See: FIXME DOCS URL')
+        # TODO: should we allow an ENV var based override?
+        # TODO: should we find the leader and re-provision?
         await self.fini()
 
     async def _initAhaRegistry(self):
@@ -1605,19 +1613,32 @@ async def initServiceStorage(self):
         pass
 
     async def initNexusSubsystem(self):
-        if self.cellparent is None:
-            await self.nexsroot.recover()
-            await self.nexsroot.startup()
 
-            mirurl = self.conf.get('mirror')
-            if mirurl is None:
-                # we think we should lead...
-                nexs
+        if self.cellparent is not None:
+            return
 
-            await self.setCellActive(self.conf.get('mirror') is None)
+        # If we are AHA enabled and think we're the leader,
+        # we must check before proceeding...
+        # TODO: or be readonly or a follower of None or something?
+        mirror = self.conf.get('mirror')
+        if self.ahaclient is not None:
+            self._ask
+            nexs = self.getNexsIndx()
+
+        await self.nexsroot.recover()
+        await self.nexsroot.startup()
+
+        # retrieve this again in case it changed
+        mirror = self.conf.get('mirror')
+        await self.setCellActive(mirror is None)
+
+        if self.minfree is not None:
+            self.schedCoro(self._runFreeSpaceLoop())
 
-            if self.minfree is not None:
-                self.schedCoro(self._runFreeSpaceLoop())
+    async def setMirror(self, url):
+        self.conf.update({'mirror': url})
+        await self.setCellActive(url is None)
+        await self.nexsroot.startup()
 
     async def _bindDmonListen(self):
 
@@ -1843,14 +1864,18 @@ async def promote(self, graceful=False):
                 logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}')
                 return
 
-        logger.debug(f'PROMOTION: Clearing mirror configuration.')
-        self.modCellConf({'mirror': None})
+        if self.ahaclient:
 
-        logger.debug(f'PROMOTION: Promoting the nexus root.')
-        await self.nexsroot.promote()
+            name = self.conf.get('aha:name')
+            proxy = await self.ahaclient.proxy(timeout=6)
+
+            logger.warning(f'PROMOTION: Updating AHA Leadership Term')
+            async with self.nexslock:
+                nexs = await self.getNexsIndx()
+                leadterm = await proxy.nextLeadTerm(self.iden, name, nexs)
+                self.setCellMeta('aha:term', leadterm.get('term'))
 
-        logger.debug(f'PROMOTION: Setting the cell as active.')
-        await self.setCellActive(True)
+        await self.setMirror(None)
 
         logger.warning(f'PROMOTION: Finished leadership promotion!')
 
@@ -1887,14 +1912,8 @@ async def handoff(self, turl, timeout=30):
                 logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion.')
                 await cell.promote()
 
-                logger.debug(f'HANDOFF: Setting the service as inactive.')
-                await self.setCellActive(False)
-
                 logger.debug(f'HANDOFF: Configuring service to sync from new leader.')
-                self.modCellConf({'mirror': turl})
-
-                logger.debug(f'HANDOFF: Restarting the nexus.')
-                await self.nexsroot.startup()
+                await self.setMirror(turl)
 
             logger.debug(f'HANDOFF: Released nexus lock.')
 

From 3badfdc57cfd5628fa090de6244db4a75ff9c088 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 15 Jul 2024 15:06:01 -0400
Subject: [PATCH 44/68] wip

---
 synapse/tests/test_lib_cell.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/tests/test_lib_cell.py b/synapse/tests/test_lib_cell.py
index 025397f635c..a46b1e9ad73 100644
--- a/synapse/tests/test_lib_cell.py
+++ b/synapse/tests/test_lib_cell.py
@@ -913,7 +913,7 @@ async def test_cell_confprint(self):
                         pass
                 stream.seek(0)
                 buf = stream.read()
-                self.isin(f'...cell API (telepath): cell://root@{dirn}:*', buf)
+                self.isin(f'...cell API (telepath): tcp://0.0.0.0:27492', buf)
                 self.isin('...cell API (https): disabled', buf)
 
     async def test_cell_initargv_conf(self):

From cbc39e3ef37ab839063b14e4eddba0d52dba2f32 Mon Sep 17 00:00:00 2001
From: epiphyte <epiphyte@vertex.link>
Date: Mon, 15 Jul 2024 20:50:49 +0000
Subject: [PATCH 45/68] Fix fstrings

---
 synapse/lib/aha.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 7e677f298d7..79bd5438ab1 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -1165,7 +1165,7 @@ async def _genHostCert(self, hostname, signas=None):
         if signas is not None:
             await self._genCaCert(signas)
 
-        if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', '{hostname}.crt')):
+        if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', f'{hostname}.crt')):
             return
 
         pkey, cert = await s_coro.executor(self.certdir.genHostCert, hostname, signas=signas, save=False)
@@ -1175,7 +1175,7 @@ async def _genHostCert(self, hostname, signas=None):
 
     async def _genUserCert(self, username, signas=None):
 
-        if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', '{username}.crt')):
+        if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', f'{username}.crt')):
             return
 
         logger.info(f'Adding user certificate for {username}')

From 7fb5d584cf3ce678bdd7fc2264b4f4ef74ed6596 Mon Sep 17 00:00:00 2001
From: epiphyte <epiphyte@vertex.link>
Date: Mon, 15 Jul 2024 21:13:43 +0000
Subject: [PATCH 46/68] Fix fstring for clone; add --only-url

---
 synapse/tools/aha/clone.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/synapse/tools/aha/clone.py b/synapse/tools/aha/clone.py
index 878b08f7425..be0c7b03938 100644
--- a/synapse/tools/aha/clone.py
+++ b/synapse/tools/aha/clone.py
@@ -22,7 +22,8 @@ async def main(argv, outp=s_output.stdout):
     pars.add_argument('dnsname', help='The DNS name of the new AHA server.')
     pars.add_argument('--port', type=int, default=27492, help='The port that the new AHA server should listen on.')
     pars.add_argument('--url', default='cell:///vertex/storage', help='The telepath URL to connect to the AHA service.')
-
+    pars.add_argument('--only-url', help='Only output the URL upon successful execution',
+                      action='store_true', default=False)
     opts = pars.parse_args(argv)
 
     async with s_telepath.withTeleEnv():
@@ -30,7 +31,11 @@ async def main(argv, outp=s_output.stdout):
         try:
             async with await s_telepath.openurl(opts.url) as aha:
                 curl = await aha.addAhaClone(opts.dnsname, port=opts.port)
-                outp.printf('one-time use URL: {curl}')
+
+                if opts.only_url:
+                    outp.printf(curl)
+                else:
+                    outp.printf(f'one-time use URL: {curl}')
                 return 0
 
         except Exception as e:

From efa7d025bd586ba4bc07fe87de039a5bf5666713 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 23 Jul 2024 14:35:06 -0400
Subject: [PATCH 47/68] wip

---
 docs/synapse/deploymentguide.rst | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index e92774e9830..70a703a4f10 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -77,7 +77,14 @@ using AHA, the only host that needs DNS or other external name resolution is the
 
 .. note::
 
-    FIXME AHA FLAT NETWORK STUFF
+    The AHA service resolver requires that registered services connect directly using IP addresses which
+    must also be reachable to AHA clients. Using an ``aha://`` telepath URL requires direct routes to the
+    service via it's AHA facing network address. If you need to provide telepath access from outside the
+    Synapse service network via any network address translation (NAT) method such as an inbound TCP proxy
+    or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``,
+    ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need
+    to specify a specific ``--dmon-port`` option when using the  ``synapse.tools.aha.provision.service``
+    command to provision the service.
 
 Choose an AHA Network Name
 --------------------------

From fe2e58382f8c367258a841db4efe59d65e442d69 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Wed, 24 Jul 2024 10:36:24 -0400
Subject: [PATCH 48/68] wip

---
 synapse/lib/aha.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 79bd5438ab1..5c445f28dc7 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -1154,8 +1154,7 @@ async def genCaCert(self, network):
     async def _genCaCert(self, network):
 
         # generate a CA cert if one does not exist
-        # ( but don't read it in if it does )
-        if os.path.isfile(os.path.join(self.dirn, 'certs', 'cas', f'{network}.crt')):
+        if self.certdir.getCaCertPath(network) is not None:
             return
 
         await self.genCaCert(network)
@@ -1165,7 +1164,7 @@ async def _genHostCert(self, hostname, signas=None):
         if signas is not None:
             await self._genCaCert(signas)
 
-        if os.path.isfile(os.path.join(self.dirn, 'certs', 'hosts', f'{hostname}.crt')):
+        if self.certdir.getHostCertPath(hostname) is not None:
             return
 
         pkey, cert = await s_coro.executor(self.certdir.genHostCert, hostname, signas=signas, save=False)
@@ -1175,7 +1174,7 @@ async def _genHostCert(self, hostname, signas=None):
 
     async def _genUserCert(self, username, signas=None):
 
-        if os.path.isfile(os.path.join(self.dirn, 'certs', 'users', f'{username}.crt')):
+        if self.certdir.getUserCertPath(username) is not None:
             return
 
         logger.info(f'Adding user certificate for {username}')

From 21ab32717eb0f6d7941414b930569b187bb47cf5 Mon Sep 17 00:00:00 2001
From: vEpiphyte <epiphyte@vertex.link>
Date: Fri, 26 Jul 2024 16:20:48 -0400
Subject: [PATCH 49/68] Visi aha defnet epiphyte (#3823)

---
 changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml |  5 +
 synapse/lib/aha.py                            | 30 +++++-
 synapse/lib/cell.py                           | 20 +++-
 synapse/lib/config.py                         | 28 ++++--
 synapse/tests/test_lib_aha.py                 | 95 ++++++++++++++++++-
 synapse/tests/test_lib_config.py              |  3 +-
 synapse/tests/test_tools_aha.py               |  7 ++
 synapse/tests/utils.py                        | 12 ++-
 8 files changed, 179 insertions(+), 21 deletions(-)
 create mode 100644 changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml

diff --git a/changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml b/changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml
new file mode 100644
index 00000000000..6b565c84e01
--- /dev/null
+++ b/changes/c2d4e325f4c46cb9ae7985097e2b364e.yaml
@@ -0,0 +1,5 @@
+---
+desc: Deprecate the ``Cell.conf.reqConfValu()`` API. This has been replaced with ``Cell.conf.req()``.
+prs: []
+type: deprecation
+...
diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 5c445f28dc7..5a87a036c34 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -153,6 +153,7 @@ async def getAhaSvc(self, name, filters=None):
         username = self.user.name.split('@')[0]
 
         if svcinfo.get('svcinfo'):
+            logger.info(f'Hinting {self.user.name} for {name}')
             svcinfo['svcinfo']['urlinfo']['user'] = username
 
         return svcinfo
@@ -359,6 +360,14 @@ async def clearAhaUserEnrolls(self):
         '''
         return await self.cell.clearAhaUserEnrolls()
 
+    @s_cell.adminapi()
+    async def clearAhaClones(self):
+        '''
+        Remove all unused AHA clone provisioning values.
+        '''
+        return await self.cell.clearAhaClones()
+
+
 class ProvDmon(s_daemon.Daemon):
 
     async def __anit__(self, aha):
@@ -386,8 +395,8 @@ async def _getSharedItem(self, name):
 
         clone = await self.aha.getAhaClone(name)
         if clone is not None:
-            mesg = f'Retrieved AHA clone info for {name}'
             host = clone.get('host')
+            mesg = f'Retrieved AHA clone info for {host} iden {name}'
             logger.info(mesg, extra=await self.aha.getLogExtra(iden=name, host=host))
             return CloneApi(self.aha, clone)
 
@@ -1221,8 +1230,8 @@ async def signHostCsr(self, csrtext, signas=None, sans=None):
 
         hostname = xcsr.subject.get_attributes_for_oid(c_x509.NameOID.COMMON_NAME)[0].value
 
-        hostpath = s_common.genpath(self.dirn, 'certs', 'hosts', f'{hostname}.crt')
-        if os.path.isfile(hostpath):
+        hostpath = self.certdir.getHostCertPath(hostname)
+        if hostpath is not None:
             os.unlink(hostpath)
 
         if signas is None:
@@ -1240,8 +1249,8 @@ async def signUserCsr(self, csrtext, signas=None):
 
         username = xcsr.subject.get_attributes_for_oid(c_x509.NameOID.COMMON_NAME)[0].value
 
-        userpath = s_common.genpath(self.dirn, 'certs', 'users', f'{username}.crt')
-        if os.path.isfile(userpath):
+        userpath = self.certdir.getUserCertPath(username)
+        if userpath is not None:
             os.unlink(userpath)
 
         if signas is None:
@@ -1304,6 +1313,10 @@ async def addAhaClone(self, host, port=27492, conf=None):
             'conf': conf,
         }
         await self._push('aha:clone:add', clone)
+
+        logger.info(f'Created AHA clone provisioning for {host} with iden {iden}',
+                     extra=await self.getLogExtra(iden=iden, name=host, netw=network))
+
         return self._getProvClientUrl(iden)
 
     @s_nexus.Pusher.onPush('aha:clone:add')
@@ -1443,6 +1456,13 @@ async def clearAhaUserEnrolls(self):
             userinfo = s_msgpack.un(byts)
             logger.info(f'Deleted user enrollment username={userinfo.get("name")}, iden={iden.decode()}')
 
+    @s_nexus.Pusher.onPushAuto('aha:clone:clear')
+    async def clearAhaClones(self):
+        for lkey, byts in self.slab.scanByFull(db='aha:clones'):
+            self.slab.delete(lkey, db='aha:clones')
+            cloninfo = s_msgpack.un(byts)
+            logger.info(f'Deleted AHA clone enrollment username={cloninfo.get("host")}, iden={s_common.ehex(lkey)}')
+
     @s_nexus.Pusher.onPushAuto('aha:svc:prov:del')
     async def delAhaSvcProv(self, iden):
         self.slab.delete(iden.encode(), db='aha:provs')
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 3cedf26a2ea..93f04a57456 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1673,15 +1673,24 @@ async def _initAhaRegistry(self):
         ahaurls = self.conf.get('aha:registry')
         if ahaurls is not None:
 
-            info = await s_telepath.addAhaUrl(ahaurls)
+            await s_telepath.addAhaUrl(ahaurls)
             if self.ahaclient is not None:
                 await self.ahaclient.fini()
 
             async def onlink(proxy):
                 ahauser = self.conf.get('aha:user', 'root')
                 newurls = await proxy.getAhaUrls(user=ahauser)
-                oldurls = tuple(self.conf.get('aha:registry'))
+                oldurls = self.conf.get('aha:registry')
+                if isinstance(oldurls, str):
+                    oldurls = (oldurls,)
+                elif isinstance(oldurls, list):
+                    oldurls = tuple(oldurls)
                 if newurls and newurls != oldurls:
+                    if oldurls[0].startswith('tcp://'):
+                        s_common.deprecated('aha:registry: tcp:// client values.')
+                        logger.warning('tcp:// based aha:registry options will be deprecated in Synapse v3.0.0')
+                        return
+
                     self.modCellConf({'aha:registry': newurls})
                     self.ahaclient.setBootUrls(newurls)
 
@@ -1694,9 +1703,14 @@ async def fini():
             self.ahaclient.onfini(fini)
 
         ahaadmin = self.conf.get('aha:admin')
+        ahauser = self.conf.get('aha:user')
+
         if ahaadmin is not None:
             await self._addAdminUser(ahaadmin)
 
+        if ahauser is not None:
+            await self._addAdminUser(ahauser)
+
     def _getDmonListen(self):
 
         lisn = self.conf.get('dmon:listen', s_common.novalu)
@@ -1741,7 +1755,7 @@ async def initNexusSubsystem(self):
 
     async def _bindDmonListen(self):
 
-        # functionalized so Raft code can bind early...
+        # functionalized so downstream code can bind early.
         if self.sockaddr is not None:
             return
 
diff --git a/synapse/lib/config.py b/synapse/lib/config.py
index d03a950e0bf..ed46d6476bf 100644
--- a/synapse/lib/config.py
+++ b/synapse/lib/config.py
@@ -392,20 +392,32 @@ def reqConfValid(self):
         else:
             return
 
-    def reqConfValu(self, name):
-        s_common.deprecated('reqConfValu()')
-        return self.req(name)
+    def reqConfValu(self, key):  # pragma: no cover
+        '''
+        Deprecated. Use ``req(key)`` API instead.
+        '''
+        s_common.deprecated('Config.reqConfValu(), use req() instead.')
+        return self.req(key)
 
-    def req(self, name):
+    def req(self, key):
         '''
-        Return a configuration value or raise NeedConfValu if it is unset.
+        Get a configuration value. If that value is not present in the schema or is not set, then raise an exception.
+
+        Args:
+            key (str): The key to require.
+
+        Returns:
+            The requested value.
         '''
-        valu = self.conf.get(name, s_common.novalu)
+        if key not in self.json_schema.get('properties', {}):
+            raise s_exc.BadArg(mesg=f'The {key} configuration option is not present in the configuration schema.',
+                               name=key)
+
+        valu = self.conf.get(key, s_common.novalu)
         if valu is not s_common.novalu:
             return valu
 
-        mesg = f'The {name} configuration option is required.'
-        raise s_exc.NeedConfValu(mesg=mesg, name=name)
+        raise s_exc.NeedConfValu(mesg=f'The {key} configuration option is required.', name=key)
 
     def reqKeyValid(self, key, value):
         '''
diff --git a/synapse/tests/test_lib_aha.py b/synapse/tests/test_lib_aha.py
index c37bfe6f621..50b6a7a921e 100644
--- a/synapse/tests/test_lib_aha.py
+++ b/synapse/tests/test_lib_aha.py
@@ -14,7 +14,6 @@
 import synapse.lib.cell as s_cell
 
 import synapse.tools.aha.list as s_a_list
-import synapse.tools.backup as s_tools_backup
 
 import synapse.tools.aha.enroll as s_tools_enroll
 import synapse.tools.aha.provision.user as s_tools_provision_user
@@ -46,6 +45,8 @@ class AhaTest(s_test.SynTest):
 
     async def test_lib_aha_clone(self):
 
+        zoinks = 'zoinks.aha.loop.vertex.link'
+
         with self.getTestDir() as dirn:
 
             dir0 = s_common.gendir(dirn, 'aha0')
@@ -58,7 +59,7 @@ async def test_lib_aha_clone(self):
                     self.len(ahacount, await proxy0.getAhaUrls())
                     self.len(ahacount, await proxy0.getAhaServers())
 
-                    purl = await proxy0.addAhaClone('zoinks.aha.loop.vertex.link')
+                    purl = await proxy0.addAhaClone(zoinks)
 
                 conf1 = {'clone': purl}
                 async with self.getTestAha(conf=conf1, dirn=dir1) as aha1:
@@ -113,6 +114,21 @@ async def test_lib_aha_clone(self):
                     self.false(aha0.isactive)
                     self.true(aha1.isactive)
 
+            # Remove 00.aha.loop.vertex.link since we're done with him + coverage
+            async with self.getTestAha(conf={'dns:name': zoinks}, dirn=dir1) as aha1:
+                async with aha1.getLocalProxy() as proxy1:
+                    srvs = await proxy1.getAhaServers()
+                    self.len(2, srvs)
+                    aha00 = [info for info in srvs if info.get('host') == '00.aha.loop.vertex.link'][0]
+                    data = await proxy1.delAhaServer(aha00.get('host'), aha00.get('port'))
+                    self.eq(data.get('host'), aha00.get('host'))
+                    self.eq(data.get('port'), aha00.get('port'))
+
+                    srvs = await proxy1.getAhaServers()
+                    self.len(1, srvs)
+                    urls = await proxy1.getAhaUrls()
+                    self.len(1, urls)
+
     async def test_lib_aha_offon(self):
         with self.getTestDir() as dirn:
             cryo0_dirn = s_common.gendir(dirn, 'cryo0')
@@ -348,6 +364,13 @@ async def test_lib_aha_basics(self):
                     await cell.ahaclient.proxy()
                     self.len(ahacount + 1, cell.conf.get('aha:registry'))
 
+                self.nn(await aha.delAhaServer('zoinks.aha.loop.vertex.link', 27492))
+                self.len(ahacount, await aha.getAhaServers())
+
+                async with self.getTestCell(s_cell.Cell, conf=conf, dirn=dirn) as cell:
+                    await cell.ahaclient.proxy()
+                    self.len(ahacount, cell.conf.get('aha:registry'))
+
     async def test_lib_aha_loadenv(self):
 
         with self.getTestDir() as dirn:
@@ -775,14 +798,18 @@ async def test_lib_aha_provision(self):
                 # We can generate urls and then drop them en-mass. They will not usable.
                 provurls = []
                 enrlursl = []
+                clonurls = []
                 async with aha.getLocalProxy() as proxy:
                     provurls.append(await proxy.addAhaSvcProv('00.cell'))
                     provurls.append(await proxy.addAhaSvcProv('01.cell', {'mirror': 'cell'}))
                     enrlursl.append(await proxy.addAhaUserEnroll('bob'))
                     enrlursl.append(await proxy.addAhaUserEnroll('alice'))
+                    clonurls.append(await proxy.addAhaClone('hehe.haha.com'))
+                    clonurls.append(await proxy.addAhaClone('wow.haha.com', port='12345'))
 
                     await proxy.clearAhaSvcProvs()
                     await proxy.clearAhaUserEnrolls()
+                    await proxy.clearAhaClones()
 
                 for url in provurls:
                     with self.raises(s_exc.NoSuchName) as cm:
@@ -792,6 +819,10 @@ async def test_lib_aha_provision(self):
                     with self.raises(s_exc.NoSuchName) as cm:
                         async with await s_telepath.openurl(url) as prox:
                             self.fail(f'Connected to an expired enrollment URL {url}')  # pragma: no cover
+                for url in clonurls:
+                    with self.raises(s_exc.NoSuchName) as cm:
+                        async with await s_telepath.openurl(url) as prox:
+                            self.fail(f'Connected to an expired clone URL {url}')  # pragma: no cover
 
     async def test_aha_httpapi(self):
 
@@ -1197,3 +1228,63 @@ async def test_aha_provision_longname(self):
                     with self.raises(s_exc.CryptoErr) as errcm:
                         await s_aha.AhaCell.anit(aha00dirn, conf=aconf)
                     self.isin('Certificate name values must be between 1-64 characters', errcm.exception.get('mesg'))
+
+    async def test_aha_prov_with_user(self):
+
+        async with self.getTestAha() as aha:
+            async with await s_base.Base.anit() as base:
+                with self.getTestDir() as dirn:
+                    user = 'synuser'
+                    dirn00 = s_common.genpath(dirn, 'cell00')
+                    dirn01 = s_common.genpath(dirn, 'cell01')
+
+                    axon00 = await base.enter_context(self.addSvcToAha(aha, '00.axon', s_axon.Axon, dirn=dirn00,
+                                                                       provinfo={'conf': {'aha:user': user}}))
+                    self.eq(axon00.conf.get('aha:user'), user)
+                    core00 = await base.enter_context(self.addSvcToAha(aha, '00.core', s_cortex.Cortex, dirn=dirn01,
+                                                                       conf={'axon': 'aha://axon...'},
+                                                                       provinfo={'conf': {'aha:user': user}}))
+                    self.eq(core00.conf.get('aha:user'), user)
+                    # Svc to svc connections use the hinted aha:user value
+                    prox = self.nn(await asyncio.wait_for(core00.axon.proxy(), timeout=12))
+                    unfo = await prox.getCellUser()
+                    self.eq(unfo.get('name'), user)
+
+    async def test_aha_cell_with_tcp(self):
+        # It's an older code, sir, but it checks out.
+        # This should be removed in Synapse v3.0.0
+
+        with self.getTestDir() as dirn:
+            ahadir = s_common.gendir(dirn, 'aha')
+            clldir = s_common.gendir(dirn, 'cell')
+            ahaconf = {
+                'aha:name': '00.aha',
+                'aha:network': 'loop.vertex.link',
+                'dmon:listen': 'tcp://127.0.0.1:0/',
+                'auth:passwd': 'secret',
+            }
+            async with await s_aha.AhaCell.anit(dirn=ahadir, conf=ahaconf) as aha:
+                urls = await aha.getAhaUrls()
+                self.len(1, urls)
+                self.true(urls[0].startswith('ssl://'))
+                ahaurl = f'tcp://root:secret@127.0.0.1:{aha.sockaddr[1]}/'
+                cllconf = {
+                    'aha:name': '00.cell',
+                    'aha:network': 'loop.vertex.link',
+                    'aha:registry': ahaurl,
+                    'dmon:listen': None,
+                }
+                async with await s_cell.Cell.anit(dirn=clldir, conf=cllconf) as cell:
+                    self.none(await cell.ahaclient.waitready(timeout=12))
+                    self.eq(cell.conf.get('aha:registry'), ahaurl)
+
+                    prox = await cell.ahaclient.proxy()
+                    await prox.fini()
+                    self.false(cell.ahaclient._t_ready.is_set())
+
+                    self.none(await cell.ahaclient.waitready(timeout=12))
+
+                # No change when restarting
+                async with await s_cell.Cell.anit(dirn=clldir, conf=cllconf) as cell:
+                    self.none(await cell.ahaclient.waitready(timeout=12))
+                    self.eq(cell.conf.get('aha:registry'), ahaurl)
diff --git a/synapse/tests/test_lib_config.py b/synapse/tests/test_lib_config.py
index f04de15b829..bc2972a8352 100644
--- a/synapse/tests/test_lib_config.py
+++ b/synapse/tests/test_lib_config.py
@@ -164,7 +164,8 @@ async def test_config_basics(self):
         # We can ensure that certain vars are loaded
         self.eq('Funky string time!', conf.req('key:string'))
         # And throw if they are not, or if the requested key isn't even schema valid
-        self.raises(s_exc.NeedConfValu, conf.req, 'key:bool:nodefval')
+        self.raises(s_exc.NeedConfValu, conf.reqConfValu, 'key:bool:nodefval')
+        self.raises(s_exc.BadArg, conf.reqConfValu, 'key:newp')
 
         # Since we're an Mutable mapping, we have some dict methods available to us
         self.len(8, conf)  # __len__
diff --git a/synapse/tests/test_tools_aha.py b/synapse/tests/test_tools_aha.py
index e3ca70d4ba0..8938a276a38 100644
--- a/synapse/tests/test_tools_aha.py
+++ b/synapse/tests/test_tools_aha.py
@@ -96,10 +96,17 @@ async def test_aha_enroll(self):
 
             argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link']
             retn, outp = await self.execToolMain(s_a_clone.main, argv)
+            self.eq(retn, 0)
             self.isin('one-time use URL:', str(outp))
 
+            argv = ['--url', aha.getLocalUrl(), '01.aha.loop.vertex.link', '--only-url']
+            retn, outp = await self.execToolMain(s_a_clone.main, argv)
+            self.eq(retn, 0)
+            self.notin('one-time use URL:', str(outp))
+
             argv = ['--url', 'newp://1.2.3.4', '01.aha.loop.vertex.link']
             retn, outp = await self.execToolMain(s_a_clone.main, argv)
+            self.eq(retn, 1)
             self.isin('ERROR: Invalid URL scheme: newp', str(outp))
 
             argv = ['--url', aha.getLocalUrl(), 'visi']
diff --git a/synapse/tests/utils.py b/synapse/tests/utils.py
index 4cb1881fe52..4608cdd884f 100644
--- a/synapse/tests/utils.py
+++ b/synapse/tests/utils.py
@@ -1429,8 +1429,16 @@ async def getTestCoreProxSvc(self, ssvc, ssvc_conf=None, core_conf=None):
                 yield core, prox, testsvc
 
     @contextlib.contextmanager
-    def mayTestDir(self, dirn):
+    def mayTestDir(self, dirn: str | None) -> contextlib.AbstractContextManager[str, None, None]:
+        '''
+        Convenience method to make a temporary directory or use an existing one.
+
+        Args:
+            dirn: Directory to use, or None.
 
+        Returns:
+            The directory to use.
+        '''
         if dirn is not None:
             yield dirn
             return
@@ -1559,7 +1567,7 @@ def getTestProxy(self, dmon, name, **kwargs):
         return s_telepath.openurl(f'tcp:///{name}', **kwargs)
 
     @contextlib.contextmanager
-    def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None):
+    def getTestDir(self, mirror=None, copyfrom=None, chdir=False, startdir=None) -> contextlib.AbstractContextManager[str, None, None]:
         '''
         Get a temporary directory for test purposes.
         This destroys the directory afterwards.

From 7a91debd2eecbfe0a585d663a1f131ee1c5e4fa3 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 30 Jul 2024 15:33:20 -0400
Subject: [PATCH 50/68] wip

---
 synapse/lib/base.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/synapse/lib/base.py b/synapse/lib/base.py
index 8040fe60c75..1fc4a19d13b 100644
--- a/synapse/lib/base.py
+++ b/synapse/lib/base.py
@@ -579,7 +579,7 @@ def sigint():
         loop.add_signal_handler(signal.SIGINT, sigint)
         loop.add_signal_handler(signal.SIGTERM, sigterm)
 
-    async def main(self):
+    async def main(self): # pragma: no cover
         '''
         Helper function to setup signal handlers for this base as the main object.
         ( use base.waitfini() to block )
@@ -791,9 +791,6 @@ async def genrtask(base):
 
             await q.put((False, None))
 
-        except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-            raise
-
         except Exception:
             if not base.isfini:
                 await q.put((False, None))

From 728da2856f1c6556f2c13283635bfdd6ec14b455 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 30 Jul 2024 15:46:30 -0400
Subject: [PATCH 51/68] wip

---
 .circleci/config.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ca8091c6cdb..fc0e859976e 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -603,7 +603,6 @@ workflows:
             branches:
               only:
                 - master
-                - visi-aha-defnet
 
       - build_docker_tag:
           requires:

From 37983b2437103c9f0062bc26dfcf28a091848b66 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 30 Jul 2024 15:47:58 -0400
Subject: [PATCH 52/68] wip

---
 synapse/lib/base.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/lib/base.py b/synapse/lib/base.py
index 1fc4a19d13b..f67b73137c7 100644
--- a/synapse/lib/base.py
+++ b/synapse/lib/base.py
@@ -408,8 +408,6 @@ async def fini(self):
         for fini in self._fini_funcs:
             try:
                 await s_coro.ornot(fini)
-            except asyncio.CancelledError:  # pragma: no cover  TODO:  remove once >= py 3.8 only
-                raise
             except Exception:
                 logger.exception(f'{self} - fini function failed: {fini}')
 
@@ -685,7 +683,9 @@ async def __aenter__(self):
 
     async def __aexit__(self, exc, cls, tb):
         if exc is None:
-            if await self.wait() is None:
+            if await self.wait() is None: # pragma: no cover
+                # these lines are 100% covered by the tests but
+                # the coverage plugin cannot seem to see them...
                 events = ','.join(self.names)
                 mesg = f'timeout waiting for {self.count} event(s): {events}'
                 raise s_exc.TimeOut(mesg=mesg)

From 8d0793030d3a32bfaeac6fa47523d6af04925e3e Mon Sep 17 00:00:00 2001
From: epiphyte <epiphyte@vertex.link>
Date: Wed, 31 Jul 2024 00:17:41 +0000
Subject: [PATCH 53/68] Remove logger statement about username hinting on aha
 lookup

---
 synapse/lib/aha.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 5a87a036c34..1bf7cdf49dd 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -153,7 +153,6 @@ async def getAhaSvc(self, name, filters=None):
         username = self.user.name.split('@')[0]
 
         if svcinfo.get('svcinfo'):
-            logger.info(f'Hinting {self.user.name} for {name}')
             svcinfo['svcinfo']['urlinfo']['user'] = username
 
         return svcinfo

From 891fc225cd373a94f998ec695de3857756a65af4 Mon Sep 17 00:00:00 2001
From: epiphyte <epiphyte@vertex.link>
Date: Wed, 31 Jul 2024 00:18:16 +0000
Subject: [PATCH 54/68] Restore ahaname logging during promote/handoff API
 calls when it is available.

---
 synapse/lib/cell.py | 38 ++++++++++++++++++++------------------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 93f04a57456..7e2c60450df 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1965,37 +1965,39 @@ async def promote(self, graceful=False):
             mesg = 'promote() called on non-mirror'
             raise s_exc.BadConfValu(mesg=mesg)
 
-        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}.')
+        _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else ''
+        logger.warning(f'PROMOTION: Performing leadership promotion graceful={graceful}{_dispname}.')
 
         if graceful:
 
             myurl = self.getMyUrl()
 
-            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff.')
+            logger.debug(f'PROMOTION: Connecting to {mirurl} to request leadership handoff{_dispname}.')
             async with await s_telepath.openurl(mirurl) as lead:
                 await lead.handoff(myurl)
-                logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}')
+                logger.warning(f'PROMOTION: Completed leadership handoff to {myurl}{_dispname}')
                 return
 
-        logger.debug(f'PROMOTION: Clearing mirror configuration.')
+        logger.debug(f'PROMOTION: Clearing mirror configuration{_dispname}.')
         self.modCellConf({'mirror': None})
 
-        logger.debug(f'PROMOTION: Promoting the nexus root.')
+        logger.debug(f'PROMOTION: Promoting the nexus root{_dispname}.')
         await self.nexsroot.promote()
 
-        logger.debug(f'PROMOTION: Setting the cell as active.')
+        logger.debug(f'PROMOTION: Setting the cell as active{_dispname}.')
         await self.setCellActive(True)
 
-        logger.warning(f'PROMOTION: Finished leadership promotion!')
+        logger.warning(f'PROMOTION: Finished leadership promotion{_dispname}.')
 
     async def handoff(self, turl, timeout=30):
         '''
         Hand off leadership to a mirror in a transactional fashion.
         '''
-        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}.')
+        _dispname = f' ahaname={self.conf.get("aha:name")}' if self.conf.get('aha:name') else ''
+        logger.warning(f'HANDOFF: Performing leadership handoff to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
         async with await s_telepath.openurl(turl) as cell:
 
-            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}.')
+            logger.debug(f'HANDOFF: Connected to {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
 
             if self.iden != await cell.getCellIden(): # pragma: no cover
                 mesg = 'Mirror handoff remote cell iden does not match!'
@@ -2005,34 +2007,34 @@ async def handoff(self, turl, timeout=30):
                 mesg = 'Cannot handoff mirror leadership to myself!'
                 raise s_exc.BadArg(mesg=mesg)
 
-            logger.debug(f'HANDOFF: Obtaining nexus lock.')
+            logger.debug(f'HANDOFF: Obtaining nexus lock{_dispname}.')
 
             async with self.nexslock:
 
-                logger.debug(f'HANDOFF: Obtained nexus lock.')
+                logger.debug(f'HANDOFF: Obtained nexus lock{_dispname}.')
                 indx = await self.getNexsIndx()
 
-                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}.')
+                logger.debug(f'HANDOFF: Waiting {timeout} seconds for mirror to reach {indx=}{_dispname}.')
                 if not await cell.waitNexsOffs(indx - 1, timeout=timeout): # pragma: no cover
                     mndx = await cell.getNexsIndx()
                     mesg = f'Remote mirror did not catch up in time: {mndx}/{indx}.'
                     raise s_exc.NotReady(mesg=mesg)
 
-                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion.')
+                logger.debug(f'HANDOFF: Mirror has caught up to the current leader, performing promotion{_dispname}.')
                 await cell.promote()
 
-                logger.debug(f'HANDOFF: Setting the service as inactive.')
+                logger.debug(f'HANDOFF: Setting the service as inactive{_dispname}.')
                 await self.setCellActive(False)
 
-                logger.debug(f'HANDOFF: Configuring service to sync from new leader.')
+                logger.debug(f'HANDOFF: Configuring service to sync from new leader{_dispname}.')
                 self.modCellConf({'mirror': turl})
 
-                logger.debug(f'HANDOFF: Restarting the nexus.')
+                logger.debug(f'HANDOFF: Restarting the nexus{_dispname}.')
                 await self.nexsroot.startup()
 
-            logger.debug(f'HANDOFF: Released nexus lock.')
+            logger.debug(f'HANDOFF: Released nexus lock{_dispname}.')
 
-        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}.')
+        logger.warning(f'HANDOFF: Done performing the leadership handoff with {s_urlhelp.sanitizeUrl(turl)}{_dispname}.')
 
     async def reqAhaProxy(self, timeout=None):
         if self.ahaclient is None:

From a2d5d0b507a39fcd43935e1023baf7a05f1eb7f8 Mon Sep 17 00:00:00 2001
From: epiphyte <epiphyte@vertex.link>
Date: Wed, 31 Jul 2024 00:25:23 +0000
Subject: [PATCH 55/68] Tweak flat network docs slightly

---
 docs/synapse/deploymentguide.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 70a703a4f10..e6fd5a0f358 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -84,7 +84,7 @@ using AHA, the only host that needs DNS or other external name resolution is the
     or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``,
     ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need
     to specify a specific ``--dmon-port`` option when using the  ``synapse.tools.aha.provision.service``
-    command to provision the service.
+    command to provision the services with static ports that you can provide mappings too.
 
 Choose an AHA Network Name
 --------------------------

From 252599d77a48a8f5de5ad70d6b67d983250d7efb Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Wed, 31 Jul 2024 11:28:00 -0400
Subject: [PATCH 56/68] changelog entries

---
 changes/06bd1eabe999e90b2b3301ee041f7017.yaml | 5 +++++
 changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml | 5 +++++
 changes/6ebc22454e67c26ce57ea4533441c9fc.yaml | 5 +++++
 changes/b652abf9758a82bb616f0037740c35e7.yaml | 5 +++++
 changes/e3b7e45e107d35ac377e867f1a8ac725.yaml | 5 +++++
 5 files changed, 25 insertions(+)
 create mode 100644 changes/06bd1eabe999e90b2b3301ee041f7017.yaml
 create mode 100644 changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml
 create mode 100644 changes/6ebc22454e67c26ce57ea4533441c9fc.yaml
 create mode 100644 changes/b652abf9758a82bb616f0037740c35e7.yaml
 create mode 100644 changes/e3b7e45e107d35ac377e867f1a8ac725.yaml

diff --git a/changes/06bd1eabe999e90b2b3301ee041f7017.yaml b/changes/06bd1eabe999e90b2b3301ee041f7017.yaml
new file mode 100644
index 00000000000..6ac21a8e68a
--- /dev/null
+++ b/changes/06bd1eabe999e90b2b3301ee041f7017.yaml
@@ -0,0 +1,5 @@
+---
+desc: Update deployment guide to include optional steps to deploy AHA mirrors.
+prs: []
+type: doc
+...
diff --git a/changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml b/changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml
new file mode 100644
index 00000000000..bd5d105ba76
--- /dev/null
+++ b/changes/46d7d7c3fd22e158bd500f03532e7b3a.yaml
@@ -0,0 +1,5 @@
+---
+desc: Updated service base class to retrieve updated AHA servers on startup.
+prs: []
+type: feat
+...
diff --git a/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml
new file mode 100644
index 00000000000..826209b9c79
--- /dev/null
+++ b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml
@@ -0,0 +1,5 @@
+---
+desc: Added ``synapse.tools.aha.clone`` command to make it easy to boostrap AHA mirrors.
+prs: []
+type: feat
+...
diff --git a/changes/b652abf9758a82bb616f0037740c35e7.yaml b/changes/b652abf9758a82bb616f0037740c35e7.yaml
new file mode 100644
index 00000000000..2340aa113eb
--- /dev/null
+++ b/changes/b652abf9758a82bb616f0037740c35e7.yaml
@@ -0,0 +1,5 @@
+---
+desc: Added support for dynamically registered AHA mirrors.
+prs: []
+type: feat
+...
diff --git a/changes/e3b7e45e107d35ac377e867f1a8ac725.yaml b/changes/e3b7e45e107d35ac377e867f1a8ac725.yaml
new file mode 100644
index 00000000000..be05f4352cf
--- /dev/null
+++ b/changes/e3b7e45e107d35ac377e867f1a8ac725.yaml
@@ -0,0 +1,5 @@
+---
+desc: Update deployment guide to clarify aha:network selection vs dns:name selection.
+prs: []
+type: doc
+...

From 26c51276aa23aef99d89b6597195250cc5baf0ff Mon Sep 17 00:00:00 2001
From: invisig0th <visi@vertex.link>
Date: Wed, 31 Jul 2024 11:46:46 -0400
Subject: [PATCH 57/68] Update changes/6ebc22454e67c26ce57ea4533441c9fc.yaml

Co-authored-by: Cisphyx <cisphyx@vertex.link>
---
 changes/6ebc22454e67c26ce57ea4533441c9fc.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml
index 826209b9c79..04f677f49f9 100644
--- a/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml
+++ b/changes/6ebc22454e67c26ce57ea4533441c9fc.yaml
@@ -1,5 +1,5 @@
 ---
-desc: Added ``synapse.tools.aha.clone`` command to make it easy to boostrap AHA mirrors.
+desc: Added ``synapse.tools.aha.clone`` command to make it easy to bootstrap AHA mirrors.
 prs: []
 type: feat
 ...

From 55d8394a41b8533dd9be67f8cb6367ef77108c2a Mon Sep 17 00:00:00 2001
From: invisig0th <visi@vertex.link>
Date: Wed, 31 Jul 2024 12:31:05 -0400
Subject: [PATCH 58/68] Update docs/synapse/deploymentguide.rst

Co-authored-by: Cisphyx <cisphyx@vertex.link>
---
 docs/synapse/deploymentguide.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index e6fd5a0f358..03b47ac38ea 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -79,7 +79,7 @@ using AHA, the only host that needs DNS or other external name resolution is the
 
     The AHA service resolver requires that registered services connect directly using IP addresses which
     must also be reachable to AHA clients. Using an ``aha://`` telepath URL requires direct routes to the
-    service via it's AHA facing network address. If you need to provide telepath access from outside the
+    service via its AHA facing network address. If you need to provide telepath access from outside the
     Synapse service network via any network address translation (NAT) method such as an inbound TCP proxy
     or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``,
     ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need

From 7da0c6e6d9cea9fcf01ff28f182766ff5b7d8711 Mon Sep 17 00:00:00 2001
From: invisig0th <visi@vertex.link>
Date: Wed, 31 Jul 2024 12:31:17 -0400
Subject: [PATCH 59/68] Update docs/synapse/deploymentguide.rst

Co-authored-by: Cisphyx <cisphyx@vertex.link>
---
 docs/synapse/deploymentguide.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 03b47ac38ea..8c169de32a1 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -84,7 +84,7 @@ using AHA, the only host that needs DNS or other external name resolution is the
     or docker/kubernetes port mapping, you will need to use ``ssl://`` based URIs and specify ``hostname``,
     ``certname``, and ``ca`` parameters which match the service's AHA registration info. You will also need
     to specify a specific ``--dmon-port`` option when using the  ``synapse.tools.aha.provision.service``
-    command to provision the services with static ports that you can provide mappings too.
+    command to provision the services with static ports that you can provide mappings to.
 
 Choose an AHA Network Name
 --------------------------

From e9af9a338c38890c1e832cdc1d7b6db405335b36 Mon Sep 17 00:00:00 2001
From: invisig0th <visi@vertex.link>
Date: Wed, 31 Jul 2024 12:31:40 -0400
Subject: [PATCH 60/68] Update synapse/lib/cell.py

Co-authored-by: Cisphyx <cisphyx@vertex.link>
---
 synapse/lib/cell.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 7e2c60450df..fa3ce5b401b 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1688,7 +1688,7 @@ async def onlink(proxy):
                 if newurls and newurls != oldurls:
                     if oldurls[0].startswith('tcp://'):
                         s_common.deprecated('aha:registry: tcp:// client values.')
-                        logger.warning('tcp:// based aha:registry options will be deprecated in Synapse v3.0.0')
+                        logger.warning('tcp:// based aha:registry options are deprecated and will be removed in Synapse v3.0.0')
                         return
 
                     self.modCellConf({'aha:registry': newurls})

From 502d23d53c6e7149c3e74434b6dafb1b255101d3 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Thu, 1 Aug 2024 07:29:02 -0400
Subject: [PATCH 61/68] wip

---
 docs/synapse/deploymentguide.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 8c169de32a1..08d60f411f4 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -175,8 +175,9 @@ For this example, we will assume you chose a DNS name for your primary AHA serve
 listed above. If so, you can simply replace ``00`` with sequential numbers and repeat this step to deploy
 however many AHA mirrors you deem appropriate.
 
-NOTE: AHA uses two default ports ETC. The following steps assume you will be running each of your AHA servers
-on a different host. The use of ``network_mode; host`` ETC
+By default, AHA uses port ``27492`` to listen for RPC connections from other Synapse services and port ``27272``
+for the provisioning listener. The following example steps assume you will be running each AHA server on separate
+hosts or in a containerized to avoid port collisions.
 
 **Inside the AHA container**
 

From 37a7a8505eced1f5b4489be90e76c4f312cdf5b6 Mon Sep 17 00:00:00 2001
From: invisig0th <visi@vertex.link>
Date: Fri, 2 Aug 2024 15:38:18 -0400
Subject: [PATCH 62/68] Update docs/synapse/deploymentguide.rst

Co-authored-by: Cisphyx <cisphyx@vertex.link>
---
 docs/synapse/deploymentguide.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/synapse/deploymentguide.rst b/docs/synapse/deploymentguide.rst
index 08d60f411f4..e8a9b9d0ce0 100644
--- a/docs/synapse/deploymentguide.rst
+++ b/docs/synapse/deploymentguide.rst
@@ -177,7 +177,7 @@ however many AHA mirrors you deem appropriate.
 
 By default, AHA uses port ``27492`` to listen for RPC connections from other Synapse services and port ``27272``
 for the provisioning listener. The following example steps assume you will be running each AHA server on separate
-hosts or in a containerized to avoid port collisions.
+hosts or in a containerized deployment to avoid port collisions.
 
 **Inside the AHA container**
 

From 221c8237bc21ebbe70a5a696c5a22869aa01bf4c Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 5 Aug 2024 10:05:26 -0400
Subject: [PATCH 63/68] wip

---
 synapse/lib/aha.py | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 1bf7cdf49dd..be454b04011 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -1159,19 +1159,8 @@ async def genCaCert(self, network):
 
         return cacert
 
-    async def _genCaCert(self, network):
-
-        # generate a CA cert if one does not exist
-        if self.certdir.getCaCertPath(network) is not None:
-            return
-
-        await self.genCaCert(network)
-
     async def _genHostCert(self, hostname, signas=None):
 
-        if signas is not None:
-            await self._genCaCert(signas)
-
         if self.certdir.getHostCertPath(hostname) is not None:
             return
 

From 052e0634be18ffbc0656d7bf967ca835e03f377c Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 6 Aug 2024 08:24:58 -0400
Subject: [PATCH 64/68] wip

---
 synapse/lib/aha.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 6df5ff2863d..4b8beec5dbe 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -1272,11 +1272,14 @@ async def getLeadTerm(self, iden):
         if byts is not None:
             return s_msgpack.un(byts)
 
-    @s_cell.from_leader()
     async def setLeadTerm(self, termdef):
         '''
         Set the current leader term for the specified service cluster.
         '''
+        if not self.isactive:
+            mesg = 'setLeadTerm() may only be called on the leader.'
+            raise s_exc.BadState(mesg=mesg)
+
         termdef['time'] = s_common.now()
         s_schema.reqValidLeadTerm(termdef)
         return await self._push('aha:lead:set', termdef)
@@ -1305,7 +1308,7 @@ async def nextLeadTerm(self, iden, name, nexs):
                 'term': term,
                 'nexs': nexs,
             }
-            return await self.addLeadTerm(leadterm)
+            return await self.setLeadTerm(leadterm)
 
     @s_cell.from_leader()
     async def mayLeadTerm(self, iden, name, term, nexs):
@@ -1330,7 +1333,7 @@ async def mayLeadTerm(self, iden, name, term, nexs):
                     'term': term,
                     'nexs': nexs,
                 }
-                await self.addLeadTerm(leadterm)
+                await self.setLeadTerm(leadterm)
                 return (STATE_LEAD, leadterm)
 
             # if we were the last known leader, we can jump right in

From 6e4d89fdc2b50ed6a990f759c22c72c087e9187e Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Tue, 6 Aug 2024 12:24:36 -0400
Subject: [PATCH 65/68] wip

---
 synapse/lib/cell.py | 31 ++++++++++++++++++++++---------
 1 file changed, 22 insertions(+), 9 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 35f958c0a57..f4d4d6bc477 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1697,7 +1697,7 @@ async def _askAhaToLead(self, proxy):
                 state, leadterm = await proxy.mayLeadTerm(self.iden, name, term, nexs)
 
                 realterm = leadterm.get('term')
-                if realterm > term:
+                if realterm > term or self.getCellMeta('aha:term') is None:
                     self.setCellMeta('aha:term', realterm)
 
                 if state == s_aha.STATE_LEAD:
@@ -1721,9 +1721,24 @@ async def _termStateInvalid(self):
         # TODO: should we find the leader and re-provision?
         await self.fini()
 
+    def getAhaRegistry(self):
+
+        urls = self.conf.get('aha:registry')
+        if isinstance(urls, str):
+            return (urls,)
+
+        if isinstance(urls, list):
+            urls = tuple(urls)
+
+        return urls
+
+    def setAhaRegistry(self, urls):
+        self.modCellConf({'aha:registry': urls})
+        self.ahaclient.setBootUrls(urls)
+
     async def _initAhaRegistry(self):
 
-        ahaurls = self.conf.get('aha:registry')
+        ahaurls = self.getAhaRegistry()
         if ahaurls is not None:
 
             await s_telepath.addAhaUrl(ahaurls)
@@ -1731,21 +1746,19 @@ async def _initAhaRegistry(self):
                 await self.ahaclient.fini()
 
             async def onlink(proxy):
+
                 ahauser = self.conf.get('aha:user', 'root')
+
+                oldurls = self.getAhaRegistry()
                 newurls = await proxy.getAhaUrls(user=ahauser)
-                oldurls = self.conf.get('aha:registry')
-                if isinstance(oldurls, str):
-                    oldurls = (oldurls,)
-                elif isinstance(oldurls, list):
-                    oldurls = tuple(oldurls)
+
                 if newurls and newurls != oldurls:
                     if oldurls[0].startswith('tcp://'):
                         s_common.deprecated('aha:registry: tcp:// client values.')
                         logger.warning('tcp:// based aha:registry options are deprecated and will be removed in Synapse v3.0.0')
                         return
 
-                    self.modCellConf({'aha:registry': newurls})
-                    self.ahaclient.setBootUrls(newurls)
+                    self.setAhaRegistry(newurls)
 
                 if self.conf.get('mirror') is None:
 

From d1e0fdb4780d3df29afbf591f2c3a312f1fda209 Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 31 Jan 2025 14:10:24 -0500
Subject: [PATCH 66/68] wip

---
 synapse/lib/aha.py  | 6 ++----
 synapse/lib/cell.py | 7 +++++++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index 9a18f4bc627..acf817331f7 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -591,6 +591,7 @@ async def _initCellBoot(self):
     async def initServiceStorage(self):
 
         self.features['callpeers'] = 1
+        self.features['leadterms'] = 1
 
         dirn = s_common.gendir(self.dirn, 'slabs', 'jsonstor')
 
@@ -1430,14 +1431,11 @@ async def getLeadTerm(self, iden):
         if byts is not None:
             return s_msgpack.un(byts)
 
+    @s_cell.from_lead()
     async def setLeadTerm(self, termdef):
         '''
         Set the current leader term for the specified service cluster.
         '''
-        if not self.isactive:
-            mesg = 'setLeadTerm() may only be called on the leader.'
-            raise s_exc.BadState(mesg=mesg)
-
         termdef['time'] = s_common.now()
         s_schema.reqValidLeadTerm(termdef)
         return await self._push('aha:lead:set', termdef)
diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 33e1a40cc4e..7b8e4d8ee83 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1753,10 +1753,16 @@ async def _runSysctlLoop(self):
     async def _askAhaToLead(self, proxy):
 
         while not self.isfini:
+
             try:
                 proxy = self.ahaclient.proxy(timeout=4)
             except Exception as e:
                 logger.error(f'Error getting AHA proxy to check leadership: {e}')
+                continue
+
+            if not proxy._hasTeleFeat('leadterms'):
+                logger.warning('AHA Server does not support tracking leadership terms. Please update!')
+                return
 
             async with self.nexslock:
 
@@ -1782,6 +1788,7 @@ async def _askAhaToLead(self, proxy):
 
                 if state == s_aha.STATE_INVALID:
                     await self._termStateInvalid()
+                    return
 
     async def _termStateInvalid(self):
         # hook point for enterprise behavior

From cd2db86e60b8323cc95814481890837de4dda77c Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Fri, 31 Jan 2025 14:19:13 -0500
Subject: [PATCH 67/68] wip

---
 synapse/lib/cell.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/synapse/lib/cell.py b/synapse/lib/cell.py
index 7b8e4d8ee83..c2c565abb76 100644
--- a/synapse/lib/cell.py
+++ b/synapse/lib/cell.py
@@ -1838,10 +1838,7 @@ async def onlink(proxy):
                     self.setAhaRegistry(newurls)
 
                 if self.conf.get('mirror') is None:
-
-                    # if i think i'm the leader, lets check with AHA
-                    ahainfo = await proxy.getCellInfo()
-                    if ahainfo['features'].get('aha:leadterm'):
+                    await self._askAhaToLead()
 
             self.ahaclient = await s_telepath.Client.anit(ahaurls, onlink=onlink)
             self.onfini(self.ahaclient)

From f555ea4749ebab15a5bab36fb981b81cad8ac95d Mon Sep 17 00:00:00 2001
From: visi <visi@vertex.link>
Date: Mon, 3 Feb 2025 09:11:50 -0500
Subject: [PATCH 68/68] wip

---
 synapse/lib/aha.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/synapse/lib/aha.py b/synapse/lib/aha.py
index acf817331f7..9e64edb26e9 100644
--- a/synapse/lib/aha.py
+++ b/synapse/lib/aha.py
@@ -1431,7 +1431,7 @@ async def getLeadTerm(self, iden):
         if byts is not None:
             return s_msgpack.un(byts)
 
-    @s_cell.from_lead()
+    @s_cell.from_leader
     async def setLeadTerm(self, termdef):
         '''
         Set the current leader term for the specified service cluster.
@@ -1446,7 +1446,7 @@ async def _setLeadTerm(self, termdef):
         lkey = s_common.uhex(termdef.get('iden'))
         self.slab.put(b'\x00' + lkey, s_msgpack.en(termdef), db='aha:leadterms')
 
-    @s_cell.from_leader()
+    @s_cell.from_leader
     async def nextLeadTerm(self, iden, name, nexs):
         '''
         Force a change in leadership for the given cluster iden.
@@ -1466,7 +1466,7 @@ async def nextLeadTerm(self, iden, name, nexs):
             }
             return await self.setLeadTerm(leadterm)
 
-    @s_cell.from_leader()
+    @s_cell.from_leader
     async def mayLeadTerm(self, iden, name, term, nexs):
         '''
         Determine if the caller is still the valid leader.